diff --git a/.travis.yml b/.travis.yml index 9b51e7983..cd5b45c10 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,5 +18,5 @@ script: - mvn clean install after_success: - - mvn clean cobertura:cobertura coveralls:report + - mvn clean test jacoco:report coveralls:report - bash ./travis/push_image.sh diff --git a/apm-collector/apm-collector-agentjvm/pom.xml b/apm-collector/apm-collector-agentjvm/pom.xml new file mode 100644 index 000000000..8f20f17ab --- /dev/null +++ b/apm-collector/apm-collector-agentjvm/pom.xml @@ -0,0 +1,32 @@ + + + + apm-collector + org.skywalking + 3.2-2017 + + 4.0.0 + + apm-collector-agentjvm + jar + + + + org.skywalking + apm-collector-cluster + ${project.version} + + + org.skywalking + apm-collector-server + ${project.version} + + + org.skywalking + apm-network + ${project.version} + + + \ No newline at end of file diff --git a/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/AgentJVMModuleContext.java b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/AgentJVMModuleContext.java new file mode 100644 index 000000000..5152b68d6 --- /dev/null +++ b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/AgentJVMModuleContext.java @@ -0,0 +1,12 @@ +package org.skywalking.apm.collector.agentjvm; + +import org.skywalking.apm.collector.core.framework.Context; + +/** + * @author pengys5 + */ +public class AgentJVMModuleContext extends Context { + public AgentJVMModuleContext(String groupName) { + super(groupName); + } +} diff --git a/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/AgentJVMModuleDefine.java b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/AgentJVMModuleDefine.java new file mode 100644 index 000000000..29826dd6a --- /dev/null +++ b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/AgentJVMModuleDefine.java @@ -0,0 +1,51 @@ +package org.skywalking.apm.collector.agentjvm; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.cluster.ClusterModuleGroupDefine; +import org.skywalking.apm.collector.core.client.Client; +import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.client.DataMonitor; +import org.skywalking.apm.collector.core.cluster.ClusterDataListenerDefine; +import org.skywalking.apm.collector.core.cluster.ClusterModuleContext; +import org.skywalking.apm.collector.core.config.ConfigParseException; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.framework.Handler; +import org.skywalking.apm.collector.core.module.ModuleDefine; +import org.skywalking.apm.collector.core.server.Server; +import org.skywalking.apm.collector.core.server.ServerException; +import org.skywalking.apm.collector.core.server.ServerHolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public abstract class AgentJVMModuleDefine extends ModuleDefine implements ClusterDataListenerDefine { + + private final Logger logger = LoggerFactory.getLogger(AgentJVMModuleDefine.class); + + @Override + public final void initialize(Map config, ServerHolder serverHolder) throws DefineException, ClientException { + try { + configParser().parse(config); + Server server = server(); + serverHolder.holdServer(server, handlerList()); + + ((ClusterModuleContext)CollectorContextHelper.INSTANCE.getContext(ClusterModuleGroupDefine.GROUP_NAME)).getDataMonitor().addListener(listener(), registration()); + } catch (ConfigParseException | ServerException e) { + throw new AgentJVMModuleException(e.getMessage(), e); + } + } + + @Override protected final Client createClient(DataMonitor dataMonitor) { + throw new UnsupportedOperationException(""); + } + + @Override public final boolean defaultModule() { + return true; + } + + public abstract List handlerList(); +} diff --git a/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/AgentJVMModuleException.java b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/AgentJVMModuleException.java new file mode 100644 index 000000000..6732f2636 --- /dev/null +++ b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/AgentJVMModuleException.java @@ -0,0 +1,16 @@ +package org.skywalking.apm.collector.agentjvm; + +import org.skywalking.apm.collector.core.module.ModuleException; + +/** + * @author pengys5 + */ +public class AgentJVMModuleException extends ModuleException { + public AgentJVMModuleException(String message) { + super(message); + } + + public AgentJVMModuleException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/AgentJVMModuleGroupDefine.java b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/AgentJVMModuleGroupDefine.java new file mode 100644 index 000000000..89b942290 --- /dev/null +++ b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/AgentJVMModuleGroupDefine.java @@ -0,0 +1,25 @@ +package org.skywalking.apm.collector.agentjvm; + +import org.skywalking.apm.collector.core.framework.Context; +import org.skywalking.apm.collector.core.module.ModuleGroupDefine; +import org.skywalking.apm.collector.core.module.ModuleInstaller; + +/** + * @author pengys5 + */ +public class AgentJVMModuleGroupDefine implements ModuleGroupDefine { + + public static final String GROUP_NAME = "agent_jvm"; + + @Override public String name() { + return GROUP_NAME; + } + + @Override public Context groupContext() { + return new AgentJVMModuleContext(GROUP_NAME); + } + + @Override public ModuleInstaller moduleInstaller() { + return new AgentJVMModuleInstaller(); + } +} diff --git a/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/AgentJVMModuleInstaller.java b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/AgentJVMModuleInstaller.java new file mode 100644 index 000000000..defe8786e --- /dev/null +++ b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/AgentJVMModuleInstaller.java @@ -0,0 +1,36 @@ +package org.skywalking.apm.collector.agentjvm; + +import java.util.Iterator; +import java.util.Map; +import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.module.ModuleDefine; +import org.skywalking.apm.collector.core.module.ModuleInstaller; +import org.skywalking.apm.collector.core.server.ServerHolder; +import org.skywalking.apm.collector.core.util.ObjectUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class AgentJVMModuleInstaller implements ModuleInstaller { + + private final Logger logger = LoggerFactory.getLogger(AgentJVMModuleInstaller.class); + + @Override public void install(Map moduleConfig, + Map moduleDefineMap, ServerHolder serverHolder) throws DefineException, ClientException { + logger.info("beginning agent jvm module install"); + + AgentJVMModuleContext context = new AgentJVMModuleContext(AgentJVMModuleGroupDefine.GROUP_NAME); + CollectorContextHelper.INSTANCE.putContext(context); + + Iterator> moduleDefineEntry = moduleDefineMap.entrySet().iterator(); + while (moduleDefineEntry.hasNext()) { + ModuleDefine moduleDefine = moduleDefineEntry.next().getValue(); + logger.info("module {} initialize", moduleDefine.getClass().getName()); + moduleDefine.initialize((ObjectUtils.isNotEmpty(moduleConfig) && moduleConfig.containsKey(moduleDefine.name())) ? moduleConfig.get(moduleDefine.name()) : null, serverHolder); + } + } +} diff --git a/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/grpc/AgentJVMGRPCConfig.java b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/grpc/AgentJVMGRPCConfig.java new file mode 100644 index 000000000..1376814cd --- /dev/null +++ b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/grpc/AgentJVMGRPCConfig.java @@ -0,0 +1,9 @@ +package org.skywalking.apm.collector.agentjvm.grpc; + +/** + * @author pengys5 + */ +public class AgentJVMGRPCConfig { + public static String HOST; + public static int PORT; +} diff --git a/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/grpc/AgentJVMGRPCConfigParser.java b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/grpc/AgentJVMGRPCConfigParser.java new file mode 100644 index 000000000..f66ea0a8a --- /dev/null +++ b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/grpc/AgentJVMGRPCConfigParser.java @@ -0,0 +1,30 @@ +package org.skywalking.apm.collector.agentjvm.grpc; + +import java.util.Map; +import org.skywalking.apm.collector.core.config.ConfigParseException; +import org.skywalking.apm.collector.core.module.ModuleConfigParser; +import org.skywalking.apm.collector.core.util.ObjectUtils; +import org.skywalking.apm.collector.core.util.StringUtils; + +/** + * @author pengys5 + */ +public class AgentJVMGRPCConfigParser implements ModuleConfigParser { + + private static final String HOST = "host"; + private static final String PORT = "port"; + + @Override public void parse(Map config) throws ConfigParseException { + if (ObjectUtils.isEmpty(config) || StringUtils.isEmpty(config.get(HOST))) { + AgentJVMGRPCConfig.HOST = "localhost"; + } else { + AgentJVMGRPCConfig.HOST = (String)config.get(HOST); + } + + if (ObjectUtils.isEmpty(config) || StringUtils.isEmpty(config.get(PORT))) { + AgentJVMGRPCConfig.PORT = 11800; + } else { + AgentJVMGRPCConfig.PORT = (Integer)config.get(PORT); + } + } +} diff --git a/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/grpc/AgentJVMGRPCDataListener.java b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/grpc/AgentJVMGRPCDataListener.java new file mode 100644 index 000000000..b25fcdb8d --- /dev/null +++ b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/grpc/AgentJVMGRPCDataListener.java @@ -0,0 +1,21 @@ +package org.skywalking.apm.collector.agentjvm.grpc; + +import org.skywalking.apm.collector.agentjvm.AgentJVMModuleGroupDefine; +import org.skywalking.apm.collector.cluster.ClusterModuleDefine; +import org.skywalking.apm.collector.core.cluster.ClusterDataListener; + +/** + * @author pengys5 + */ +public class AgentJVMGRPCDataListener extends ClusterDataListener { + + public static final String PATH = ClusterModuleDefine.BASE_CATALOG + "." + AgentJVMModuleGroupDefine.GROUP_NAME + "." + AgentJVMGRPCModuleDefine.MODULE_NAME; + + @Override public String path() { + return PATH; + } + + @Override public void addressChangedNotify() { + + } +} diff --git a/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/grpc/AgentJVMGRPCModuleDefine.java b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/grpc/AgentJVMGRPCModuleDefine.java new file mode 100644 index 000000000..358adedfb --- /dev/null +++ b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/grpc/AgentJVMGRPCModuleDefine.java @@ -0,0 +1,51 @@ +package org.skywalking.apm.collector.agentjvm.grpc; + +import java.util.LinkedList; +import java.util.List; +import org.skywalking.apm.collector.agentjvm.AgentJVMModuleDefine; +import org.skywalking.apm.collector.agentjvm.AgentJVMModuleGroupDefine; +import org.skywalking.apm.collector.agentjvm.grpc.handler.JVMMetricsServiceHandler; +import org.skywalking.apm.collector.core.cluster.ClusterDataListener; +import org.skywalking.apm.collector.core.framework.Handler; +import org.skywalking.apm.collector.core.module.ModuleConfigParser; +import org.skywalking.apm.collector.core.module.ModuleRegistration; +import org.skywalking.apm.collector.core.server.Server; +import org.skywalking.apm.collector.server.grpc.GRPCServer; + +/** + * @author pengys5 + */ +public class AgentJVMGRPCModuleDefine extends AgentJVMModuleDefine { + + public static final String MODULE_NAME = "grpc"; + + @Override protected String group() { + return AgentJVMModuleGroupDefine.GROUP_NAME; + } + + @Override public String name() { + return MODULE_NAME; + } + + @Override protected ModuleConfigParser configParser() { + return new AgentJVMGRPCConfigParser(); + } + + @Override protected Server server() { + return new GRPCServer(AgentJVMGRPCConfig.HOST, AgentJVMGRPCConfig.PORT); + } + + @Override protected ModuleRegistration registration() { + return new AgentJVMGRPCModuleRegistration(); + } + + @Override public ClusterDataListener listener() { + return new AgentJVMGRPCDataListener(); + } + + @Override public List handlerList() { + List handlers = new LinkedList<>(); + handlers.add(new JVMMetricsServiceHandler()); + return handlers; + } +} diff --git a/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/grpc/AgentJVMGRPCModuleRegistration.java b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/grpc/AgentJVMGRPCModuleRegistration.java new file mode 100644 index 000000000..ca249dea7 --- /dev/null +++ b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/grpc/AgentJVMGRPCModuleRegistration.java @@ -0,0 +1,13 @@ +package org.skywalking.apm.collector.agentjvm.grpc; + +import org.skywalking.apm.collector.core.module.ModuleRegistration; + +/** + * @author pengys5 + */ +public class AgentJVMGRPCModuleRegistration extends ModuleRegistration { + + @Override public Value buildValue() { + return new Value(AgentJVMGRPCConfig.HOST, AgentJVMGRPCConfig.PORT, null); + } +} diff --git a/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/grpc/handler/JVMMetricsServiceHandler.java b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/grpc/handler/JVMMetricsServiceHandler.java new file mode 100644 index 000000000..7166b980d --- /dev/null +++ b/apm-collector/apm-collector-agentjvm/src/main/java/org/skywalking/apm/collector/agentjvm/grpc/handler/JVMMetricsServiceHandler.java @@ -0,0 +1,17 @@ +package org.skywalking.apm.collector.agentjvm.grpc.handler; + +import io.grpc.stub.StreamObserver; +import org.skywalking.apm.collector.server.grpc.GRPCHandler; +import org.skywalking.apm.network.proto.Downstream; +import org.skywalking.apm.network.proto.JVMMetrics; +import org.skywalking.apm.network.proto.JVMMetricsServiceGrpc; + +/** + * @author pengys5 + */ +public class JVMMetricsServiceHandler extends JVMMetricsServiceGrpc.JVMMetricsServiceImplBase implements GRPCHandler { + + @Override public void collect(JVMMetrics request, StreamObserver responseObserver) { + super.collect(request, responseObserver); + } +} diff --git a/apm-collector/apm-collector-agentjvm/src/main/resources/META-INF/defines/group.define b/apm-collector/apm-collector-agentjvm/src/main/resources/META-INF/defines/group.define new file mode 100644 index 000000000..0b2741b98 --- /dev/null +++ b/apm-collector/apm-collector-agentjvm/src/main/resources/META-INF/defines/group.define @@ -0,0 +1 @@ +org.skywalking.apm.collector.agentjvm.AgentJVMModuleGroupDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-agentjvm/src/main/resources/META-INF/defines/module.define b/apm-collector/apm-collector-agentjvm/src/main/resources/META-INF/defines/module.define new file mode 100644 index 000000000..78d6e4a41 --- /dev/null +++ b/apm-collector/apm-collector-agentjvm/src/main/resources/META-INF/defines/module.define @@ -0,0 +1 @@ +org.skywalking.apm.collector.agentjvm.grpc.AgentJVMGRPCModuleDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-agentregister/pom.xml b/apm-collector/apm-collector-agentregister/pom.xml new file mode 100644 index 000000000..258ffa456 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/pom.xml @@ -0,0 +1,37 @@ + + + + apm-collector + org.skywalking + 3.2-2017 + + 4.0.0 + + apm-collector-agentregister + jar + + + + org.skywalking + apm-collector-cluster + ${project.version} + + + org.skywalking + apm-collector-server + ${project.version} + + + org.skywalking + apm-network + ${project.version} + + + org.skywalking + apm-collector-agentstream + ${project.version} + + + \ No newline at end of file diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/AgentRegisterModuleContext.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/AgentRegisterModuleContext.java new file mode 100644 index 000000000..0bea80ed0 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/AgentRegisterModuleContext.java @@ -0,0 +1,12 @@ +package org.skywalking.apm.collector.agentregister; + +import org.skywalking.apm.collector.core.framework.Context; + +/** + * @author pengys5 + */ +public class AgentRegisterModuleContext extends Context { + public AgentRegisterModuleContext(String groupName) { + super(groupName); + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/AgentRegisterModuleDefine.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/AgentRegisterModuleDefine.java new file mode 100644 index 000000000..976e78472 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/AgentRegisterModuleDefine.java @@ -0,0 +1,51 @@ +package org.skywalking.apm.collector.agentregister; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.cluster.ClusterModuleGroupDefine; +import org.skywalking.apm.collector.core.client.Client; +import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.client.DataMonitor; +import org.skywalking.apm.collector.core.cluster.ClusterDataListenerDefine; +import org.skywalking.apm.collector.core.cluster.ClusterModuleContext; +import org.skywalking.apm.collector.core.config.ConfigParseException; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.framework.Handler; +import org.skywalking.apm.collector.core.module.ModuleDefine; +import org.skywalking.apm.collector.core.server.Server; +import org.skywalking.apm.collector.core.server.ServerException; +import org.skywalking.apm.collector.core.server.ServerHolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public abstract class AgentRegisterModuleDefine extends ModuleDefine implements ClusterDataListenerDefine { + + private final Logger logger = LoggerFactory.getLogger(AgentRegisterModuleDefine.class); + + @Override + public final void initialize(Map config, ServerHolder serverHolder) throws DefineException, ClientException { + try { + configParser().parse(config); + Server server = server(); + serverHolder.holdServer(server, handlerList()); + + ((ClusterModuleContext)CollectorContextHelper.INSTANCE.getContext(ClusterModuleGroupDefine.GROUP_NAME)).getDataMonitor().addListener(listener(), registration()); + } catch (ConfigParseException | ServerException e) { + throw new AgentRegisterModuleException(e.getMessage(), e); + } + } + + @Override protected final Client createClient(DataMonitor dataMonitor) { + throw new UnsupportedOperationException(""); + } + + @Override public final boolean defaultModule() { + return true; + } + + public abstract List handlerList(); +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/AgentRegisterModuleException.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/AgentRegisterModuleException.java new file mode 100644 index 000000000..061937a7e --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/AgentRegisterModuleException.java @@ -0,0 +1,16 @@ +package org.skywalking.apm.collector.agentregister; + +import org.skywalking.apm.collector.core.module.ModuleException; + +/** + * @author pengys5 + */ +public class AgentRegisterModuleException extends ModuleException { + public AgentRegisterModuleException(String message) { + super(message); + } + + public AgentRegisterModuleException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/AgentRegisterModuleGroupDefine.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/AgentRegisterModuleGroupDefine.java new file mode 100644 index 000000000..d59aa5b85 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/AgentRegisterModuleGroupDefine.java @@ -0,0 +1,25 @@ +package org.skywalking.apm.collector.agentregister; + +import org.skywalking.apm.collector.core.framework.Context; +import org.skywalking.apm.collector.core.module.ModuleGroupDefine; +import org.skywalking.apm.collector.core.module.ModuleInstaller; + +/** + * @author pengys5 + */ +public class AgentRegisterModuleGroupDefine implements ModuleGroupDefine { + + public static final String GROUP_NAME = "agent_register"; + + @Override public String name() { + return GROUP_NAME; + } + + @Override public Context groupContext() { + return new AgentRegisterModuleContext(GROUP_NAME); + } + + @Override public ModuleInstaller moduleInstaller() { + return new AgentRegisterModuleInstaller(); + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/AgentRegisterModuleInstaller.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/AgentRegisterModuleInstaller.java new file mode 100644 index 000000000..0bf2218d1 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/AgentRegisterModuleInstaller.java @@ -0,0 +1,36 @@ +package org.skywalking.apm.collector.agentregister; + +import java.util.Iterator; +import java.util.Map; +import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.module.ModuleDefine; +import org.skywalking.apm.collector.core.module.ModuleInstaller; +import org.skywalking.apm.collector.core.server.ServerHolder; +import org.skywalking.apm.collector.core.util.ObjectUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class AgentRegisterModuleInstaller implements ModuleInstaller { + + private final Logger logger = LoggerFactory.getLogger(AgentRegisterModuleInstaller.class); + + @Override public void install(Map moduleConfig, + Map moduleDefineMap, ServerHolder serverHolder) throws DefineException, ClientException { + logger.info("beginning agent register module install"); + + AgentRegisterModuleContext context = new AgentRegisterModuleContext(AgentRegisterModuleGroupDefine.GROUP_NAME); + CollectorContextHelper.INSTANCE.putContext(context); + + Iterator> moduleDefineEntry = moduleDefineMap.entrySet().iterator(); + while (moduleDefineEntry.hasNext()) { + ModuleDefine moduleDefine = moduleDefineEntry.next().getValue(); + logger.info("module {} initialize", moduleDefine.getClass().getName()); + moduleDefine.initialize((ObjectUtils.isNotEmpty(moduleConfig) && moduleConfig.containsKey(moduleDefine.name())) ? moduleConfig.get(moduleDefine.name()) : null, serverHolder); + } + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDService.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDService.java new file mode 100644 index 000000000..2cf836e90 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDService.java @@ -0,0 +1,37 @@ +package org.skywalking.apm.collector.agentregister.application; + +import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine; +import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterRemoteWorker; +import org.skywalking.apm.collector.agentstream.worker.register.application.dao.IApplicationDAO; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.stream.StreamModuleContext; +import org.skywalking.apm.collector.stream.StreamModuleGroupDefine; +import org.skywalking.apm.collector.stream.worker.WorkerInvokeException; +import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class ApplicationIDService { + + private final Logger logger = LoggerFactory.getLogger(ApplicationIDService.class); + + public int getOrCreate(String applicationCode) { + IApplicationDAO dao = (IApplicationDAO)DAOContainer.INSTANCE.get(IApplicationDAO.class.getName()); + int applicationId = dao.getApplicationId(applicationCode); + + if (applicationId == 0) { + StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); + ApplicationDataDefine.Application application = new ApplicationDataDefine.Application(applicationCode, applicationCode, 0); + try { + context.getClusterWorkerContext().lookup(ApplicationRegisterRemoteWorker.WorkerRole.INSTANCE).tell(application); + } catch (WorkerNotFoundException | WorkerInvokeException e) { + logger.error(e.getMessage(), e); + } + } + return applicationId; + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/AgentRegisterGRPCConfig.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/AgentRegisterGRPCConfig.java new file mode 100644 index 000000000..2d2ad6bc7 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/AgentRegisterGRPCConfig.java @@ -0,0 +1,9 @@ +package org.skywalking.apm.collector.agentregister.grpc; + +/** + * @author pengys5 + */ +public class AgentRegisterGRPCConfig { + public static String HOST; + public static int PORT; +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/AgentRegisterGRPCConfigParser.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/AgentRegisterGRPCConfigParser.java new file mode 100644 index 000000000..90f0d39da --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/AgentRegisterGRPCConfigParser.java @@ -0,0 +1,30 @@ +package org.skywalking.apm.collector.agentregister.grpc; + +import java.util.Map; +import org.skywalking.apm.collector.core.config.ConfigParseException; +import org.skywalking.apm.collector.core.module.ModuleConfigParser; +import org.skywalking.apm.collector.core.util.ObjectUtils; +import org.skywalking.apm.collector.core.util.StringUtils; + +/** + * @author pengys5 + */ +public class AgentRegisterGRPCConfigParser implements ModuleConfigParser { + + private static final String HOST = "host"; + private static final String PORT = "port"; + + @Override public void parse(Map config) throws ConfigParseException { + if (ObjectUtils.isEmpty(config) || StringUtils.isEmpty(config.get(HOST))) { + AgentRegisterGRPCConfig.HOST = "localhost"; + } else { + AgentRegisterGRPCConfig.HOST = (String)config.get(HOST); + } + + if (ObjectUtils.isEmpty(config) || StringUtils.isEmpty(config.get(PORT))) { + AgentRegisterGRPCConfig.PORT = 11800; + } else { + AgentRegisterGRPCConfig.PORT = (Integer)config.get(PORT); + } + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/AgentRegisterGRPCDataListener.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/AgentRegisterGRPCDataListener.java new file mode 100644 index 000000000..6ff6004d9 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/AgentRegisterGRPCDataListener.java @@ -0,0 +1,20 @@ +package org.skywalking.apm.collector.agentregister.grpc; + +import org.skywalking.apm.collector.agentregister.AgentRegisterModuleGroupDefine; +import org.skywalking.apm.collector.cluster.ClusterModuleDefine; +import org.skywalking.apm.collector.core.cluster.ClusterDataListener; + +/** + * @author pengys5 + */ +public class AgentRegisterGRPCDataListener extends ClusterDataListener { + + public static final String PATH = ClusterModuleDefine.BASE_CATALOG + "." + AgentRegisterModuleGroupDefine.GROUP_NAME + "." + AgentRegisterGRPCModuleDefine.MODULE_NAME; + + @Override public String path() { + return PATH; + } + + @Override public void addressChangedNotify() { + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/AgentRegisterGRPCModuleDefine.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/AgentRegisterGRPCModuleDefine.java new file mode 100644 index 000000000..509009c37 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/AgentRegisterGRPCModuleDefine.java @@ -0,0 +1,55 @@ +package org.skywalking.apm.collector.agentregister.grpc; + +import java.util.LinkedList; +import java.util.List; +import org.skywalking.apm.collector.agentregister.AgentRegisterModuleDefine; +import org.skywalking.apm.collector.agentregister.AgentRegisterModuleGroupDefine; +import org.skywalking.apm.collector.agentregister.grpc.handler.ApplicationRegisterServiceHandler; +import org.skywalking.apm.collector.agentregister.grpc.handler.InstanceDiscoveryServiceHandler; +import org.skywalking.apm.collector.agentregister.grpc.handler.ServiceNameDiscoveryServiceHandler; +import org.skywalking.apm.collector.core.cluster.ClusterDataListener; +import org.skywalking.apm.collector.core.framework.Handler; +import org.skywalking.apm.collector.core.module.ModuleConfigParser; +import org.skywalking.apm.collector.core.module.ModuleRegistration; +import org.skywalking.apm.collector.core.server.Server; +import org.skywalking.apm.collector.server.grpc.GRPCServer; + +/** + * @author pengys5 + */ +public class AgentRegisterGRPCModuleDefine extends AgentRegisterModuleDefine { + + public static final String MODULE_NAME = "grpc"; + + @Override protected String group() { + return AgentRegisterModuleGroupDefine.GROUP_NAME; + } + + @Override public String name() { + return MODULE_NAME; + } + + @Override protected ModuleConfigParser configParser() { + return new AgentRegisterGRPCConfigParser(); + } + + @Override protected Server server() { + return new GRPCServer(AgentRegisterGRPCConfig.HOST, AgentRegisterGRPCConfig.PORT); + } + + @Override protected ModuleRegistration registration() { + return new AgentRegisterGRPCModuleRegistration(); + } + + @Override public ClusterDataListener listener() { + return new AgentRegisterGRPCDataListener(); + } + + @Override public List handlerList() { + List handlers = new LinkedList<>(); + handlers.add(new ApplicationRegisterServiceHandler()); + handlers.add(new InstanceDiscoveryServiceHandler()); + handlers.add(new ServiceNameDiscoveryServiceHandler()); + return handlers; + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/AgentRegisterGRPCModuleRegistration.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/AgentRegisterGRPCModuleRegistration.java new file mode 100644 index 000000000..bc97dba60 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/AgentRegisterGRPCModuleRegistration.java @@ -0,0 +1,13 @@ +package org.skywalking.apm.collector.agentregister.grpc; + +import org.skywalking.apm.collector.core.module.ModuleRegistration; + +/** + * @author pengys5 + */ +public class AgentRegisterGRPCModuleRegistration extends ModuleRegistration { + + @Override public Value buildValue() { + return new Value(AgentRegisterGRPCConfig.HOST, AgentRegisterGRPCConfig.PORT, null); + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandler.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandler.java new file mode 100644 index 000000000..413c35552 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandler.java @@ -0,0 +1,36 @@ +package org.skywalking.apm.collector.agentregister.grpc.handler; + +import com.google.protobuf.ProtocolStringList; +import io.grpc.stub.StreamObserver; +import org.skywalking.apm.collector.agentregister.application.ApplicationIDService; +import org.skywalking.apm.collector.server.grpc.GRPCHandler; +import org.skywalking.apm.network.proto.Application; +import org.skywalking.apm.network.proto.ApplicationMapping; +import org.skywalking.apm.network.proto.ApplicationRegisterServiceGrpc; +import org.skywalking.apm.network.proto.KeyWithIntegerValue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class ApplicationRegisterServiceHandler extends ApplicationRegisterServiceGrpc.ApplicationRegisterServiceImplBase implements GRPCHandler { + + private final Logger logger = LoggerFactory.getLogger(ApplicationRegisterServiceHandler.class); + + private ApplicationIDService applicationIDService = new ApplicationIDService(); + + @Override public void register(Application request, StreamObserver responseObserver) { + logger.debug("register application"); + ProtocolStringList applicationCodes = request.getApplicationCodeList(); + for (int i = 0; i < applicationCodes.size(); i++) { + String applicationCode = applicationCodes.get(i); + int applicationId = applicationIDService.getOrCreate(applicationCode); + + KeyWithIntegerValue value = KeyWithIntegerValue.newBuilder().setKey(applicationCode).setValue(applicationId).build(); + ApplicationMapping mapping = ApplicationMapping.newBuilder().addApplication(i, value).build(); + responseObserver.onNext(mapping); + } + responseObserver.onCompleted(); + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/InstanceDiscoveryServiceHandler.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/InstanceDiscoveryServiceHandler.java new file mode 100644 index 000000000..20ead8290 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/InstanceDiscoveryServiceHandler.java @@ -0,0 +1,46 @@ +package org.skywalking.apm.collector.agentregister.grpc.handler; + +import io.grpc.stub.StreamObserver; +import org.skywalking.apm.collector.agentregister.instance.InstanceIDService; +import org.skywalking.apm.collector.server.grpc.GRPCHandler; +import org.skywalking.apm.network.proto.ApplicationInstance; +import org.skywalking.apm.network.proto.ApplicationInstanceHeartbeat; +import org.skywalking.apm.network.proto.ApplicationInstanceMapping; +import org.skywalking.apm.network.proto.ApplicationInstanceRecover; +import org.skywalking.apm.network.proto.Downstream; +import org.skywalking.apm.network.proto.InstanceDiscoveryServiceGrpc; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class InstanceDiscoveryServiceHandler extends InstanceDiscoveryServiceGrpc.InstanceDiscoveryServiceImplBase implements GRPCHandler { + + private final Logger logger = LoggerFactory.getLogger(InstanceDiscoveryServiceHandler.class); + + private InstanceIDService instanceIDService = new InstanceIDService(); + + @Override + public void register(ApplicationInstance request, StreamObserver responseObserver) { + int instanceId = instanceIDService.getOrCreate(request.getApplicationId(), request.getAgentUUID(), request.getRegisterTime()); + ApplicationInstanceMapping.Builder builder = ApplicationInstanceMapping.newBuilder(); + builder.setApplicationId(request.getApplicationId()); + builder.setApplicationInstanceId(instanceId); + responseObserver.onNext(builder.build()); + responseObserver.onCompleted(); + } + + @Override public void heartbeat(ApplicationInstanceHeartbeat request, StreamObserver responseObserver) { + instanceIDService.heartBeat(request.getApplicationInstanceId(), request.getHeartbeatTime()); + responseObserver.onNext(Downstream.newBuilder().build()); + responseObserver.onCompleted(); + } + + @Override + public void registerRecover(ApplicationInstanceRecover request, StreamObserver responseObserver) { + instanceIDService.recover(request.getApplicationInstanceId(), request.getApplicationId(), request.getRegisterTime()); + responseObserver.onNext(Downstream.newBuilder().build()); + responseObserver.onCompleted(); + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/ServiceNameDiscoveryServiceHandler.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/ServiceNameDiscoveryServiceHandler.java new file mode 100644 index 000000000..1009658a7 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/grpc/handler/ServiceNameDiscoveryServiceHandler.java @@ -0,0 +1,42 @@ +package org.skywalking.apm.collector.agentregister.grpc.handler; + +import io.grpc.stub.StreamObserver; +import java.util.List; +import org.skywalking.apm.collector.agentregister.servicename.ServiceNameService; +import org.skywalking.apm.collector.server.grpc.GRPCHandler; +import org.skywalking.apm.network.proto.ServiceNameCollection; +import org.skywalking.apm.network.proto.ServiceNameDiscoveryServiceGrpc; +import org.skywalking.apm.network.proto.ServiceNameElement; +import org.skywalking.apm.network.proto.ServiceNameMappingCollection; +import org.skywalking.apm.network.proto.ServiceNameMappingElement; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class ServiceNameDiscoveryServiceHandler extends ServiceNameDiscoveryServiceGrpc.ServiceNameDiscoveryServiceImplBase implements GRPCHandler { + + private final Logger logger = LoggerFactory.getLogger(ServiceNameDiscoveryServiceHandler.class); + + private ServiceNameService serviceNameService = new ServiceNameService(); + + @Override public void discovery(ServiceNameCollection request, + StreamObserver responseObserver) { + List serviceNameElementList = request.getElementsList(); + + ServiceNameMappingCollection.Builder builder = ServiceNameMappingCollection.newBuilder(); + for (ServiceNameElement serviceNameElement : serviceNameElementList) { + int applicationId = serviceNameElement.getApplicationId(); + String serviceName = serviceNameElement.getServiceName(); + int serviceId = serviceNameService.getOrCreate(applicationId, serviceName); + ServiceNameMappingElement.Builder mappingElement = ServiceNameMappingElement.newBuilder(); + mappingElement.setServiceId(serviceId); + mappingElement.setElement(serviceNameElement); + builder.addElements(mappingElement); + } + + responseObserver.onNext(builder.build()); + responseObserver.onCompleted(); + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/instance/InstanceIDService.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/instance/InstanceIDService.java new file mode 100644 index 000000000..4ac42d8bd --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/instance/InstanceIDService.java @@ -0,0 +1,49 @@ +package org.skywalking.apm.collector.agentregister.instance; + +import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterRemoteWorker; +import org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceDataDefine; +import org.skywalking.apm.collector.agentstream.worker.register.instance.dao.IInstanceDAO; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.stream.StreamModuleContext; +import org.skywalking.apm.collector.stream.StreamModuleGroupDefine; +import org.skywalking.apm.collector.stream.worker.WorkerInvokeException; +import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class InstanceIDService { + + private final Logger logger = LoggerFactory.getLogger(InstanceIDService.class); + + public int getOrCreate(int applicationId, String agentUUID, long registerTime) { + IInstanceDAO dao = (IInstanceDAO)DAOContainer.INSTANCE.get(IInstanceDAO.class.getName()); + int instanceId = dao.getInstanceId(applicationId, agentUUID); + + if (instanceId == 0) { + StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); + InstanceDataDefine.Instance instance = new InstanceDataDefine.Instance("0", applicationId, agentUUID, registerTime, 0); + try { + context.getClusterWorkerContext().lookup(ApplicationRegisterRemoteWorker.WorkerRole.INSTANCE).tell(instance); + } catch (WorkerNotFoundException | WorkerInvokeException e) { + logger.error(e.getMessage(), e); + } + } + return applicationId; + } + + public void heartBeat(int instanceId, long heartbeatTime) { + IInstanceDAO dao = (IInstanceDAO)DAOContainer.INSTANCE.get(IInstanceDAO.class.getName()); + dao.updateHeartbeatTime(instanceId, heartbeatTime); + } + + public void recover(int instanceId, int applicationId, long registerTime) { + IInstanceDAO dao = (IInstanceDAO)DAOContainer.INSTANCE.get(IInstanceDAO.class.getName()); + + InstanceDataDefine.Instance instance = new InstanceDataDefine.Instance(String.valueOf(instanceId), applicationId, "", registerTime, instanceId); + dao.save(instance); + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/servicename/ServiceNameService.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/servicename/ServiceNameService.java new file mode 100644 index 000000000..259acc984 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/servicename/ServiceNameService.java @@ -0,0 +1,37 @@ +package org.skywalking.apm.collector.agentregister.servicename; + +import org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameDataDefine; +import org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameRegisterRemoteWorker; +import org.skywalking.apm.collector.agentstream.worker.register.servicename.dao.IServiceNameDAO; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.stream.StreamModuleContext; +import org.skywalking.apm.collector.stream.StreamModuleGroupDefine; +import org.skywalking.apm.collector.stream.worker.WorkerInvokeException; +import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class ServiceNameService { + + private final Logger logger = LoggerFactory.getLogger(ServiceNameService.class); + + public int getOrCreate(int applicationId, String serviceName) { + IServiceNameDAO dao = (IServiceNameDAO)DAOContainer.INSTANCE.get(IServiceNameDAO.class.getName()); + int serviceId = dao.getServiceId(applicationId, serviceName); + + if (serviceId == 0) { + StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); + ServiceNameDataDefine.ServiceName service = new ServiceNameDataDefine.ServiceName("0", serviceName, applicationId, 0); + try { + context.getClusterWorkerContext().lookup(ServiceNameRegisterRemoteWorker.WorkerRole.INSTANCE).tell(service); + } catch (WorkerNotFoundException | WorkerInvokeException e) { + logger.error(e.getMessage(), e); + } + } + return serviceId; + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/group.define b/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/group.define new file mode 100644 index 000000000..0afe036bd --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/group.define @@ -0,0 +1 @@ +org.skywalking.apm.collector.agentregister.AgentRegisterModuleGroupDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/module.define b/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/module.define new file mode 100644 index 000000000..70f99f0a2 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/module.define @@ -0,0 +1 @@ +org.skywalking.apm.collector.agentregister.grpc.AgentRegisterGRPCModuleDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-agentregister/src/test/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandlerTestCase.java b/apm-collector/apm-collector-agentregister/src/test/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandlerTestCase.java new file mode 100644 index 000000000..6f29bbc72 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/test/java/org/skywalking/apm/collector/agentregister/grpc/handler/ApplicationRegisterServiceHandlerTestCase.java @@ -0,0 +1,24 @@ +package org.skywalking.apm.collector.agentregister.grpc.handler; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import org.skywalking.apm.network.proto.Application; +import org.skywalking.apm.network.proto.ApplicationMapping; +import org.skywalking.apm.network.proto.ApplicationRegisterServiceGrpc; + +/** + * @author pengys5 + */ +public class ApplicationRegisterServiceHandlerTestCase { + + private ApplicationRegisterServiceGrpc.ApplicationRegisterServiceBlockingStub stub; + + public void testRegister() { + ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", 11800).usePlaintext(true).build(); + stub = ApplicationRegisterServiceGrpc.newBlockingStub(channel); + + Application application = Application.newBuilder().addApplicationCode("test141").build(); + ApplicationMapping mapping = stub.register(application); + System.out.println(mapping.getApplication(0).getKey() + ", " + mapping.getApplication(0).getValue()); + } +} diff --git a/apm-collector/apm-collector-agentserver/pom.xml b/apm-collector/apm-collector-agentserver/pom.xml new file mode 100644 index 000000000..8f174702d --- /dev/null +++ b/apm-collector/apm-collector-agentserver/pom.xml @@ -0,0 +1,42 @@ + + + + apm-collector + org.skywalking + 3.2-2017 + + 4.0.0 + + apm-collector-agentserver + jar + + + + org.skywalking + apm-collector-cluster + ${project.version} + + + org.skywalking + apm-collector-server + ${project.version} + + + org.skywalking + apm-collector-agentstream + ${project.version} + + + org.skywalking + apm-collector-agentregister + ${project.version} + + + org.skywalking + apm-collector-ui + ${project.version} + + + \ No newline at end of file diff --git a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/AgentServerModuleContext.java b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/AgentServerModuleContext.java new file mode 100644 index 000000000..f6db9c4ab --- /dev/null +++ b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/AgentServerModuleContext.java @@ -0,0 +1,13 @@ +package org.skywalking.apm.collector.agentserver; + +import org.skywalking.apm.collector.core.framework.Context; + +/** + * @author pengys5 + */ +public class AgentServerModuleContext extends Context { + + public AgentServerModuleContext(String groupName) { + super(groupName); + } +} diff --git a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/AgentServerModuleDefine.java b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/AgentServerModuleDefine.java new file mode 100644 index 000000000..e7e7d829a --- /dev/null +++ b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/AgentServerModuleDefine.java @@ -0,0 +1,47 @@ +package org.skywalking.apm.collector.agentserver; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.cluster.ClusterModuleGroupDefine; +import org.skywalking.apm.collector.core.client.Client; +import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.client.DataMonitor; +import org.skywalking.apm.collector.core.cluster.ClusterDataListenerDefine; +import org.skywalking.apm.collector.core.cluster.ClusterModuleContext; +import org.skywalking.apm.collector.core.config.ConfigParseException; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.framework.Handler; +import org.skywalking.apm.collector.core.module.ModuleDefine; +import org.skywalking.apm.collector.core.server.Server; +import org.skywalking.apm.collector.core.server.ServerException; +import org.skywalking.apm.collector.core.server.ServerHolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public abstract class AgentServerModuleDefine extends ModuleDefine implements ClusterDataListenerDefine { + + private final Logger logger = LoggerFactory.getLogger(AgentServerModuleDefine.class); + + @Override + public final void initialize(Map config, ServerHolder serverHolder) throws DefineException, ClientException { + try { + configParser().parse(config); + Server server = server(); + serverHolder.holdServer(server, handlerList()); + + ((ClusterModuleContext)CollectorContextHelper.INSTANCE.getContext(ClusterModuleGroupDefine.GROUP_NAME)).getDataMonitor().addListener(listener(), registration()); + } catch (ConfigParseException | ServerException e) { + throw new AgentServerModuleException(e.getMessage(), e); + } + } + + @Override protected final Client createClient(DataMonitor dataMonitor) { + throw new UnsupportedOperationException(""); + } + + public abstract List handlerList(); +} diff --git a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/AgentServerModuleException.java b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/AgentServerModuleException.java new file mode 100644 index 000000000..c9f7d2a97 --- /dev/null +++ b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/AgentServerModuleException.java @@ -0,0 +1,17 @@ +package org.skywalking.apm.collector.agentserver; + +import org.skywalking.apm.collector.core.module.ModuleException; + +/** + * @author pengys5 + */ +public class AgentServerModuleException extends ModuleException { + + public AgentServerModuleException(String message) { + super(message); + } + + public AgentServerModuleException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/AgentServerModuleGroupDefine.java b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/AgentServerModuleGroupDefine.java new file mode 100644 index 000000000..9539be419 --- /dev/null +++ b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/AgentServerModuleGroupDefine.java @@ -0,0 +1,25 @@ +package org.skywalking.apm.collector.agentserver; + +import org.skywalking.apm.collector.core.framework.Context; +import org.skywalking.apm.collector.core.module.ModuleGroupDefine; +import org.skywalking.apm.collector.core.module.ModuleInstaller; + +/** + * @author pengys5 + */ +public class AgentServerModuleGroupDefine implements ModuleGroupDefine { + + public static final String GROUP_NAME = "agent_server"; + + @Override public String name() { + return GROUP_NAME; + } + + @Override public Context groupContext() { + return new AgentServerModuleContext(GROUP_NAME); + } + + @Override public ModuleInstaller moduleInstaller() { + return new AgentServerModuleInstaller(); + } +} diff --git a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/AgentServerModuleInstaller.java b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/AgentServerModuleInstaller.java new file mode 100644 index 000000000..c63216104 --- /dev/null +++ b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/AgentServerModuleInstaller.java @@ -0,0 +1,37 @@ +package org.skywalking.apm.collector.agentserver; + +import java.util.Iterator; +import java.util.Map; +import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.module.ModuleDefine; +import org.skywalking.apm.collector.core.module.ModuleInstaller; +import org.skywalking.apm.collector.core.server.ServerHolder; +import org.skywalking.apm.collector.core.util.ObjectUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class AgentServerModuleInstaller implements ModuleInstaller { + + private final Logger logger = LoggerFactory.getLogger(AgentServerModuleInstaller.class); + + @Override public void install(Map moduleConfig, + Map moduleDefineMap, ServerHolder serverHolder) throws DefineException, ClientException { + logger.info("beginning agent server module install"); + + AgentServerModuleContext context = new AgentServerModuleContext(AgentServerModuleGroupDefine.GROUP_NAME); + CollectorContextHelper.INSTANCE.putContext(context); + + logger.info("could not configure agent server module, use the default"); + Iterator> moduleDefineEntry = moduleDefineMap.entrySet().iterator(); + while (moduleDefineEntry.hasNext()) { + ModuleDefine moduleDefine = moduleDefineEntry.next().getValue(); + logger.info("module {} initialize", moduleDefine.getClass().getName()); + moduleDefine.initialize((ObjectUtils.isNotEmpty(moduleConfig) && moduleConfig.containsKey(moduleDefine.name())) ? moduleConfig.get(moduleDefine.name()) : null, serverHolder); + } + } +} diff --git a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/AgentServerJettyConfig.java b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/AgentServerJettyConfig.java new file mode 100644 index 000000000..1bad21878 --- /dev/null +++ b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/AgentServerJettyConfig.java @@ -0,0 +1,10 @@ +package org.skywalking.apm.collector.agentserver.jetty; + +/** + * @author pengys5 + */ +public class AgentServerJettyConfig { + public static String HOST; + public static int PORT; + public static String CONTEXT_PATH; +} diff --git a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/AgentServerJettyConfigParser.java b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/AgentServerJettyConfigParser.java new file mode 100644 index 000000000..2625be078 --- /dev/null +++ b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/AgentServerJettyConfigParser.java @@ -0,0 +1,36 @@ +package org.skywalking.apm.collector.agentserver.jetty; + +import java.util.Map; +import org.skywalking.apm.collector.core.config.ConfigParseException; +import org.skywalking.apm.collector.core.module.ModuleConfigParser; +import org.skywalking.apm.collector.core.util.ObjectUtils; +import org.skywalking.apm.collector.core.util.StringUtils; + +/** + * @author pengys5 + */ +public class AgentServerJettyConfigParser implements ModuleConfigParser { + + private static final String HOST = "host"; + private static final String PORT = "port"; + public static final String CONTEXT_PATH = "contextPath"; + + @Override public void parse(Map config) throws ConfigParseException { + AgentServerJettyConfig.CONTEXT_PATH = "/"; + + if (ObjectUtils.isEmpty(config) || StringUtils.isEmpty(config.get(HOST))) { + AgentServerJettyConfig.HOST = "localhost"; + } else { + AgentServerJettyConfig.HOST = (String)config.get(HOST); + } + + if (ObjectUtils.isEmpty(config) || StringUtils.isEmpty(config.get(PORT))) { + AgentServerJettyConfig.PORT = 10800; + } else { + AgentServerJettyConfig.PORT = (Integer)config.get(PORT); + } + if (ObjectUtils.isNotEmpty(config) && StringUtils.isNotEmpty(config.get(CONTEXT_PATH))) { + AgentServerJettyConfig.CONTEXT_PATH = (String)config.get(CONTEXT_PATH); + } + } +} diff --git a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/AgentServerJettyDataListener.java b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/AgentServerJettyDataListener.java new file mode 100644 index 000000000..1ce44e9d8 --- /dev/null +++ b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/AgentServerJettyDataListener.java @@ -0,0 +1,19 @@ +package org.skywalking.apm.collector.agentserver.jetty; + +import org.skywalking.apm.collector.agentserver.AgentServerModuleGroupDefine; +import org.skywalking.apm.collector.cluster.ClusterModuleDefine; +import org.skywalking.apm.collector.core.cluster.ClusterDataListener; + +/** + * @author pengys5 + */ +public class AgentServerJettyDataListener extends ClusterDataListener { + + @Override public String path() { + return ClusterModuleDefine.BASE_CATALOG + "." + AgentServerModuleGroupDefine.GROUP_NAME + "." + AgentServerJettyModuleDefine.MODULE_NAME; + } + + @Override public void addressChangedNotify() { + + } +} diff --git a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/AgentServerJettyModuleDefine.java b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/AgentServerJettyModuleDefine.java new file mode 100644 index 000000000..43314323c --- /dev/null +++ b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/AgentServerJettyModuleDefine.java @@ -0,0 +1,57 @@ +package org.skywalking.apm.collector.agentserver.jetty; + +import java.util.LinkedList; +import java.util.List; +import org.skywalking.apm.collector.agentserver.AgentServerModuleDefine; +import org.skywalking.apm.collector.agentserver.AgentServerModuleGroupDefine; +import org.skywalking.apm.collector.agentserver.jetty.handler.AgentStreamGRPCServerHandler; +import org.skywalking.apm.collector.agentserver.jetty.handler.AgentStreamJettyServerHandler; +import org.skywalking.apm.collector.core.cluster.ClusterDataListener; +import org.skywalking.apm.collector.core.framework.Handler; +import org.skywalking.apm.collector.core.module.ModuleConfigParser; +import org.skywalking.apm.collector.core.module.ModuleRegistration; +import org.skywalking.apm.collector.core.server.Server; +import org.skywalking.apm.collector.server.jetty.JettyServer; + +/** + * @author pengys5 + */ +public class AgentServerJettyModuleDefine extends AgentServerModuleDefine { + + public static final String MODULE_NAME = "jetty"; + + @Override protected String group() { + return AgentServerModuleGroupDefine.GROUP_NAME; + } + + @Override public String name() { + return MODULE_NAME; + } + + @Override public boolean defaultModule() { + return true; + } + + @Override protected ModuleConfigParser configParser() { + return new AgentServerJettyConfigParser(); + } + + @Override protected Server server() { + return new JettyServer(AgentServerJettyConfig.HOST, AgentServerJettyConfig.PORT, AgentServerJettyConfig.CONTEXT_PATH); + } + + @Override protected ModuleRegistration registration() { + return new AgentServerJettyModuleRegistration(); + } + + @Override public ClusterDataListener listener() { + return new AgentServerJettyDataListener(); + } + + @Override public List handlerList() { + List handlers = new LinkedList<>(); + handlers.add(new AgentStreamGRPCServerHandler()); + handlers.add(new AgentStreamJettyServerHandler()); + return handlers; + } +} diff --git a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/AgentServerJettyModuleRegistration.java b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/AgentServerJettyModuleRegistration.java new file mode 100644 index 000000000..5f2f2e0bf --- /dev/null +++ b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/AgentServerJettyModuleRegistration.java @@ -0,0 +1,13 @@ +package org.skywalking.apm.collector.agentserver.jetty; + +import org.skywalking.apm.collector.core.module.ModuleRegistration; + +/** + * @author pengys5 + */ +public class AgentServerJettyModuleRegistration extends ModuleRegistration { + + @Override public Value buildValue() { + return new Value(AgentServerJettyConfig.HOST, AgentServerJettyConfig.PORT, AgentServerJettyConfig.CONTEXT_PATH); + } +} diff --git a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamGRPCServerHandler.java b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamGRPCServerHandler.java new file mode 100644 index 000000000..2c4e4bebb --- /dev/null +++ b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamGRPCServerHandler.java @@ -0,0 +1,33 @@ +package org.skywalking.apm.collector.agentserver.jetty.handler; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import java.util.List; +import javax.servlet.http.HttpServletRequest; +import org.skywalking.apm.collector.agentstream.grpc.AgentStreamGRPCDataListener; +import org.skywalking.apm.collector.cluster.ClusterModuleGroupDefine; +import org.skywalking.apm.collector.core.cluster.ClusterModuleContext; +import org.skywalking.apm.collector.core.cluster.ClusterModuleRegistrationReader; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.server.jetty.ArgumentsParseException; +import org.skywalking.apm.collector.server.jetty.JettyHandler; + +/** + * @author pengys5 + */ +public class AgentStreamGRPCServerHandler extends JettyHandler { + + @Override public String pathSpec() { + return "/agentstream/grpc"; + } + + @Override protected JsonElement doGet(HttpServletRequest req) throws ArgumentsParseException { + ClusterModuleRegistrationReader reader = ((ClusterModuleContext)CollectorContextHelper.INSTANCE.getContext(ClusterModuleGroupDefine.GROUP_NAME)).getReader(); + List servers = reader.read(AgentStreamGRPCDataListener.PATH); + JsonArray serverArray = new JsonArray(); + servers.forEach(server -> { + serverArray.add(server); + }); + return serverArray; + } +} diff --git a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamJettyServerHandler.java b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamJettyServerHandler.java new file mode 100644 index 000000000..dcaf05cfa --- /dev/null +++ b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamJettyServerHandler.java @@ -0,0 +1,33 @@ +package org.skywalking.apm.collector.agentserver.jetty.handler; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import java.util.List; +import javax.servlet.http.HttpServletRequest; +import org.skywalking.apm.collector.agentstream.jetty.AgentStreamJettyDataListener; +import org.skywalking.apm.collector.cluster.ClusterModuleGroupDefine; +import org.skywalking.apm.collector.core.cluster.ClusterModuleContext; +import org.skywalking.apm.collector.core.cluster.ClusterModuleRegistrationReader; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.server.jetty.ArgumentsParseException; +import org.skywalking.apm.collector.server.jetty.JettyHandler; + +/** + * @author pengys5 + */ +public class AgentStreamJettyServerHandler extends JettyHandler { + + @Override public String pathSpec() { + return "/agentstream/jetty"; + } + + @Override protected JsonElement doGet(HttpServletRequest req) throws ArgumentsParseException { + ClusterModuleRegistrationReader reader = ((ClusterModuleContext)CollectorContextHelper.INSTANCE.getContext(ClusterModuleGroupDefine.GROUP_NAME)).getReader(); + List servers = reader.read(AgentStreamJettyDataListener.PATH); + JsonArray serverArray = new JsonArray(); + servers.forEach(server -> { + serverArray.add(server); + }); + return serverArray; + } +} diff --git a/apm-collector/apm-collector-agentserver/src/main/resources/META-INF/defines/group.define b/apm-collector/apm-collector-agentserver/src/main/resources/META-INF/defines/group.define new file mode 100644 index 000000000..229ed8a56 --- /dev/null +++ b/apm-collector/apm-collector-agentserver/src/main/resources/META-INF/defines/group.define @@ -0,0 +1 @@ +org.skywalking.apm.collector.agentserver.AgentServerModuleGroupDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-agentserver/src/main/resources/META-INF/defines/module.define b/apm-collector/apm-collector-agentserver/src/main/resources/META-INF/defines/module.define new file mode 100644 index 000000000..cdd20e3bf --- /dev/null +++ b/apm-collector/apm-collector-agentserver/src/main/resources/META-INF/defines/module.define @@ -0,0 +1 @@ +org.skywalking.apm.collector.agentserver.jetty.AgentServerJettyModuleDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/pom.xml b/apm-collector/apm-collector-agentstream/pom.xml index 8ef444a8a..7cf497179 100644 --- a/apm-collector/apm-collector-agentstream/pom.xml +++ b/apm-collector/apm-collector-agentstream/pom.xml @@ -15,7 +15,12 @@ org.skywalking - apm-collector-core + apm-collector-stream + ${project.version} + + + org.skywalking + apm-collector-cluster ${project.version} @@ -23,5 +28,10 @@ apm-collector-server ${project.version} + + org.skywalking + apm-collector-storage + ${project.version} + \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/grpc/AgentStreamGRPCModuleDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/grpc/AgentStreamGRPCModuleDefine.java deleted file mode 100644 index 5efeb7a58..000000000 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/grpc/AgentStreamGRPCModuleDefine.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.skywalking.apm.collector.agent.stream.server.grpc; - -import org.skywalking.apm.collector.core.agentstream.AgentStreamModuleDefine; -import org.skywalking.apm.collector.core.module.ModuleConfigParser; -import org.skywalking.apm.collector.core.module.ModuleGroup; -import org.skywalking.apm.collector.core.module.ModuleRegistration; -import org.skywalking.apm.collector.core.server.Server; -import org.skywalking.apm.collector.server.grpc.GRPCServer; - -/** - * @author pengys5 - */ -public class AgentStreamGRPCModuleDefine extends AgentStreamModuleDefine { - - @Override protected ModuleGroup group() { - return ModuleGroup.AgentStream; - } - - @Override public String name() { - return "grpc"; - } - - @Override public boolean defaultModule() { - return true; - } - - @Override protected ModuleConfigParser configParser() { - return new AgentStreamGRPCConfigParser(); - } - - @Override protected Server server() { - return new GRPCServer(AgentStreamGRPCConfig.HOST, AgentStreamGRPCConfig.PORT); - } - - @Override protected ModuleRegistration registration() { - return new AgentStreamGRPCModuleRegistration(); - } -} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleContext.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleContext.java new file mode 100644 index 000000000..f80a15ba5 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleContext.java @@ -0,0 +1,13 @@ +package org.skywalking.apm.collector.agentstream; + +import org.skywalking.apm.collector.core.framework.Context; + +/** + * @author pengys5 + */ +public class AgentStreamModuleContext extends Context { + + public AgentStreamModuleContext(String groupName) { + super(groupName); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleDefine.java new file mode 100644 index 000000000..153508827 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleDefine.java @@ -0,0 +1,51 @@ +package org.skywalking.apm.collector.agentstream; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.cluster.ClusterModuleGroupDefine; +import org.skywalking.apm.collector.core.client.Client; +import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.client.DataMonitor; +import org.skywalking.apm.collector.core.cluster.ClusterDataListenerDefine; +import org.skywalking.apm.collector.core.cluster.ClusterModuleContext; +import org.skywalking.apm.collector.core.config.ConfigParseException; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.framework.Handler; +import org.skywalking.apm.collector.core.module.ModuleDefine; +import org.skywalking.apm.collector.core.server.Server; +import org.skywalking.apm.collector.core.server.ServerException; +import org.skywalking.apm.collector.core.server.ServerHolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public abstract class AgentStreamModuleDefine extends ModuleDefine implements ClusterDataListenerDefine { + + private final Logger logger = LoggerFactory.getLogger(AgentStreamModuleDefine.class); + + @Override + public final void initialize(Map config, ServerHolder serverHolder) throws DefineException, ClientException { + try { + configParser().parse(config); + Server server = server(); + serverHolder.holdServer(server, handlerList()); + + ((ClusterModuleContext)CollectorContextHelper.INSTANCE.getContext(ClusterModuleGroupDefine.GROUP_NAME)).getDataMonitor().addListener(listener(), registration()); + } catch (ConfigParseException | ServerException e) { + throw new AgentStreamModuleException(e.getMessage(), e); + } + } + + @Override protected final Client createClient(DataMonitor dataMonitor) { + throw new UnsupportedOperationException(""); + } + + @Override public final boolean defaultModule() { + return true; + } + + public abstract List handlerList(); +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/agentstream/AgentStreamModuleException.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleException.java similarity index 86% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/agentstream/AgentStreamModuleException.java rename to apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleException.java index f46602bbd..e51b3f6f6 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/agentstream/AgentStreamModuleException.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleException.java @@ -1,4 +1,4 @@ -package org.skywalking.apm.collector.core.agentstream; +package org.skywalking.apm.collector.agentstream; import org.skywalking.apm.collector.core.module.ModuleException; diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleGroupDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleGroupDefine.java new file mode 100644 index 000000000..216e2ef2e --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleGroupDefine.java @@ -0,0 +1,25 @@ +package org.skywalking.apm.collector.agentstream; + +import org.skywalking.apm.collector.core.framework.Context; +import org.skywalking.apm.collector.core.module.ModuleGroupDefine; +import org.skywalking.apm.collector.core.module.ModuleInstaller; + +/** + * @author pengys5 + */ +public class AgentStreamModuleGroupDefine implements ModuleGroupDefine { + + public static final String GROUP_NAME = "agent_stream"; + + @Override public String name() { + return GROUP_NAME; + } + + @Override public Context groupContext() { + return new AgentStreamModuleContext(GROUP_NAME); + } + + @Override public ModuleInstaller moduleInstaller() { + return new AgentStreamModuleInstaller(); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleInstaller.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleInstaller.java new file mode 100644 index 000000000..bf5227d48 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleInstaller.java @@ -0,0 +1,39 @@ +package org.skywalking.apm.collector.agentstream; + +import java.util.Iterator; +import java.util.Map; +import org.skywalking.apm.collector.agentstream.worker.storage.PersistenceTimer; +import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.module.ModuleDefine; +import org.skywalking.apm.collector.core.module.ModuleInstaller; +import org.skywalking.apm.collector.core.server.ServerHolder; +import org.skywalking.apm.collector.core.util.ObjectUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class AgentStreamModuleInstaller implements ModuleInstaller { + + private final Logger logger = LoggerFactory.getLogger(AgentStreamModuleInstaller.class); + + @Override public void install(Map moduleConfig, + Map moduleDefineMap, ServerHolder serverHolder) throws DefineException, ClientException { + logger.info("beginning agent stream module install"); + + AgentStreamModuleContext context = new AgentStreamModuleContext(AgentStreamModuleGroupDefine.GROUP_NAME); + CollectorContextHelper.INSTANCE.putContext(context); + + Iterator> moduleDefineEntry = moduleDefineMap.entrySet().iterator(); + while (moduleDefineEntry.hasNext()) { + ModuleDefine moduleDefine = moduleDefineEntry.next().getValue(); + logger.info("module {} initialize", moduleDefine.getClass().getName()); + moduleDefine.initialize((ObjectUtils.isNotEmpty(moduleConfig) && moduleConfig.containsKey(moduleDefine.name())) ? moduleConfig.get(moduleDefine.name()) : null, serverHolder); + } + + new PersistenceTimer().start(); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/grpc/AgentStreamGRPCConfig.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/AgentStreamGRPCConfig.java similarity index 66% rename from apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/grpc/AgentStreamGRPCConfig.java rename to apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/AgentStreamGRPCConfig.java index 950775c50..d04feda9f 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/grpc/AgentStreamGRPCConfig.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/AgentStreamGRPCConfig.java @@ -1,4 +1,4 @@ -package org.skywalking.apm.collector.agent.stream.server.grpc; +package org.skywalking.apm.collector.agentstream.grpc; /** * @author pengys5 diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/grpc/AgentStreamGRPCConfigParser.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/AgentStreamGRPCConfigParser.java similarity index 56% rename from apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/grpc/AgentStreamGRPCConfigParser.java rename to apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/AgentStreamGRPCConfigParser.java index 05fd728a7..f6e8eca80 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/grpc/AgentStreamGRPCConfigParser.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/AgentStreamGRPCConfigParser.java @@ -1,8 +1,9 @@ -package org.skywalking.apm.collector.agent.stream.server.grpc; +package org.skywalking.apm.collector.agentstream.grpc; import java.util.Map; import org.skywalking.apm.collector.core.config.ConfigParseException; import org.skywalking.apm.collector.core.module.ModuleConfigParser; +import org.skywalking.apm.collector.core.util.ObjectUtils; import org.skywalking.apm.collector.core.util.StringUtils; /** @@ -14,16 +15,16 @@ public class AgentStreamGRPCConfigParser implements ModuleConfigParser { private static final String PORT = "port"; @Override public void parse(Map config) throws ConfigParseException { - AgentStreamGRPCConfig.HOST = (String)config.get(HOST); - - if (StringUtils.isEmpty(AgentStreamGRPCConfig.HOST)) { - throw new ConfigParseException(""); + if (ObjectUtils.isEmpty(config) || StringUtils.isEmpty(config.get(HOST))) { + AgentStreamGRPCConfig.HOST = "localhost"; + } else { + AgentStreamGRPCConfig.HOST = (String)config.get(HOST); } - if (StringUtils.isEmpty(config.get(PORT))) { - throw new ConfigParseException(""); + + if (ObjectUtils.isEmpty(config) || StringUtils.isEmpty(config.get(PORT))) { + AgentStreamGRPCConfig.PORT = 11800; } else { AgentStreamGRPCConfig.PORT = (Integer)config.get(PORT); - } } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/AgentStreamGRPCDataListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/AgentStreamGRPCDataListener.java new file mode 100644 index 000000000..dd3428ec3 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/AgentStreamGRPCDataListener.java @@ -0,0 +1,20 @@ +package org.skywalking.apm.collector.agentstream.grpc; + +import org.skywalking.apm.collector.agentstream.AgentStreamModuleGroupDefine; +import org.skywalking.apm.collector.cluster.ClusterModuleDefine; +import org.skywalking.apm.collector.core.cluster.ClusterDataListener; + +/** + * @author pengys5 + */ +public class AgentStreamGRPCDataListener extends ClusterDataListener { + + public static final String PATH = ClusterModuleDefine.BASE_CATALOG + "." + AgentStreamModuleGroupDefine.GROUP_NAME + "." + AgentStreamGRPCModuleDefine.MODULE_NAME; + + @Override public String path() { + return PATH; + } + + @Override public void addressChangedNotify() { + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/AgentStreamGRPCModuleDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/AgentStreamGRPCModuleDefine.java new file mode 100644 index 000000000..1cc1bdfad --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/AgentStreamGRPCModuleDefine.java @@ -0,0 +1,51 @@ +package org.skywalking.apm.collector.agentstream.grpc; + +import java.util.LinkedList; +import java.util.List; +import org.skywalking.apm.collector.agentstream.AgentStreamModuleDefine; +import org.skywalking.apm.collector.agentstream.AgentStreamModuleGroupDefine; +import org.skywalking.apm.collector.agentstream.grpc.handler.TraceSegmentServiceHandler; +import org.skywalking.apm.collector.core.cluster.ClusterDataListener; +import org.skywalking.apm.collector.core.framework.Handler; +import org.skywalking.apm.collector.core.module.ModuleConfigParser; +import org.skywalking.apm.collector.core.module.ModuleRegistration; +import org.skywalking.apm.collector.core.server.Server; +import org.skywalking.apm.collector.server.grpc.GRPCServer; + +/** + * @author pengys5 + */ +public class AgentStreamGRPCModuleDefine extends AgentStreamModuleDefine { + + public static final String MODULE_NAME = "grpc"; + + @Override protected String group() { + return AgentStreamModuleGroupDefine.GROUP_NAME; + } + + @Override public String name() { + return MODULE_NAME; + } + + @Override protected ModuleConfigParser configParser() { + return new AgentStreamGRPCConfigParser(); + } + + @Override protected Server server() { + return new GRPCServer(AgentStreamGRPCConfig.HOST, AgentStreamGRPCConfig.PORT); + } + + @Override protected ModuleRegistration registration() { + return new AgentStreamGRPCModuleRegistration(); + } + + @Override public ClusterDataListener listener() { + return new AgentStreamGRPCDataListener(); + } + + @Override public List handlerList() { + List handlers = new LinkedList<>(); + handlers.add(new TraceSegmentServiceHandler()); + return handlers; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/grpc/AgentStreamGRPCModuleRegistration.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/AgentStreamGRPCModuleRegistration.java similarity index 83% rename from apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/grpc/AgentStreamGRPCModuleRegistration.java rename to apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/AgentStreamGRPCModuleRegistration.java index 347b29b4e..800ebba24 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/grpc/AgentStreamGRPCModuleRegistration.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/AgentStreamGRPCModuleRegistration.java @@ -1,4 +1,4 @@ -package org.skywalking.apm.collector.agent.stream.server.grpc; +package org.skywalking.apm.collector.agentstream.grpc; import org.skywalking.apm.collector.core.module.ModuleRegistration; diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandler.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandler.java new file mode 100644 index 000000000..70887d0c1 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandler.java @@ -0,0 +1,47 @@ +package org.skywalking.apm.collector.agentstream.grpc.handler; + +import com.google.protobuf.InvalidProtocolBufferException; +import io.grpc.stub.StreamObserver; +import java.util.List; +import org.skywalking.apm.collector.agentstream.worker.segment.SegmentParse; +import org.skywalking.apm.collector.server.grpc.GRPCHandler; +import org.skywalking.apm.network.proto.Downstream; +import org.skywalking.apm.network.proto.TraceSegmentObject; +import org.skywalking.apm.network.proto.TraceSegmentServiceGrpc; +import org.skywalking.apm.network.proto.UniqueId; +import org.skywalking.apm.network.proto.UpstreamSegment; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class TraceSegmentServiceHandler extends TraceSegmentServiceGrpc.TraceSegmentServiceImplBase implements GRPCHandler { + + private final Logger logger = LoggerFactory.getLogger(TraceSegmentServiceHandler.class); + + @Override public StreamObserver collect(StreamObserver responseObserver) { + return new StreamObserver() { + @Override public void onNext(UpstreamSegment segment) { + logger.debug("receive segment"); + SegmentParse segmentParse = new SegmentParse(); + try { + List traceIds = segment.getGlobalTraceIdsList(); + TraceSegmentObject segmentObject = TraceSegmentObject.parseFrom(segment.getSegment()); + segmentParse.parse(traceIds, segmentObject); + } catch (InvalidProtocolBufferException e) { + logger.error(e.getMessage(), e); + } + } + + @Override public void onError(Throwable throwable) { + logger.error(throwable.getMessage(), throwable); + } + + @Override public void onCompleted() { + responseObserver.onNext(Downstream.newBuilder().build()); + responseObserver.onCompleted(); + } + }; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/jetty/AgentStreamJettyConfig.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyConfig.java similarity index 72% rename from apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/jetty/AgentStreamJettyConfig.java rename to apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyConfig.java index 240d5ef1e..5ed3e24c5 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/jetty/AgentStreamJettyConfig.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyConfig.java @@ -1,4 +1,4 @@ -package org.skywalking.apm.collector.agent.stream.server.jetty; +package org.skywalking.apm.collector.agentstream.jetty; /** * @author pengys5 diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/jetty/AgentStreamJettyConfigParser.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyConfigParser.java similarity index 58% rename from apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/jetty/AgentStreamJettyConfigParser.java rename to apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyConfigParser.java index d50a463c2..d7e4c4ceb 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/jetty/AgentStreamJettyConfigParser.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyConfigParser.java @@ -1,8 +1,9 @@ -package org.skywalking.apm.collector.agent.stream.server.jetty; +package org.skywalking.apm.collector.agentstream.jetty; import java.util.Map; import org.skywalking.apm.collector.core.config.ConfigParseException; import org.skywalking.apm.collector.core.module.ModuleConfigParser; +import org.skywalking.apm.collector.core.util.ObjectUtils; import org.skywalking.apm.collector.core.util.StringUtils; /** @@ -15,18 +16,20 @@ public class AgentStreamJettyConfigParser implements ModuleConfigParser { public static final String CONTEXT_PATH = "contextPath"; @Override public void parse(Map config) throws ConfigParseException { - AgentStreamJettyConfig.HOST = (String)config.get(HOST); AgentStreamJettyConfig.CONTEXT_PATH = "/"; - if (StringUtils.isEmpty(AgentStreamJettyConfig.HOST)) { - throw new ConfigParseException(""); + if (ObjectUtils.isEmpty(config) || StringUtils.isEmpty(config.get(HOST))) { + AgentStreamJettyConfig.HOST = "localhost"; + } else { + AgentStreamJettyConfig.HOST = (String)config.get(HOST); } - if (StringUtils.isEmpty(config.get(PORT))) { - throw new ConfigParseException(""); + + if (ObjectUtils.isEmpty(config) || StringUtils.isEmpty(config.get(PORT))) { + AgentStreamJettyConfig.PORT = 12800; } else { AgentStreamJettyConfig.PORT = (Integer)config.get(PORT); } - if (!StringUtils.isEmpty(config.get(CONTEXT_PATH))) { + if (ObjectUtils.isNotEmpty(config) && StringUtils.isNotEmpty(config.get(CONTEXT_PATH))) { AgentStreamJettyConfig.CONTEXT_PATH = (String)config.get(CONTEXT_PATH); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyDataListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyDataListener.java new file mode 100644 index 000000000..375c84754 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyDataListener.java @@ -0,0 +1,20 @@ +package org.skywalking.apm.collector.agentstream.jetty; + +import org.skywalking.apm.collector.agentstream.AgentStreamModuleGroupDefine; +import org.skywalking.apm.collector.cluster.ClusterModuleDefine; +import org.skywalking.apm.collector.core.cluster.ClusterDataListener; + +/** + * @author pengys5 + */ +public class AgentStreamJettyDataListener extends ClusterDataListener { + + public static final String PATH = ClusterModuleDefine.BASE_CATALOG + "." + AgentStreamModuleGroupDefine.GROUP_NAME + "." + AgentStreamJettyModuleDefine.MODULE_NAME; + + @Override public String path() { + return PATH; + } + + @Override public void addressChangedNotify() { + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/jetty/AgentStreamJettyModuleDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyModuleDefine.java similarity index 53% rename from apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/jetty/AgentStreamJettyModuleDefine.java rename to apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyModuleDefine.java index b23de9696..d20996ebd 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/jetty/AgentStreamJettyModuleDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyModuleDefine.java @@ -1,8 +1,11 @@ -package org.skywalking.apm.collector.agent.stream.server.jetty; +package org.skywalking.apm.collector.agentstream.jetty; -import org.skywalking.apm.collector.core.agentstream.AgentStreamModuleDefine; +import java.util.List; +import org.skywalking.apm.collector.agentstream.AgentStreamModuleDefine; +import org.skywalking.apm.collector.agentstream.AgentStreamModuleGroupDefine; +import org.skywalking.apm.collector.core.cluster.ClusterDataListener; +import org.skywalking.apm.collector.core.framework.Handler; import org.skywalking.apm.collector.core.module.ModuleConfigParser; -import org.skywalking.apm.collector.core.module.ModuleGroup; import org.skywalking.apm.collector.core.module.ModuleRegistration; import org.skywalking.apm.collector.core.server.Server; import org.skywalking.apm.collector.server.jetty.JettyServer; @@ -12,16 +15,14 @@ import org.skywalking.apm.collector.server.jetty.JettyServer; */ public class AgentStreamJettyModuleDefine extends AgentStreamModuleDefine { - @Override protected ModuleGroup group() { - return ModuleGroup.AgentStream; + public static final String MODULE_NAME = "jetty"; + + @Override protected String group() { + return AgentStreamModuleGroupDefine.GROUP_NAME; } @Override public String name() { - return "jetty"; - } - - @Override public boolean defaultModule() { - return false; + return MODULE_NAME; } @Override protected ModuleConfigParser configParser() { @@ -35,4 +36,12 @@ public class AgentStreamJettyModuleDefine extends AgentStreamModuleDefine { @Override protected ModuleRegistration registration() { return new AgentStreamJettyModuleRegistration(); } + + @Override public ClusterDataListener listener() { + return new AgentStreamJettyDataListener(); + } + + @Override public List handlerList() { + return null; + } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/jetty/AgentStreamJettyModuleRegistration.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyModuleRegistration.java similarity index 51% rename from apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/jetty/AgentStreamJettyModuleRegistration.java rename to apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyModuleRegistration.java index 82ac5aa62..0b6527306 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agent/stream/server/jetty/AgentStreamJettyModuleRegistration.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyModuleRegistration.java @@ -1,6 +1,5 @@ -package org.skywalking.apm.collector.agent.stream.server.jetty; +package org.skywalking.apm.collector.agentstream.jetty; -import com.google.gson.JsonObject; import org.skywalking.apm.collector.core.module.ModuleRegistration; /** @@ -9,8 +8,6 @@ import org.skywalking.apm.collector.core.module.ModuleRegistration; public class AgentStreamJettyModuleRegistration extends ModuleRegistration { @Override public Value buildValue() { - JsonObject data = new JsonObject(); - data.addProperty(AgentStreamJettyConfigParser.CONTEXT_PATH, AgentStreamJettyConfig.CONTEXT_PATH); - return new Value(AgentStreamJettyConfig.HOST, AgentStreamJettyConfig.PORT, data); + return new Value(AgentStreamJettyConfig.HOST, AgentStreamJettyConfig.PORT, AgentStreamJettyConfig.CONTEXT_PATH); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/AgentStreamModuleDefineException.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/AgentStreamModuleDefineException.java new file mode 100644 index 000000000..c57b297fb --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/AgentStreamModuleDefineException.java @@ -0,0 +1,17 @@ +package org.skywalking.apm.collector.agentstream.worker; + +import org.skywalking.apm.collector.core.framework.DefineException; + +/** + * @author pengys5 + */ +public class AgentStreamModuleDefineException extends DefineException { + + public AgentStreamModuleDefineException(String message) { + super(message); + } + + public AgentStreamModuleDefineException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/CommonTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/CommonTable.java new file mode 100644 index 000000000..c66f627ad --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/CommonTable.java @@ -0,0 +1,11 @@ +package org.skywalking.apm.collector.agentstream.worker; + +/** + * @author pengys5 + */ +public class CommonTable { + public static final String TABLE_TYPE = "type"; + public static final String COLUMN_ID = "id"; + public static final String COLUMN_AGG = "agg"; + public static final String COLUMN_TIME_BUCKET = "time_bucket"; +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/Const.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/Const.java new file mode 100644 index 000000000..b0d8ec415 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/Const.java @@ -0,0 +1,13 @@ +package org.skywalking.apm.collector.agentstream.worker; + +/** + * @author pengys5 + */ +public class Const { + public static final String ID_SPLIT = "..-.."; + public static final String IDS_SPLIT = "\\.\\.-\\.\\."; + public static final String PEERS_FRONT_SPLIT = "["; + public static final String PEERS_BEHIND_SPLIT = "]"; + public static final String USER_CODE = "User"; + public static final String SEGMENT_SPAN_SPLIT = "S"; +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/config/CacheSizeConfig.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/config/CacheSizeConfig.java new file mode 100644 index 000000000..fe792ff4f --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/config/CacheSizeConfig.java @@ -0,0 +1,17 @@ +package org.skywalking.apm.collector.agentstream.worker.config; + +/** + * @author pengys5 + */ +public class CacheSizeConfig { + + public static class Cache { + public static class Analysis { + public static int SIZE = 1024; + } + + public static class Persistence { + public static int SIZE = 5000; + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/config/WorkerConfig.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/config/WorkerConfig.java new file mode 100644 index 000000000..9c94a6ae8 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/config/WorkerConfig.java @@ -0,0 +1,125 @@ +package org.skywalking.apm.collector.agentstream.worker.config; + +/** + * @author pengys5 + */ +public class WorkerConfig { + + public static class WorkerNum { + public static class Node { + public static class NodeCompAgg { + public static int VALUE = 2; + } + + public static class NodeMappingDayAgg { + public static int VALUE = 2; + } + + public static class NodeMappingHourAgg { + public static int VALUE = 2; + } + + public static class NodeMappingMinuteAgg { + public static int VALUE = 2; + } + } + + public static class NodeRef { + public static class NodeRefDayAgg { + public static int VALUE = 2; + } + + public static class NodeRefHourAgg { + public static int VALUE = 2; + } + + public static class NodeRefMinuteAgg { + public static int VALUE = 2; + } + + public static class NodeRefResSumDayAgg { + public static int VALUE = 2; + } + + public static class NodeRefResSumHourAgg { + public static int VALUE = 2; + } + + public static class NodeRefResSumMinuteAgg { + public static int VALUE = 2; + } + } + + public static class GlobalTrace { + public static class GlobalTraceAgg { + public static int VALUE = 2; + } + } + } + + public static class Queue { + public static class GlobalTrace { + public static class GlobalTraceAnalysis { + public static int SIZE = 1024; + } + } + + public static class Segment { + public static class SegmentAnalysis { + public static int SIZE = 1024; + } + + public static class SegmentCostAnalysis { + public static int SIZE = 4096; + } + + public static class SegmentExceptionAnalysis { + public static int SIZE = 4096; + } + } + + public static class Node { + public static class NodeCompAnalysis { + public static int SIZE = 1024; + } + + public static class NodeMappingDayAnalysis { + public static int SIZE = 1024; + } + + public static class NodeMappingHourAnalysis { + public static int SIZE = 1024; + } + + public static class NodeMappingMinuteAnalysis { + public static int SIZE = 1024; + } + } + + public static class NodeRef { + public static class NodeRefDayAnalysis { + public static int SIZE = 1024; + } + + public static class NodeRefHourAnalysis { + public static int SIZE = 1024; + } + + public static class NodeRefMinuteAnalysis { + public static int SIZE = 1024; + } + + public static class NodeRefResSumDayAnalysis { + public static int SIZE = 1024; + } + + public static class NodeRefResSumHourAnalysis { + public static int SIZE = 1024; + } + + public static class NodeRefResSumMinuteAnalysis { + public static int SIZE = 1024; + } + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/GlobalTracePersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/GlobalTracePersistenceWorker.java new file mode 100644 index 000000000..ba58718b6 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/GlobalTracePersistenceWorker.java @@ -0,0 +1,70 @@ +package org.skywalking.apm.collector.agentstream.worker.global; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.agentstream.worker.global.dao.IGlobalTraceDAO; +import org.skywalking.apm.collector.agentstream.worker.global.define.GlobalTraceDataDefine; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.RollingSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class GlobalTracePersistenceWorker extends PersistenceWorker { + + public GlobalTracePersistenceWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + super.preStart(); + } + + @Override protected List prepareBatch(Map dataMap) { + IGlobalTraceDAO dao = (IGlobalTraceDAO)DAOContainer.INSTANCE.get(IGlobalTraceDAO.class.getName()); + return dao.prepareBatch(dataMap); + } + + public static class Factory extends AbstractLocalAsyncWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public GlobalTracePersistenceWorker workerInstance(ClusterWorkerContext clusterContext) { + return new GlobalTracePersistenceWorker(role(), clusterContext); + } + + @Override + public int queueSize() { + return 1024; + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return GlobalTracePersistenceWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new RollingSelector(); + } + + @Override public DataDefine dataDefine() { + return new GlobalTraceDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/GlobalTraceSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/GlobalTraceSpanListener.java new file mode 100644 index 000000000..64c3c1920 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/GlobalTraceSpanListener.java @@ -0,0 +1,60 @@ +package org.skywalking.apm.collector.agentstream.worker.global; + +import java.util.ArrayList; +import java.util.List; +import org.skywalking.apm.collector.agentstream.worker.global.define.GlobalTraceDataDefine; +import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.GlobalTraceIdsListener; +import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.stream.StreamModuleContext; +import org.skywalking.apm.collector.stream.StreamModuleGroupDefine; +import org.skywalking.apm.collector.stream.worker.WorkerInvokeException; +import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; +import org.skywalking.apm.network.proto.SpanObject; +import org.skywalking.apm.network.proto.UniqueId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class GlobalTraceSpanListener implements FirstSpanListener, GlobalTraceIdsListener { + + private final Logger logger = LoggerFactory.getLogger(GlobalTraceSpanListener.class); + + private List globalTraceIds = new ArrayList<>(); + private String segmentId; + private long timeBucket; + + @Override + public void parseFirst(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + this.timeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(spanObject.getStartTime()); + this.segmentId = segmentId; + } + + @Override public void parseGlobalTraceId(UniqueId uniqueId) { + StringBuilder globalTraceIdBuilder = new StringBuilder(); + uniqueId.getIdPartsList().forEach(globalTraceIdBuilder::append); + globalTraceIds.add(globalTraceIdBuilder.toString()); + } + + @Override public void build() { + logger.debug("global trace listener build"); + StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); + + for (String globalTraceId : globalTraceIds) { + GlobalTraceDataDefine.GlobalTrace globalTrace = new GlobalTraceDataDefine.GlobalTrace(); + globalTrace.setGlobalTraceId(globalTraceId); + globalTrace.setId(segmentId + globalTraceId); + globalTrace.setSegmentId(segmentId); + globalTrace.setTimeBucket(timeBucket); + try { + logger.debug("send to global trace persistence worker, id: {}", globalTrace.getId()); + context.getClusterWorkerContext().lookup(GlobalTracePersistenceWorker.WorkerRole.INSTANCE).tell(globalTrace.transform()); + } catch (WorkerInvokeException | WorkerNotFoundException e) { + logger.error(e.getMessage(), e); + } + } + } +} \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/GlobalTraceEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/GlobalTraceEsDAO.java new file mode 100644 index 000000000..4c1c49707 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/GlobalTraceEsDAO.java @@ -0,0 +1,35 @@ +package org.skywalking.apm.collector.agentstream.worker.global.dao; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.skywalking.apm.collector.agentstream.worker.global.define.GlobalTraceTable; +import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class GlobalTraceEsDAO extends EsDAO implements IGlobalTraceDAO { + + private final Logger logger = LoggerFactory.getLogger(GlobalTraceEsDAO.class); + + @Override public List prepareBatch(Map dataMap) { + List indexRequestBuilders = new ArrayList<>(); + dataMap.forEach((id, data) -> { + logger.debug("global trace prepareBatch, id: {}", id); + Map source = new HashMap(); + source.put(GlobalTraceTable.COLUMN_SEGMENT_ID, data.getDataString(1)); + source.put(GlobalTraceTable.COLUMN_GLOBAL_TRACE_ID, data.getDataString(2)); + source.put(GlobalTraceTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + logger.debug("global trace source: {}", source.toString()); + IndexRequestBuilder builder = getClient().prepareIndex(GlobalTraceTable.TABLE, id).setSource(source); + indexRequestBuilders.add(builder); + }); + return indexRequestBuilders; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/GlobalTraceH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/GlobalTraceH2DAO.java new file mode 100644 index 000000000..1a34c616a --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/GlobalTraceH2DAO.java @@ -0,0 +1,15 @@ +package org.skywalking.apm.collector.agentstream.worker.global.dao; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.storage.h2.dao.H2DAO; + +/** + * @author pengys5 + */ +public class GlobalTraceH2DAO extends H2DAO implements IGlobalTraceDAO { + + @Override public List prepareBatch(Map map) { + return null; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/IGlobalTraceDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/IGlobalTraceDAO.java new file mode 100644 index 000000000..4138f423c --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/IGlobalTraceDAO.java @@ -0,0 +1,12 @@ +package org.skywalking.apm.collector.agentstream.worker.global.dao; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; + +/** + * @author pengys5 + */ +public interface IGlobalTraceDAO { + List prepareBatch(Map dataMap); +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/define/GlobalTraceDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/define/GlobalTraceDataDefine.java new file mode 100644 index 000000000..671c20232 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/define/GlobalTraceDataDefine.java @@ -0,0 +1,108 @@ +package org.skywalking.apm.collector.agentstream.worker.global.define; + +import org.skywalking.apm.collector.remote.grpc.proto.RemoteData; +import org.skywalking.apm.collector.stream.worker.impl.data.Attribute; +import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.impl.data.TransformToData; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation; + +/** + * @author pengys5 + */ +public class GlobalTraceDataDefine extends DataDefine { + + @Override public int defineId() { + return 403; + } + + @Override protected int initialCapacity() { + return 4; + } + + @Override protected void attributeDefine() { + addAttribute(0, new Attribute(GlobalTraceTable.COLUMN_ID, AttributeType.STRING, new NonOperation())); + addAttribute(1, new Attribute(GlobalTraceTable.COLUMN_SEGMENT_ID, AttributeType.STRING, new CoverOperation())); + addAttribute(2, new Attribute(GlobalTraceTable.COLUMN_GLOBAL_TRACE_ID, AttributeType.STRING, new CoverOperation())); + addAttribute(3, new Attribute(GlobalTraceTable.COLUMN_TIME_BUCKET, AttributeType.LONG, new CoverOperation())); + } + + @Override public Object deserialize(RemoteData remoteData) { + String id = remoteData.getDataStrings(0); + String segmentId = remoteData.getDataStrings(1); + String globalTraceId = remoteData.getDataStrings(2); + Long timeBucket = remoteData.getDataLongs(0); + return new GlobalTrace(id, segmentId, globalTraceId, timeBucket); + } + + @Override public RemoteData serialize(Object object) { + GlobalTrace globalTrace = (GlobalTrace)object; + RemoteData.Builder builder = RemoteData.newBuilder(); + builder.addDataStrings(globalTrace.getId()); + builder.addDataStrings(globalTrace.getSegmentId()); + builder.addDataStrings(globalTrace.getGlobalTraceId()); + builder.addDataLongs(globalTrace.getTimeBucket()); + return builder.build(); + } + + public static class GlobalTrace implements TransformToData { + private String id; + private String segmentId; + private String globalTraceId; + private long timeBucket; + + GlobalTrace(String id, String segmentId, String globalTraceId, long timeBucket) { + this.id = id; + this.segmentId = segmentId; + this.globalTraceId = globalTraceId; + this.timeBucket = timeBucket; + } + + public GlobalTrace() { + } + + @Override public Data transform() { + GlobalTraceDataDefine define = new GlobalTraceDataDefine(); + Data data = define.build(id); + data.setDataString(0, this.id); + data.setDataString(1, this.segmentId); + data.setDataString(2, this.globalTraceId); + data.setDataLong(0, this.timeBucket); + return data; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getSegmentId() { + return segmentId; + } + + public void setSegmentId(String segmentId) { + this.segmentId = segmentId; + } + + public String getGlobalTraceId() { + return globalTraceId; + } + + public void setGlobalTraceId(String globalTraceId) { + this.globalTraceId = globalTraceId; + } + + public long getTimeBucket() { + return timeBucket; + } + + public void setTimeBucket(long timeBucket) { + this.timeBucket = timeBucket; + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/define/GlobalTraceEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/define/GlobalTraceEsTableDefine.java new file mode 100644 index 000000000..e780b3b75 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/define/GlobalTraceEsTableDefine.java @@ -0,0 +1,32 @@ +package org.skywalking.apm.collector.agentstream.worker.global.define; + +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine; +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine; + +/** + * @author pengys5 + */ +public class GlobalTraceEsTableDefine extends ElasticSearchTableDefine { + + public GlobalTraceEsTableDefine() { + super(GlobalTraceTable.TABLE); + } + + @Override public int refreshInterval() { + return 2; + } + + @Override public int numberOfShards() { + return 2; + } + + @Override public int numberOfReplicas() { + return 0; + } + + @Override public void initialize() { + addColumn(new ElasticSearchColumnDefine(GlobalTraceTable.COLUMN_SEGMENT_ID, ElasticSearchColumnDefine.Type.Keyword.name())); + addColumn(new ElasticSearchColumnDefine(GlobalTraceTable.COLUMN_GLOBAL_TRACE_ID, ElasticSearchColumnDefine.Type.Keyword.name())); + addColumn(new ElasticSearchColumnDefine(GlobalTraceTable.COLUMN_TIME_BUCKET, ElasticSearchColumnDefine.Type.Long.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/define/GlobalTraceH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/define/GlobalTraceH2TableDefine.java new file mode 100644 index 000000000..089003510 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/define/GlobalTraceH2TableDefine.java @@ -0,0 +1,21 @@ +package org.skywalking.apm.collector.agentstream.worker.global.define; + +import org.skywalking.apm.collector.storage.h2.define.H2ColumnDefine; +import org.skywalking.apm.collector.storage.h2.define.H2TableDefine; + +/** + * @author pengys5 + */ +public class GlobalTraceH2TableDefine extends H2TableDefine { + + public GlobalTraceH2TableDefine() { + super(GlobalTraceTable.TABLE); + } + + @Override public void initialize() { + addColumn(new H2ColumnDefine(GlobalTraceTable.COLUMN_ID, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(GlobalTraceTable.COLUMN_SEGMENT_ID, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(GlobalTraceTable.COLUMN_GLOBAL_TRACE_ID, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(GlobalTraceTable.COLUMN_TIME_BUCKET, H2ColumnDefine.Type.Bigint.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/define/GlobalTraceTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/define/GlobalTraceTable.java new file mode 100644 index 000000000..9c6ab7fc4 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/define/GlobalTraceTable.java @@ -0,0 +1,12 @@ +package org.skywalking.apm.collector.agentstream.worker.global.define; + +import org.skywalking.apm.collector.agentstream.worker.CommonTable; + +/** + * @author pengys5 + */ +public class GlobalTraceTable extends CommonTable { + public static final String TABLE = "global_trace"; + public static final String COLUMN_SEGMENT_ID = "segment_id"; + public static final String COLUMN_GLOBAL_TRACE_ID = "global_trace_id"; +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentAggregationWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentAggregationWorker.java new file mode 100644 index 000000000..85349b10c --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentAggregationWorker.java @@ -0,0 +1,66 @@ +package org.skywalking.apm.collector.agentstream.worker.node.component; + +import org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentDataDefine; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; +import org.skywalking.apm.collector.stream.worker.WorkerRefs; +import org.skywalking.apm.collector.stream.worker.impl.AggregationWorker; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class NodeComponentAggregationWorker extends AggregationWorker { + + public NodeComponentAggregationWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + super.preStart(); + } + + @Override protected WorkerRefs nextWorkRef(String id) throws WorkerNotFoundException { + return getClusterContext().lookup(NodeComponentRemoteWorker.WorkerRole.INSTANCE); + } + + public static class Factory extends AbstractLocalAsyncWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public NodeComponentAggregationWorker workerInstance(ClusterWorkerContext clusterContext) { + return new NodeComponentAggregationWorker(role(), clusterContext); + } + + @Override + public int queueSize() { + return 1024; + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return NodeComponentAggregationWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new HashCodeSelector(); + } + + @Override public DataDefine dataDefine() { + return new NodeComponentDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentPersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentPersistenceWorker.java new file mode 100644 index 000000000..2e7ccf0b9 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentPersistenceWorker.java @@ -0,0 +1,70 @@ +package org.skywalking.apm.collector.agentstream.worker.node.component; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.agentstream.worker.node.component.dao.INodeComponentDAO; +import org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentDataDefine; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class NodeComponentPersistenceWorker extends PersistenceWorker { + + public NodeComponentPersistenceWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + super.preStart(); + } + + @Override protected List prepareBatch(Map dataMap) { + INodeComponentDAO dao = (INodeComponentDAO)DAOContainer.INSTANCE.get(INodeComponentDAO.class.getName()); + return dao.prepareBatch(dataMap); + } + + public static class Factory extends AbstractLocalAsyncWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public NodeComponentPersistenceWorker workerInstance(ClusterWorkerContext clusterContext) { + return new NodeComponentPersistenceWorker(role(), clusterContext); + } + + @Override + public int queueSize() { + return 1024; + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return NodeComponentPersistenceWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new HashCodeSelector(); + } + + @Override public DataDefine dataDefine() { + return new NodeComponentDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentRemoteWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentRemoteWorker.java new file mode 100644 index 000000000..8ed1e4114 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentRemoteWorker.java @@ -0,0 +1,60 @@ +package org.skywalking.apm.collector.agentstream.worker.node.component; + +import org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentDataDefine; +import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorker; +import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerException; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class NodeComponentRemoteWorker extends AbstractRemoteWorker { + + protected NodeComponentRemoteWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + + } + + @Override protected void onWork(Object message) throws WorkerException { + getClusterContext().lookup(NodeComponentPersistenceWorker.WorkerRole.INSTANCE).tell(message); + } + + public static class Factory extends AbstractRemoteWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public NodeComponentRemoteWorker workerInstance(ClusterWorkerContext clusterContext) { + return new NodeComponentRemoteWorker(role(), clusterContext); + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return NodeComponentRemoteWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new HashCodeSelector(); + } + + @Override public DataDefine dataDefine() { + return new NodeComponentDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java new file mode 100644 index 000000000..304018b1c --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java @@ -0,0 +1,67 @@ +package org.skywalking.apm.collector.agentstream.worker.node.component; + +import java.util.ArrayList; +import java.util.List; +import org.skywalking.apm.collector.agentstream.worker.Const; +import org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentDataDefine; +import org.skywalking.apm.collector.agentstream.worker.segment.EntrySpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.ExitSpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener; +import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.stream.StreamModuleContext; +import org.skywalking.apm.collector.stream.StreamModuleGroupDefine; +import org.skywalking.apm.collector.stream.worker.WorkerInvokeException; +import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; +import org.skywalking.apm.network.proto.SpanObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class NodeComponentSpanListener implements EntrySpanListener, ExitSpanListener, FirstSpanListener { + + private final Logger logger = LoggerFactory.getLogger(NodeComponentSpanListener.class); + + private List nodeComponents = new ArrayList<>(); + private long timeBucket; + + @Override + public void parseExit(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + String peers = spanObject.getPeer(); + if (spanObject.getPeerId() == 0) { + peers = String.valueOf(spanObject.getPeerId()); + } + String agg = spanObject.getComponentId() + Const.ID_SPLIT + peers; + nodeComponents.add(agg); + } + + @Override + public void parseEntry(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + String peers = String.valueOf(applicationId); + String agg = spanObject.getComponentId() + Const.ID_SPLIT + peers; + nodeComponents.add(agg); + } + + @Override + public void parseFirst(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + timeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(spanObject.getStartTime()); + } + + @Override public void build() { + StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); + for (String agg : nodeComponents) { + NodeComponentDataDefine.NodeComponent nodeComponent = new NodeComponentDataDefine.NodeComponent(); + nodeComponent.setId(timeBucket + Const.ID_SPLIT + agg); + nodeComponent.setAgg(agg); + nodeComponent.setTimeBucket(timeBucket); + try { + logger.debug("send to node component aggregation worker, id: {}", nodeComponent.getId()); + context.getClusterWorkerContext().lookup(NodeComponentAggregationWorker.WorkerRole.INSTANCE).tell(nodeComponent.transform()); + } catch (WorkerInvokeException | WorkerNotFoundException e) { + logger.error(e.getMessage(), e); + } + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/INodeComponentDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/INodeComponentDAO.java new file mode 100644 index 000000000..ffd9087fd --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/INodeComponentDAO.java @@ -0,0 +1,12 @@ +package org.skywalking.apm.collector.agentstream.worker.node.component.dao; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; + +/** + * @author pengys5 + */ +public interface INodeComponentDAO { + List prepareBatch(Map dataMap); +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentEsDAO.java new file mode 100644 index 000000000..012b508e6 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentEsDAO.java @@ -0,0 +1,29 @@ +package org.skywalking.apm.collector.agentstream.worker.node.component.dao; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentTable; +import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; + +/** + * @author pengys5 + */ +public class NodeComponentEsDAO extends EsDAO implements INodeComponentDAO { + + @Override public List prepareBatch(Map dataMap) { + List indexRequestBuilders = new ArrayList<>(); + dataMap.forEach((id, data) -> { + Map source = new HashMap(); + source.put(NodeComponentTable.COLUMN_AGG, data.getDataString(1)); + source.put(NodeComponentTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + + IndexRequestBuilder builder = getClient().prepareIndex(NodeComponentTable.TABLE, id).setSource(source); + indexRequestBuilders.add(builder); + }); + return indexRequestBuilders; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentH2DAO.java new file mode 100644 index 000000000..c581fc5ed --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentH2DAO.java @@ -0,0 +1,15 @@ +package org.skywalking.apm.collector.agentstream.worker.node.component.dao; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.storage.h2.dao.H2DAO; + +/** + * @author pengys5 + */ +public class NodeComponentH2DAO extends H2DAO implements INodeComponentDAO { + + @Override public List prepareBatch(Map map) { + return null; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java new file mode 100644 index 000000000..caa691ce9 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java @@ -0,0 +1,86 @@ +package org.skywalking.apm.collector.agentstream.worker.node.component.define; + +import org.skywalking.apm.collector.remote.grpc.proto.RemoteData; +import org.skywalking.apm.collector.stream.worker.impl.data.Attribute; +import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.impl.data.TransformToData; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation; + +/** + * @author pengys5 + */ +public class NodeComponentDataDefine extends DataDefine { + + @Override public int defineId() { + return 101; + } + + @Override protected int initialCapacity() { + return 3; + } + + @Override protected void attributeDefine() { + addAttribute(0, new Attribute(NodeComponentTable.COLUMN_ID, AttributeType.STRING, new NonOperation())); + addAttribute(1, new Attribute(NodeComponentTable.COLUMN_AGG, AttributeType.STRING, new CoverOperation())); + addAttribute(2, new Attribute(NodeComponentTable.COLUMN_TIME_BUCKET, AttributeType.LONG, new CoverOperation())); + } + + @Override public Object deserialize(RemoteData remoteData) { + return null; + } + + @Override public RemoteData serialize(Object object) { + return null; + } + + public static class NodeComponent implements TransformToData { + private String id; + private String agg; + private long timeBucket; + + public NodeComponent(String id, String agg, long timeBucket) { + this.id = id; + this.agg = agg; + this.timeBucket = timeBucket; + } + + public NodeComponent() { + } + + @Override public Data transform() { + NodeComponentDataDefine define = new NodeComponentDataDefine(); + Data data = define.build(id); + data.setDataString(0, this.id); + data.setDataString(1, this.agg); + data.setDataLong(0, this.timeBucket); + return data; + } + + public String getId() { + return id; + } + + public String getAgg() { + return agg; + } + + public long getTimeBucket() { + return timeBucket; + } + + public void setId(String id) { + this.id = id; + } + + public void setAgg(String agg) { + this.agg = agg; + } + + public void setTimeBucket(long timeBucket) { + this.timeBucket = timeBucket; + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentEsTableDefine.java new file mode 100644 index 000000000..e4e8e2809 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentEsTableDefine.java @@ -0,0 +1,31 @@ +package org.skywalking.apm.collector.agentstream.worker.node.component.define; + +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine; +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine; + +/** + * @author pengys5 + */ +public class NodeComponentEsTableDefine extends ElasticSearchTableDefine { + + public NodeComponentEsTableDefine() { + super(NodeComponentTable.TABLE); + } + + @Override public int refreshInterval() { + return 2; + } + + @Override public int numberOfShards() { + return 2; + } + + @Override public int numberOfReplicas() { + return 0; + } + + @Override public void initialize() { + addColumn(new ElasticSearchColumnDefine(NodeComponentTable.COLUMN_AGG, ElasticSearchColumnDefine.Type.Keyword.name())); + addColumn(new ElasticSearchColumnDefine(NodeComponentTable.COLUMN_TIME_BUCKET, ElasticSearchColumnDefine.Type.Long.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentH2TableDefine.java new file mode 100644 index 000000000..62b77b04e --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentH2TableDefine.java @@ -0,0 +1,20 @@ +package org.skywalking.apm.collector.agentstream.worker.node.component.define; + +import org.skywalking.apm.collector.storage.h2.define.H2ColumnDefine; +import org.skywalking.apm.collector.storage.h2.define.H2TableDefine; + +/** + * @author pengys5 + */ +public class NodeComponentH2TableDefine extends H2TableDefine { + + public NodeComponentH2TableDefine() { + super(NodeComponentTable.TABLE); + } + + @Override public void initialize() { + addColumn(new H2ColumnDefine(NodeComponentTable.COLUMN_ID, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(NodeComponentTable.COLUMN_AGG, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(NodeComponentTable.COLUMN_TIME_BUCKET, H2ColumnDefine.Type.Bigint.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentTable.java new file mode 100644 index 000000000..1bce8df83 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentTable.java @@ -0,0 +1,10 @@ +package org.skywalking.apm.collector.agentstream.worker.node.component.define; + +import org.skywalking.apm.collector.agentstream.worker.CommonTable; + +/** + * @author pengys5 + */ +public class NodeComponentTable extends CommonTable { + public static final String TABLE = "node_component"; +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingAggregationWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingAggregationWorker.java new file mode 100644 index 000000000..3d33a2a6c --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingAggregationWorker.java @@ -0,0 +1,66 @@ +package org.skywalking.apm.collector.agentstream.worker.node.mapping; + +import org.skywalking.apm.collector.agentstream.worker.node.mapping.define.NodeMappingDataDefine; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; +import org.skywalking.apm.collector.stream.worker.WorkerRefs; +import org.skywalking.apm.collector.stream.worker.impl.AggregationWorker; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class NodeMappingAggregationWorker extends AggregationWorker { + + public NodeMappingAggregationWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + super.preStart(); + } + + @Override protected WorkerRefs nextWorkRef(String id) throws WorkerNotFoundException { + return getClusterContext().lookup(NodeMappingRemoteWorker.WorkerRole.INSTANCE); + } + + public static class Factory extends AbstractLocalAsyncWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public NodeMappingAggregationWorker workerInstance(ClusterWorkerContext clusterContext) { + return new NodeMappingAggregationWorker(role(), clusterContext); + } + + @Override + public int queueSize() { + return 1024; + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return NodeMappingAggregationWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new HashCodeSelector(); + } + + @Override public DataDefine dataDefine() { + return new NodeMappingDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingPersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingPersistenceWorker.java new file mode 100644 index 000000000..48a882d5a --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingPersistenceWorker.java @@ -0,0 +1,70 @@ +package org.skywalking.apm.collector.agentstream.worker.node.mapping; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.agentstream.worker.node.mapping.dao.INodeMappingDAO; +import org.skywalking.apm.collector.agentstream.worker.node.mapping.define.NodeMappingDataDefine; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class NodeMappingPersistenceWorker extends PersistenceWorker { + + public NodeMappingPersistenceWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + super.preStart(); + } + + @Override protected List prepareBatch(Map dataMap) { + INodeMappingDAO dao = (INodeMappingDAO)DAOContainer.INSTANCE.get(INodeMappingDAO.class.getName()); + return dao.prepareBatch(dataMap); + } + + public static class Factory extends AbstractLocalAsyncWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public NodeMappingPersistenceWorker workerInstance(ClusterWorkerContext clusterContext) { + return new NodeMappingPersistenceWorker(role(), clusterContext); + } + + @Override + public int queueSize() { + return 1024; + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return NodeMappingPersistenceWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new HashCodeSelector(); + } + + @Override public DataDefine dataDefine() { + return new NodeMappingDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingRemoteWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingRemoteWorker.java new file mode 100644 index 000000000..e5ccaaad4 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingRemoteWorker.java @@ -0,0 +1,60 @@ +package org.skywalking.apm.collector.agentstream.worker.node.mapping; + +import org.skywalking.apm.collector.agentstream.worker.node.mapping.define.NodeMappingDataDefine; +import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorker; +import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerException; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class NodeMappingRemoteWorker extends AbstractRemoteWorker { + + protected NodeMappingRemoteWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + + } + + @Override protected void onWork(Object message) throws WorkerException { + getClusterContext().lookup(NodeMappingPersistenceWorker.WorkerRole.INSTANCE).tell(message); + } + + public static class Factory extends AbstractRemoteWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public NodeMappingRemoteWorker workerInstance(ClusterWorkerContext clusterContext) { + return new NodeMappingRemoteWorker(role(), clusterContext); + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return NodeMappingRemoteWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new HashCodeSelector(); + } + + @Override public DataDefine dataDefine() { + return new NodeMappingDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingSpanListener.java new file mode 100644 index 000000000..42bec6d17 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingSpanListener.java @@ -0,0 +1,64 @@ +package org.skywalking.apm.collector.agentstream.worker.node.mapping; + +import java.util.ArrayList; +import java.util.List; +import org.skywalking.apm.collector.agentstream.worker.Const; +import org.skywalking.apm.collector.agentstream.worker.node.mapping.define.NodeMappingDataDefine; +import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.RefsListener; +import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.stream.StreamModuleContext; +import org.skywalking.apm.collector.stream.StreamModuleGroupDefine; +import org.skywalking.apm.collector.stream.worker.WorkerInvokeException; +import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; +import org.skywalking.apm.network.proto.SpanObject; +import org.skywalking.apm.network.proto.TraceSegmentReference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class NodeMappingSpanListener implements RefsListener, FirstSpanListener { + + private final Logger logger = LoggerFactory.getLogger(NodeMappingSpanListener.class); + + private List nodeMappings = new ArrayList<>(); + private long timeBucket; + + @Override public void parseRef(TraceSegmentReference reference, int applicationId, int applicationInstanceId, + String segmentId) { + logger.debug("node mapping listener parse reference"); + String peers = Const.PEERS_FRONT_SPLIT + reference.getNetworkAddressId() + Const.PEERS_BEHIND_SPLIT; + if (reference.getNetworkAddressId() == 0) { + peers = Const.PEERS_FRONT_SPLIT + reference.getNetworkAddress() + Const.PEERS_BEHIND_SPLIT; + } + + String agg = applicationId + Const.ID_SPLIT + peers; + nodeMappings.add(agg); + } + + @Override + public void parseFirst(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + timeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(spanObject.getStartTime()); + } + + @Override public void build() { + logger.debug("node mapping listener build"); + StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); + for (String agg : nodeMappings) { + NodeMappingDataDefine.NodeMapping nodeMapping = new NodeMappingDataDefine.NodeMapping(); + nodeMapping.setId(timeBucket + Const.ID_SPLIT + agg); + nodeMapping.setAgg(agg); + nodeMapping.setTimeBucket(timeBucket); + + try { + logger.debug("send to node mapping aggregation worker, id: {}", nodeMapping.getId()); + context.getClusterWorkerContext().lookup(NodeMappingAggregationWorker.WorkerRole.INSTANCE).tell(nodeMapping.transform()); + } catch (WorkerInvokeException | WorkerNotFoundException e) { + logger.error(e.getMessage(), e); + } + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/INodeMappingDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/INodeMappingDAO.java new file mode 100644 index 000000000..f864617a2 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/INodeMappingDAO.java @@ -0,0 +1,12 @@ +package org.skywalking.apm.collector.agentstream.worker.node.mapping.dao; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; + +/** + * @author pengys5 + */ +public interface INodeMappingDAO { + List prepareBatch(Map dataMap); +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/NodeMappingEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/NodeMappingEsDAO.java new file mode 100644 index 000000000..4a36d0378 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/NodeMappingEsDAO.java @@ -0,0 +1,29 @@ +package org.skywalking.apm.collector.agentstream.worker.node.mapping.dao; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.skywalking.apm.collector.agentstream.worker.node.mapping.define.NodeMappingTable; +import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; + +/** + * @author pengys5 + */ +public class NodeMappingEsDAO extends EsDAO implements INodeMappingDAO { + + @Override public List prepareBatch(Map dataMap) { + List indexRequestBuilders = new ArrayList<>(); + dataMap.forEach((id, data) -> { + Map source = new HashMap(); + source.put(NodeMappingTable.COLUMN_AGG, data.getDataString(1)); + source.put(NodeMappingTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + + IndexRequestBuilder builder = getClient().prepareIndex(NodeMappingTable.TABLE, id).setSource(source); + indexRequestBuilders.add(builder); + }); + return indexRequestBuilders; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/NodeMappingH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/NodeMappingH2DAO.java new file mode 100644 index 000000000..8ad24c091 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/NodeMappingH2DAO.java @@ -0,0 +1,15 @@ +package org.skywalking.apm.collector.agentstream.worker.node.mapping.dao; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.storage.h2.dao.H2DAO; + +/** + * @author pengys5 + */ +public class NodeMappingH2DAO extends H2DAO implements INodeMappingDAO { + + @Override public List prepareBatch(Map map) { + return null; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingDataDefine.java new file mode 100644 index 000000000..2a86aa541 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingDataDefine.java @@ -0,0 +1,86 @@ +package org.skywalking.apm.collector.agentstream.worker.node.mapping.define; + +import org.skywalking.apm.collector.remote.grpc.proto.RemoteData; +import org.skywalking.apm.collector.stream.worker.impl.data.Attribute; +import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.impl.data.TransformToData; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation; + +/** + * @author pengys5 + */ +public class NodeMappingDataDefine extends DataDefine { + + @Override public int defineId() { + return 102; + } + + @Override protected int initialCapacity() { + return 3; + } + + @Override protected void attributeDefine() { + addAttribute(0, new Attribute(NodeMappingTable.COLUMN_ID, AttributeType.STRING, new NonOperation())); + addAttribute(1, new Attribute(NodeMappingTable.COLUMN_AGG, AttributeType.STRING, new CoverOperation())); + addAttribute(2, new Attribute(NodeMappingTable.COLUMN_TIME_BUCKET, AttributeType.LONG, new CoverOperation())); + } + + @Override public Object deserialize(RemoteData remoteData) { + return null; + } + + @Override public RemoteData serialize(Object object) { + return null; + } + + public static class NodeMapping implements TransformToData { + private String id; + private String agg; + private long timeBucket; + + public NodeMapping(String id, String agg, long timeBucket) { + this.id = id; + this.agg = agg; + this.timeBucket = timeBucket; + } + + public NodeMapping() { + } + + @Override public Data transform() { + NodeMappingDataDefine define = new NodeMappingDataDefine(); + Data data = define.build(id); + data.setDataString(0, this.id); + data.setDataString(1, this.agg); + data.setDataLong(0, this.timeBucket); + return data; + } + + public String getId() { + return id; + } + + public String getAgg() { + return agg; + } + + public long getTimeBucket() { + return timeBucket; + } + + public void setId(String id) { + this.id = id; + } + + public void setAgg(String agg) { + this.agg = agg; + } + + public void setTimeBucket(long timeBucket) { + this.timeBucket = timeBucket; + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingEsTableDefine.java new file mode 100644 index 000000000..018189675 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingEsTableDefine.java @@ -0,0 +1,31 @@ +package org.skywalking.apm.collector.agentstream.worker.node.mapping.define; + +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine; +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine; + +/** + * @author pengys5 + */ +public class NodeMappingEsTableDefine extends ElasticSearchTableDefine { + + public NodeMappingEsTableDefine() { + super(NodeMappingTable.TABLE); + } + + @Override public int refreshInterval() { + return 2; + } + + @Override public int numberOfShards() { + return 2; + } + + @Override public int numberOfReplicas() { + return 0; + } + + @Override public void initialize() { + addColumn(new ElasticSearchColumnDefine(NodeMappingTable.COLUMN_AGG, ElasticSearchColumnDefine.Type.Keyword.name())); + addColumn(new ElasticSearchColumnDefine(NodeMappingTable.COLUMN_TIME_BUCKET, ElasticSearchColumnDefine.Type.Long.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingH2TableDefine.java new file mode 100644 index 000000000..3805df48c --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingH2TableDefine.java @@ -0,0 +1,20 @@ +package org.skywalking.apm.collector.agentstream.worker.node.mapping.define; + +import org.skywalking.apm.collector.storage.h2.define.H2ColumnDefine; +import org.skywalking.apm.collector.storage.h2.define.H2TableDefine; + +/** + * @author pengys5 + */ +public class NodeMappingH2TableDefine extends H2TableDefine { + + public NodeMappingH2TableDefine() { + super(NodeMappingTable.TABLE); + } + + @Override public void initialize() { + addColumn(new H2ColumnDefine(NodeMappingTable.COLUMN_ID, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(NodeMappingTable.COLUMN_AGG, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(NodeMappingTable.COLUMN_TIME_BUCKET, H2ColumnDefine.Type.Bigint.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingTable.java new file mode 100644 index 000000000..811b297a2 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingTable.java @@ -0,0 +1,10 @@ +package org.skywalking.apm.collector.agentstream.worker.node.mapping.define; + +import org.skywalking.apm.collector.agentstream.worker.CommonTable; + +/** + * @author pengys5 + */ +public class NodeMappingTable extends CommonTable { + public static final String TABLE = "node_mapping"; +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefAggregationWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefAggregationWorker.java new file mode 100644 index 000000000..ad9035492 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefAggregationWorker.java @@ -0,0 +1,66 @@ +package org.skywalking.apm.collector.agentstream.worker.noderef.reference; + +import org.skywalking.apm.collector.agentstream.worker.noderef.reference.define.NodeRefDataDefine; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; +import org.skywalking.apm.collector.stream.worker.WorkerRefs; +import org.skywalking.apm.collector.stream.worker.impl.AggregationWorker; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class NodeRefAggregationWorker extends AggregationWorker { + + public NodeRefAggregationWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + super.preStart(); + } + + @Override protected WorkerRefs nextWorkRef(String id) throws WorkerNotFoundException { + return getClusterContext().lookup(NodeRefRemoteWorker.WorkerRole.INSTANCE); + } + + public static class Factory extends AbstractLocalAsyncWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public NodeRefAggregationWorker workerInstance(ClusterWorkerContext clusterContext) { + return new NodeRefAggregationWorker(role(), clusterContext); + } + + @Override + public int queueSize() { + return 1024; + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return NodeRefAggregationWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new HashCodeSelector(); + } + + @Override public DataDefine dataDefine() { + return new NodeRefDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefPersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefPersistenceWorker.java new file mode 100644 index 000000000..06e363a1a --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefPersistenceWorker.java @@ -0,0 +1,70 @@ +package org.skywalking.apm.collector.agentstream.worker.noderef.reference; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.agentstream.worker.noderef.reference.dao.INodeReferenceDAO; +import org.skywalking.apm.collector.agentstream.worker.noderef.reference.define.NodeRefDataDefine; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class NodeRefPersistenceWorker extends PersistenceWorker { + + public NodeRefPersistenceWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + super.preStart(); + } + + @Override protected List prepareBatch(Map dataMap) { + INodeReferenceDAO dao = (INodeReferenceDAO)DAOContainer.INSTANCE.get(INodeReferenceDAO.class.getName()); + return dao.prepareBatch(dataMap); + } + + public static class Factory extends AbstractLocalAsyncWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public NodeRefPersistenceWorker workerInstance(ClusterWorkerContext clusterContext) { + return new NodeRefPersistenceWorker(role(), clusterContext); + } + + @Override + public int queueSize() { + return 1024; + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return NodeRefPersistenceWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new HashCodeSelector(); + } + + @Override public DataDefine dataDefine() { + return new NodeRefDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefRemoteWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefRemoteWorker.java new file mode 100644 index 000000000..292e652c6 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefRemoteWorker.java @@ -0,0 +1,60 @@ +package org.skywalking.apm.collector.agentstream.worker.noderef.reference; + +import org.skywalking.apm.collector.agentstream.worker.noderef.reference.define.NodeRefDataDefine; +import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorker; +import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerException; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class NodeRefRemoteWorker extends AbstractRemoteWorker { + + protected NodeRefRemoteWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + + } + + @Override protected void onWork(Object message) throws WorkerException { + getClusterContext().lookup(NodeRefPersistenceWorker.WorkerRole.INSTANCE).tell(message); + } + + public static class Factory extends AbstractRemoteWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public NodeRefRemoteWorker workerInstance(ClusterWorkerContext clusterContext) { + return new NodeRefRemoteWorker(role(), clusterContext); + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return NodeRefRemoteWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new HashCodeSelector(); + } + + @Override public DataDefine dataDefine() { + return new NodeRefDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefSpanListener.java new file mode 100644 index 000000000..0ab99bcdc --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefSpanListener.java @@ -0,0 +1,86 @@ +package org.skywalking.apm.collector.agentstream.worker.noderef.reference; + +import java.util.ArrayList; +import java.util.List; +import org.skywalking.apm.collector.agentstream.worker.Const; +import org.skywalking.apm.collector.agentstream.worker.noderef.reference.define.NodeRefDataDefine; +import org.skywalking.apm.collector.agentstream.worker.segment.EntrySpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.ExitSpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.RefsListener; +import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.stream.StreamModuleContext; +import org.skywalking.apm.collector.stream.StreamModuleGroupDefine; +import org.skywalking.apm.collector.stream.worker.WorkerInvokeException; +import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; +import org.skywalking.apm.network.proto.SpanObject; +import org.skywalking.apm.network.proto.TraceSegmentReference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class NodeRefSpanListener implements EntrySpanListener, ExitSpanListener, FirstSpanListener, RefsListener { + + private final Logger logger = LoggerFactory.getLogger(NodeRefSpanListener.class); + + private List nodeExitReferences = new ArrayList<>(); + private List nodeEntryReferences = new ArrayList<>(); + private long timeBucket; + private boolean hasReference = false; + + @Override + public void parseExit(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + String front = String.valueOf(applicationId); + String behind = String.valueOf(spanObject.getPeerId()); + if (spanObject.getPeerId() == 0) { + behind = spanObject.getPeer(); + } + + String agg = front + Const.ID_SPLIT + behind; + nodeExitReferences.add(agg); + } + + @Override + public void parseEntry(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + String behind = String.valueOf(applicationId); + String front = Const.USER_CODE; + + String agg = front + Const.ID_SPLIT + behind; + nodeEntryReferences.add(agg); + } + + @Override + public void parseFirst(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + timeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(spanObject.getStartTime()); + } + + @Override public void parseRef(TraceSegmentReference reference, int applicationId, int applicationInstanceId, + String segmentId) { + hasReference = true; + } + + @Override public void build() { + logger.debug("node reference listener build"); + StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); + if (!hasReference) { + nodeExitReferences.addAll(nodeEntryReferences); + } + + for (String agg : nodeExitReferences) { + NodeRefDataDefine.NodeReference nodeReference = new NodeRefDataDefine.NodeReference(); + nodeReference.setId(timeBucket + Const.ID_SPLIT + agg); + nodeReference.setAgg(agg); + nodeReference.setTimeBucket(timeBucket); + + try { + logger.debug("send to node reference aggregation worker, id: {}", nodeReference.getId()); + context.getClusterWorkerContext().lookup(NodeRefAggregationWorker.WorkerRole.INSTANCE).tell(nodeReference.transform()); + } catch (WorkerInvokeException | WorkerNotFoundException e) { + logger.error(e.getMessage(), e); + } + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/INodeReferenceDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/INodeReferenceDAO.java new file mode 100644 index 000000000..122829dfb --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/INodeReferenceDAO.java @@ -0,0 +1,12 @@ +package org.skywalking.apm.collector.agentstream.worker.noderef.reference.dao; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; + +/** + * @author pengys5 + */ +public interface INodeReferenceDAO { + List prepareBatch(Map dataMap); +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/NodeReferenceEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/NodeReferenceEsDAO.java new file mode 100644 index 000000000..d1334bec6 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/NodeReferenceEsDAO.java @@ -0,0 +1,29 @@ +package org.skywalking.apm.collector.agentstream.worker.noderef.reference.dao; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.skywalking.apm.collector.agentstream.worker.noderef.reference.define.NodeRefTable; +import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; + +/** + * @author pengys5 + */ +public class NodeReferenceEsDAO extends EsDAO implements INodeReferenceDAO { + + @Override public List prepareBatch(Map dataMap) { + List indexRequestBuilders = new ArrayList<>(); + dataMap.forEach((id, data) -> { + Map source = new HashMap(); + source.put(NodeRefTable.COLUMN_AGG, data.getDataString(1)); + source.put(NodeRefTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + + IndexRequestBuilder builder = getClient().prepareIndex(NodeRefTable.TABLE, id).setSource(source); + indexRequestBuilders.add(builder); + }); + return indexRequestBuilders; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/NodeReferenceH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/NodeReferenceH2DAO.java new file mode 100644 index 000000000..bc2d7a0a1 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/NodeReferenceH2DAO.java @@ -0,0 +1,15 @@ +package org.skywalking.apm.collector.agentstream.worker.noderef.reference.dao; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.storage.h2.dao.H2DAO; + +/** + * @author pengys5 + */ +public class NodeReferenceH2DAO extends H2DAO implements INodeReferenceDAO { + + @Override public List prepareBatch(Map map) { + return null; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefDataDefine.java new file mode 100644 index 000000000..91c39c567 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefDataDefine.java @@ -0,0 +1,96 @@ +package org.skywalking.apm.collector.agentstream.worker.noderef.reference.define; + +import org.skywalking.apm.collector.remote.grpc.proto.RemoteData; +import org.skywalking.apm.collector.stream.worker.impl.data.Attribute; +import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.impl.data.TransformToData; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation; + +/** + * @author pengys5 + */ +public class NodeRefDataDefine extends DataDefine { + + public static final int DEFINE_ID = 201; + + @Override public int defineId() { + return DEFINE_ID; + } + + @Override protected int initialCapacity() { + return 3; + } + + @Override protected void attributeDefine() { + addAttribute(0, new Attribute(NodeRefTable.COLUMN_ID, AttributeType.STRING, new NonOperation())); + addAttribute(1, new Attribute(NodeRefTable.COLUMN_AGG, AttributeType.STRING, new CoverOperation())); + addAttribute(2, new Attribute(NodeRefTable.COLUMN_TIME_BUCKET, AttributeType.LONG, new CoverOperation())); + } + + @Override public Object deserialize(RemoteData remoteData) { + String id = remoteData.getDataStrings(0); + String agg = remoteData.getDataStrings(1); + long timeBucket = remoteData.getDataLongs(0); + return new NodeReference(id, agg, timeBucket); + } + + @Override public RemoteData serialize(Object object) { + NodeReference nodeReference = (NodeReference)object; + RemoteData.Builder builder = RemoteData.newBuilder(); + builder.addDataStrings(nodeReference.getId()); + builder.addDataStrings(nodeReference.getAgg()); + builder.addDataLongs(nodeReference.getTimeBucket()); + return builder.build(); + } + + public static class NodeReference implements TransformToData { + private String id; + private String agg; + private long timeBucket; + + public NodeReference(String id, String agg, long timeBucket) { + this.id = id; + this.agg = agg; + this.timeBucket = timeBucket; + } + + public NodeReference() { + } + + @Override public Data transform() { + NodeRefDataDefine define = new NodeRefDataDefine(); + Data data = define.build(id); + data.setDataString(0, this.id); + data.setDataString(1, this.agg); + data.setDataLong(0, this.timeBucket); + return data; + } + + public String getId() { + return id; + } + + public String getAgg() { + return agg; + } + + public long getTimeBucket() { + return timeBucket; + } + + public void setId(String id) { + this.id = id; + } + + public void setAgg(String agg) { + this.agg = agg; + } + + public void setTimeBucket(long timeBucket) { + this.timeBucket = timeBucket; + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefEsTableDefine.java new file mode 100644 index 000000000..bc6b880e4 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefEsTableDefine.java @@ -0,0 +1,31 @@ +package org.skywalking.apm.collector.agentstream.worker.noderef.reference.define; + +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine; +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine; + +/** + * @author pengys5 + */ +public class NodeRefEsTableDefine extends ElasticSearchTableDefine { + + public NodeRefEsTableDefine() { + super(NodeRefTable.TABLE); + } + + @Override public int refreshInterval() { + return 2; + } + + @Override public int numberOfShards() { + return 2; + } + + @Override public int numberOfReplicas() { + return 0; + } + + @Override public void initialize() { + addColumn(new ElasticSearchColumnDefine(NodeRefTable.COLUMN_AGG, ElasticSearchColumnDefine.Type.Keyword.name())); + addColumn(new ElasticSearchColumnDefine(NodeRefTable.COLUMN_TIME_BUCKET, ElasticSearchColumnDefine.Type.Long.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefH2TableDefine.java new file mode 100644 index 000000000..c2815b1de --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefH2TableDefine.java @@ -0,0 +1,19 @@ +package org.skywalking.apm.collector.agentstream.worker.noderef.reference.define; + +import org.skywalking.apm.collector.storage.h2.define.H2ColumnDefine; +import org.skywalking.apm.collector.storage.h2.define.H2TableDefine; + +/** + * @author pengys5 + */ +public class NodeRefH2TableDefine extends H2TableDefine { + + public NodeRefH2TableDefine() { + super(NodeRefTable.TABLE); + } + + @Override public void initialize() { + addColumn(new H2ColumnDefine(NodeRefTable.COLUMN_AGG, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(NodeRefTable.COLUMN_TIME_BUCKET, H2ColumnDefine.Type.Bigint.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefTable.java new file mode 100644 index 000000000..17c058b09 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefTable.java @@ -0,0 +1,10 @@ +package org.skywalking.apm.collector.agentstream.worker.noderef.reference.define; + +import org.skywalking.apm.collector.agentstream.worker.CommonTable; + +/** + * @author pengys5 + */ +public class NodeRefTable extends CommonTable { + public static final String TABLE = "node_reference"; +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumAggregationWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumAggregationWorker.java new file mode 100644 index 000000000..4e6d4c8d8 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumAggregationWorker.java @@ -0,0 +1,66 @@ +package org.skywalking.apm.collector.agentstream.worker.noderef.summary; + +import org.skywalking.apm.collector.agentstream.worker.noderef.summary.define.NodeRefSumDataDefine; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; +import org.skywalking.apm.collector.stream.worker.WorkerRefs; +import org.skywalking.apm.collector.stream.worker.impl.AggregationWorker; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class NodeRefSumAggregationWorker extends AggregationWorker { + + public NodeRefSumAggregationWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + super.preStart(); + } + + @Override protected WorkerRefs nextWorkRef(String id) throws WorkerNotFoundException { + return getClusterContext().lookup(NodeRefSumRemoteWorker.WorkerRole.INSTANCE); + } + + public static class Factory extends AbstractLocalAsyncWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public NodeRefSumAggregationWorker workerInstance(ClusterWorkerContext clusterContext) { + return new NodeRefSumAggregationWorker(role(), clusterContext); + } + + @Override + public int queueSize() { + return 1024; + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return NodeRefSumAggregationWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new HashCodeSelector(); + } + + @Override public DataDefine dataDefine() { + return new NodeRefSumDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumPersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumPersistenceWorker.java new file mode 100644 index 000000000..96dead7e0 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumPersistenceWorker.java @@ -0,0 +1,70 @@ +package org.skywalking.apm.collector.agentstream.worker.noderef.summary; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.agentstream.worker.noderef.summary.dao.INodeRefSumDAO; +import org.skywalking.apm.collector.agentstream.worker.noderef.summary.define.NodeRefSumDataDefine; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class NodeRefSumPersistenceWorker extends PersistenceWorker { + + public NodeRefSumPersistenceWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + super.preStart(); + } + + @Override protected List prepareBatch(Map dataMap) { + INodeRefSumDAO dao = (INodeRefSumDAO)DAOContainer.INSTANCE.get(INodeRefSumDAO.class.getName()); + return dao.prepareBatch(dataMap); + } + + public static class Factory extends AbstractLocalAsyncWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public NodeRefSumPersistenceWorker workerInstance(ClusterWorkerContext clusterContext) { + return new NodeRefSumPersistenceWorker(role(), clusterContext); + } + + @Override + public int queueSize() { + return 1024; + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return NodeRefSumPersistenceWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new HashCodeSelector(); + } + + @Override public DataDefine dataDefine() { + return new NodeRefSumDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumRemoteWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumRemoteWorker.java new file mode 100644 index 000000000..5936faa68 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumRemoteWorker.java @@ -0,0 +1,60 @@ +package org.skywalking.apm.collector.agentstream.worker.noderef.summary; + +import org.skywalking.apm.collector.agentstream.worker.noderef.summary.define.NodeRefSumDataDefine; +import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorker; +import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerException; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class NodeRefSumRemoteWorker extends AbstractRemoteWorker { + + protected NodeRefSumRemoteWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + + } + + @Override protected void onWork(Object message) throws WorkerException { + getClusterContext().lookup(NodeRefSumPersistenceWorker.WorkerRole.INSTANCE).tell(message); + } + + public static class Factory extends AbstractRemoteWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public NodeRefSumRemoteWorker workerInstance(ClusterWorkerContext clusterContext) { + return new NodeRefSumRemoteWorker(role(), clusterContext); + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return NodeRefSumRemoteWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new HashCodeSelector(); + } + + @Override public DataDefine dataDefine() { + return new NodeRefSumDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumSpanListener.java new file mode 100644 index 000000000..d9ace3ae8 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumSpanListener.java @@ -0,0 +1,105 @@ +package org.skywalking.apm.collector.agentstream.worker.noderef.summary; + +import java.util.ArrayList; +import java.util.List; +import org.skywalking.apm.collector.agentstream.worker.Const; +import org.skywalking.apm.collector.agentstream.worker.noderef.summary.define.NodeRefSumDataDefine; +import org.skywalking.apm.collector.agentstream.worker.segment.EntrySpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.ExitSpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.RefsListener; +import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.stream.StreamModuleContext; +import org.skywalking.apm.collector.stream.StreamModuleGroupDefine; +import org.skywalking.apm.collector.stream.worker.WorkerInvokeException; +import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; +import org.skywalking.apm.network.proto.SpanObject; +import org.skywalking.apm.network.proto.TraceSegmentReference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class NodeRefSumSpanListener implements EntrySpanListener, ExitSpanListener, FirstSpanListener, RefsListener { + + private final Logger logger = LoggerFactory.getLogger(NodeRefSumSpanListener.class); + + private List nodeExitReferences = new ArrayList<>(); + private List nodeEntryReferences = new ArrayList<>(); + private long timeBucket; + private boolean hasReference = false; + + @Override + public void parseExit(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + String front = String.valueOf(applicationId); + String behind = String.valueOf(spanObject.getPeerId()); + if (spanObject.getPeerId() == 0) { + behind = spanObject.getPeer(); + } + + String agg = front + Const.ID_SPLIT + behind; + nodeExitReferences.add(buildNodeRefSum(spanObject.getStartTime(), spanObject.getEndTime(), agg, spanObject.getIsError())); + } + + @Override + public void parseEntry(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + String behind = String.valueOf(applicationId); + String front = Const.USER_CODE; + + String agg = front + Const.ID_SPLIT + behind; + nodeEntryReferences.add(buildNodeRefSum(spanObject.getStartTime(), spanObject.getEndTime(), agg, spanObject.getIsError())); + } + + private NodeRefSumDataDefine.NodeReferenceSum buildNodeRefSum(long startTime, long endTime, String agg, + boolean isError) { + NodeRefSumDataDefine.NodeReferenceSum referenceSum = new NodeRefSumDataDefine.NodeReferenceSum(); + referenceSum.setAgg(agg); + + long cost = endTime - startTime; + if (cost <= 1000 && !isError) { + referenceSum.setOneSecondLess(1L); + } else if (1000 < cost && cost <= 3000 && !isError) { + referenceSum.setThreeSecondLess(1L); + } else if (3000 < cost && cost <= 5000 && !isError) { + referenceSum.setFiveSecondLess(1L); + } else if (5000 < cost && !isError) { + referenceSum.setFiveSecondGreater(1L); + } else { + referenceSum.setError(1L); + } + referenceSum.setSummary(1L); + return referenceSum; + } + + @Override + public void parseFirst(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + timeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(spanObject.getStartTime()); + } + + @Override public void parseRef(TraceSegmentReference reference, int applicationId, int applicationInstanceId, + String segmentId) { + hasReference = true; + } + + @Override public void build() { + logger.debug("node reference summary listener build"); + StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); + if (!hasReference) { + nodeExitReferences.addAll(nodeEntryReferences); + } + + for (NodeRefSumDataDefine.NodeReferenceSum referenceSum : nodeExitReferences) { + referenceSum.setId(timeBucket + Const.ID_SPLIT + referenceSum.getAgg()); + referenceSum.setTimeBucket(timeBucket); + + try { + logger.debug("send to node reference summary aggregation worker, id: {}", referenceSum.getId()); + context.getClusterWorkerContext().lookup(NodeRefSumAggregationWorker.WorkerRole.INSTANCE).tell(referenceSum.transform()); + } catch (WorkerInvokeException | WorkerNotFoundException e) { + logger.error(e.getMessage(), e); + } + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/INodeRefSumDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/INodeRefSumDAO.java new file mode 100644 index 000000000..ef4f69737 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/INodeRefSumDAO.java @@ -0,0 +1,12 @@ +package org.skywalking.apm.collector.agentstream.worker.noderef.summary.dao; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; + +/** + * @author pengys5 + */ +public interface INodeRefSumDAO { + List prepareBatch(Map dataMap); +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumEsDAO.java new file mode 100644 index 000000000..a20099784 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumEsDAO.java @@ -0,0 +1,35 @@ +package org.skywalking.apm.collector.agentstream.worker.noderef.summary.dao; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.skywalking.apm.collector.agentstream.worker.noderef.summary.define.NodeRefSumTable; +import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; + +/** + * @author pengys5 + */ +public class NodeRefSumEsDAO extends EsDAO implements INodeRefSumDAO { + + @Override public List prepareBatch(Map dataMap) { + List indexRequestBuilders = new ArrayList<>(); + dataMap.forEach((id, data) -> { + Map source = new HashMap(); + source.put(NodeRefSumTable.COLUMN_ONE_SECOND_LESS, data.getDataLong(0)); + source.put(NodeRefSumTable.COLUMN_THREE_SECOND_LESS, data.getDataLong(1)); + source.put(NodeRefSumTable.COLUMN_FIVE_SECOND_LESS, data.getDataLong(2)); + source.put(NodeRefSumTable.COLUMN_FIVE_SECOND_GREATER, data.getDataLong(3)); + source.put(NodeRefSumTable.COLUMN_ERROR, data.getDataLong(4)); + source.put(NodeRefSumTable.COLUMN_SUMMARY, data.getDataLong(5)); + source.put(NodeRefSumTable.COLUMN_AGG, data.getDataString(1)); + source.put(NodeRefSumTable.COLUMN_TIME_BUCKET, data.getDataLong(6)); + + IndexRequestBuilder builder = getClient().prepareIndex(NodeRefSumTable.TABLE, id).setSource(source); + indexRequestBuilders.add(builder); + }); + return indexRequestBuilders; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumH2DAO.java new file mode 100644 index 000000000..2a3b83501 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumH2DAO.java @@ -0,0 +1,15 @@ +package org.skywalking.apm.collector.agentstream.worker.noderef.summary.dao; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.storage.h2.dao.H2DAO; + +/** + * @author pengys5 + */ +public class NodeRefSumH2DAO extends H2DAO implements INodeRefSumDAO { + + @Override public List prepareBatch(Map map) { + return null; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumDataDefine.java new file mode 100644 index 000000000..cecfae5b5 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumDataDefine.java @@ -0,0 +1,181 @@ +package org.skywalking.apm.collector.agentstream.worker.noderef.summary.define; + +import org.skywalking.apm.collector.remote.grpc.proto.RemoteData; +import org.skywalking.apm.collector.stream.worker.impl.data.Attribute; +import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.impl.data.TransformToData; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.AddOperation; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation; + +/** + * @author pengys5 + */ +public class NodeRefSumDataDefine extends DataDefine { + + public static final int DEFINE_ID = 202; + + @Override public int defineId() { + return DEFINE_ID; + } + + @Override protected int initialCapacity() { + return 9; + } + + @Override protected void attributeDefine() { + addAttribute(0, new Attribute(NodeRefSumTable.COLUMN_ID, AttributeType.STRING, new NonOperation())); + addAttribute(1, new Attribute(NodeRefSumTable.COLUMN_ONE_SECOND_LESS, AttributeType.LONG, new AddOperation())); + addAttribute(2, new Attribute(NodeRefSumTable.COLUMN_THREE_SECOND_LESS, AttributeType.LONG, new AddOperation())); + addAttribute(3, new Attribute(NodeRefSumTable.COLUMN_FIVE_SECOND_LESS, AttributeType.LONG, new AddOperation())); + addAttribute(4, new Attribute(NodeRefSumTable.COLUMN_FIVE_SECOND_GREATER, AttributeType.LONG, new AddOperation())); + addAttribute(5, new Attribute(NodeRefSumTable.COLUMN_ERROR, AttributeType.LONG, new AddOperation())); + addAttribute(6, new Attribute(NodeRefSumTable.COLUMN_SUMMARY, AttributeType.LONG, new AddOperation())); + addAttribute(7, new Attribute(NodeRefSumTable.COLUMN_AGG, AttributeType.STRING, new NonOperation())); + addAttribute(8, new Attribute(NodeRefSumTable.COLUMN_TIME_BUCKET, AttributeType.LONG, new NonOperation())); + } + + @Override public Object deserialize(RemoteData remoteData) { + String id = remoteData.getDataStrings(0); + String agg = remoteData.getDataStrings(1); + Long oneSecondLess = remoteData.getDataLongs(0); + Long threeSecondLess = remoteData.getDataLongs(1); + Long fiveSecondLess = remoteData.getDataLongs(2); + Long fiveSecondGreater = remoteData.getDataLongs(3); + Long error = remoteData.getDataLongs(4); + Long summary = remoteData.getDataLongs(5); + long timeBucket = remoteData.getDataLongs(6); + return new NodeReferenceSum(id, oneSecondLess, threeSecondLess, fiveSecondLess, fiveSecondGreater, error, summary, agg, timeBucket); + } + + @Override public RemoteData serialize(Object object) { + NodeReferenceSum nodeReferenceSum = (NodeReferenceSum)object; + RemoteData.Builder builder = RemoteData.newBuilder(); + builder.addDataStrings(nodeReferenceSum.getId()); + builder.addDataStrings(nodeReferenceSum.getAgg()); + builder.addDataLongs(nodeReferenceSum.getOneSecondLess()); + builder.addDataLongs(nodeReferenceSum.getThreeSecondLess()); + builder.addDataLongs(nodeReferenceSum.getFiveSecondLess()); + builder.addDataLongs(nodeReferenceSum.getFiveSecondGreater()); + builder.addDataLongs(nodeReferenceSum.getError()); + builder.addDataLongs(nodeReferenceSum.getSummary()); + builder.addDataLongs(nodeReferenceSum.getTimeBucket()); + return builder.build(); + } + + public static class NodeReferenceSum implements TransformToData { + private String id; + private Long oneSecondLess = 0L; + private Long threeSecondLess = 0L; + private Long fiveSecondLess = 0L; + private Long fiveSecondGreater = 0L; + private Long error = 0L; + private Long summary = 0L; + private String agg; + private long timeBucket; + + public NodeReferenceSum(String id, Long oneSecondLess, Long threeSecondLess, Long fiveSecondLess, + Long fiveSecondGreater, Long error, Long summary, String agg, long timeBucket) { + this.id = id; + this.oneSecondLess = oneSecondLess; + this.threeSecondLess = threeSecondLess; + this.fiveSecondLess = fiveSecondLess; + this.fiveSecondGreater = fiveSecondGreater; + this.error = error; + this.summary = summary; + this.agg = agg; + this.timeBucket = timeBucket; + } + + public NodeReferenceSum() { + } + + @Override public Data transform() { + NodeRefSumDataDefine define = new NodeRefSumDataDefine(); + Data data = define.build(id); + data.setDataString(0, this.id); + data.setDataString(1, this.agg); + data.setDataLong(0, this.oneSecondLess); + data.setDataLong(1, this.threeSecondLess); + data.setDataLong(2, this.fiveSecondLess); + data.setDataLong(3, this.fiveSecondGreater); + data.setDataLong(4, this.error); + data.setDataLong(5, this.summary); + data.setDataLong(6, this.timeBucket); + return data; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public Long getOneSecondLess() { + return oneSecondLess; + } + + public void setOneSecondLess(Long oneSecondLess) { + this.oneSecondLess = oneSecondLess; + } + + public Long getThreeSecondLess() { + return threeSecondLess; + } + + public void setThreeSecondLess(Long threeSecondLess) { + this.threeSecondLess = threeSecondLess; + } + + public Long getFiveSecondLess() { + return fiveSecondLess; + } + + public void setFiveSecondLess(Long fiveSecondLess) { + this.fiveSecondLess = fiveSecondLess; + } + + public Long getFiveSecondGreater() { + return fiveSecondGreater; + } + + public void setFiveSecondGreater(Long fiveSecondGreater) { + this.fiveSecondGreater = fiveSecondGreater; + } + + public Long getError() { + return error; + } + + public void setError(Long error) { + this.error = error; + } + + public Long getSummary() { + return summary; + } + + public void setSummary(Long summary) { + this.summary = summary; + } + + public String getAgg() { + return agg; + } + + public void setAgg(String agg) { + this.agg = agg; + } + + public long getTimeBucket() { + return timeBucket; + } + + public void setTimeBucket(long timeBucket) { + this.timeBucket = timeBucket; + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumEsTableDefine.java new file mode 100644 index 000000000..6fd0b818b --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumEsTableDefine.java @@ -0,0 +1,37 @@ +package org.skywalking.apm.collector.agentstream.worker.noderef.summary.define; + +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine; +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine; + +/** + * @author pengys5 + */ +public class NodeRefSumEsTableDefine extends ElasticSearchTableDefine { + + public NodeRefSumEsTableDefine() { + super(NodeRefSumTable.TABLE); + } + + @Override public int refreshInterval() { + return 2; + } + + @Override public int numberOfShards() { + return 2; + } + + @Override public int numberOfReplicas() { + return 0; + } + + @Override public void initialize() { + addColumn(new ElasticSearchColumnDefine(NodeRefSumTable.COLUMN_ONE_SECOND_LESS, ElasticSearchColumnDefine.Type.Long.name())); + addColumn(new ElasticSearchColumnDefine(NodeRefSumTable.COLUMN_THREE_SECOND_LESS, ElasticSearchColumnDefine.Type.Long.name())); + addColumn(new ElasticSearchColumnDefine(NodeRefSumTable.COLUMN_FIVE_SECOND_LESS, ElasticSearchColumnDefine.Type.Long.name())); + addColumn(new ElasticSearchColumnDefine(NodeRefSumTable.COLUMN_FIVE_SECOND_GREATER, ElasticSearchColumnDefine.Type.Long.name())); + addColumn(new ElasticSearchColumnDefine(NodeRefSumTable.COLUMN_ERROR, ElasticSearchColumnDefine.Type.Long.name())); + addColumn(new ElasticSearchColumnDefine(NodeRefSumTable.COLUMN_SUMMARY, ElasticSearchColumnDefine.Type.Long.name())); + addColumn(new ElasticSearchColumnDefine(NodeRefSumTable.COLUMN_AGG, ElasticSearchColumnDefine.Type.Keyword.name())); + addColumn(new ElasticSearchColumnDefine(NodeRefSumTable.COLUMN_TIME_BUCKET, ElasticSearchColumnDefine.Type.Long.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumH2TableDefine.java new file mode 100644 index 000000000..a31257102 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumH2TableDefine.java @@ -0,0 +1,25 @@ +package org.skywalking.apm.collector.agentstream.worker.noderef.summary.define; + +import org.skywalking.apm.collector.storage.h2.define.H2ColumnDefine; +import org.skywalking.apm.collector.storage.h2.define.H2TableDefine; + +/** + * @author pengys5 + */ +public class NodeRefSumH2TableDefine extends H2TableDefine { + + public NodeRefSumH2TableDefine() { + super(NodeRefSumTable.TABLE); + } + + @Override public void initialize() { + addColumn(new H2ColumnDefine(NodeRefSumTable.COLUMN_ONE_SECOND_LESS, H2ColumnDefine.Type.Bigint.name())); + addColumn(new H2ColumnDefine(NodeRefSumTable.COLUMN_THREE_SECOND_LESS, H2ColumnDefine.Type.Bigint.name())); + addColumn(new H2ColumnDefine(NodeRefSumTable.COLUMN_FIVE_SECOND_LESS, H2ColumnDefine.Type.Bigint.name())); + addColumn(new H2ColumnDefine(NodeRefSumTable.COLUMN_FIVE_SECOND_GREATER, H2ColumnDefine.Type.Bigint.name())); + addColumn(new H2ColumnDefine(NodeRefSumTable.COLUMN_ERROR, H2ColumnDefine.Type.Bigint.name())); + addColumn(new H2ColumnDefine(NodeRefSumTable.COLUMN_SUMMARY, H2ColumnDefine.Type.Bigint.name())); + addColumn(new H2ColumnDefine(NodeRefSumTable.COLUMN_AGG, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(NodeRefSumTable.COLUMN_TIME_BUCKET, H2ColumnDefine.Type.Bigint.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumTable.java new file mode 100644 index 000000000..0c07ae9a8 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumTable.java @@ -0,0 +1,16 @@ +package org.skywalking.apm.collector.agentstream.worker.noderef.summary.define; + +import org.skywalking.apm.collector.agentstream.worker.CommonTable; + +/** + * @author pengys5 + */ +public class NodeRefSumTable extends CommonTable { + public static final String TABLE = "node_reference_sum"; + public static final String COLUMN_ONE_SECOND_LESS = "one_second_less"; + public static final String COLUMN_THREE_SECOND_LESS = "three_second_less"; + public static final String COLUMN_FIVE_SECOND_LESS = "five_second_less"; + public static final String COLUMN_FIVE_SECOND_GREATER = "five_second_greater"; + public static final String COLUMN_ERROR = "error"; + public static final String COLUMN_SUMMARY = "summary"; +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/IdAutoIncrement.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/IdAutoIncrement.java new file mode 100644 index 000000000..d9864e4e7 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/IdAutoIncrement.java @@ -0,0 +1,24 @@ +package org.skywalking.apm.collector.agentstream.worker.register; + +/** + * @author pengys5 + */ +public enum IdAutoIncrement { + INSTANCE; + + public int increment(int min, int max) { + int instanceId; + if (min == max) { + instanceId = -1; + } else if (min + max == 0) { + instanceId = max + 1; + } else if (min + max > 0) { + instanceId = min - 1; + } else if (max < 0) { + instanceId = 1; + } else { + instanceId = max + 1; + } + return instanceId; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationDataDefine.java new file mode 100644 index 000000000..cb964a3a4 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationDataDefine.java @@ -0,0 +1,82 @@ +package org.skywalking.apm.collector.agentstream.worker.register.application; + +import org.skywalking.apm.collector.remote.grpc.proto.RemoteData; +import org.skywalking.apm.collector.stream.worker.impl.data.Attribute; +import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation; + +/** + * @author pengys5 + */ +public class ApplicationDataDefine extends DataDefine { + + public static final int DEFINE_ID = 101; + + @Override public int defineId() { + return DEFINE_ID; + } + + @Override protected int initialCapacity() { + return 3; + } + + @Override protected void attributeDefine() { + addAttribute(0, new Attribute("id", AttributeType.STRING, new NonOperation())); + addAttribute(1, new Attribute(ApplicationTable.COLUMN_APPLICATION_CODE, AttributeType.STRING, new CoverOperation())); + addAttribute(2, new Attribute(ApplicationTable.COLUMN_APPLICATION_ID, AttributeType.INTEGER, new CoverOperation())); + } + + @Override public Object deserialize(RemoteData remoteData) { + String id = remoteData.getDataStrings(0); + String applicationCode = remoteData.getDataStrings(1); + int applicationId = remoteData.getDataIntegers(0); + return new Application(id, applicationCode, applicationId); + } + + @Override public RemoteData serialize(Object object) { + Application application = (Application)object; + RemoteData.Builder builder = RemoteData.newBuilder(); + builder.addDataStrings(application.getId()); + builder.addDataStrings(application.getApplicationCode()); + builder.addDataIntegers(application.getApplicationId()); + return builder.build(); + } + + public static class Application { + private String id; + private String applicationCode; + private int applicationId; + + public Application(String id, String applicationCode, int applicationId) { + this.id = id; + this.applicationCode = applicationCode; + this.applicationId = applicationId; + } + + public String getId() { + return id; + } + + public String getApplicationCode() { + return applicationCode; + } + + public int getApplicationId() { + return applicationId; + } + + public void setId(String id) { + this.id = id; + } + + public void setApplicationCode(String applicationCode) { + this.applicationCode = applicationCode; + } + + public void setApplicationId(int applicationId) { + this.applicationId = applicationId; + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationEsTableDefine.java new file mode 100644 index 000000000..e5e5faded --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationEsTableDefine.java @@ -0,0 +1,31 @@ +package org.skywalking.apm.collector.agentstream.worker.register.application; + +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine; +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine; + +/** + * @author pengys5 + */ +public class ApplicationEsTableDefine extends ElasticSearchTableDefine { + + public ApplicationEsTableDefine() { + super(ApplicationTable.TABLE); + } + + @Override public int refreshInterval() { + return 0; + } + + @Override public int numberOfShards() { + return 2; + } + + @Override public int numberOfReplicas() { + return 0; + } + + @Override public void initialize() { + addColumn(new ElasticSearchColumnDefine(ApplicationTable.COLUMN_APPLICATION_CODE, ElasticSearchColumnDefine.Type.Keyword.name())); + addColumn(new ElasticSearchColumnDefine(ApplicationTable.COLUMN_APPLICATION_ID, ElasticSearchColumnDefine.Type.Integer.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationH2TableDefine.java new file mode 100644 index 000000000..ef77965c2 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationH2TableDefine.java @@ -0,0 +1,19 @@ +package org.skywalking.apm.collector.agentstream.worker.register.application; + +import org.skywalking.apm.collector.storage.h2.define.H2ColumnDefine; +import org.skywalking.apm.collector.storage.h2.define.H2TableDefine; + +/** + * @author pengys5 + */ +public class ApplicationH2TableDefine extends H2TableDefine { + + public ApplicationH2TableDefine() { + super(ApplicationTable.TABLE); + } + + @Override public void initialize() { + addColumn(new H2ColumnDefine(ApplicationTable.COLUMN_APPLICATION_CODE, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(ApplicationTable.COLUMN_APPLICATION_ID, H2ColumnDefine.Type.Int.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterRemoteWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterRemoteWorker.java new file mode 100644 index 000000000..fad05955a --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterRemoteWorker.java @@ -0,0 +1,64 @@ +package org.skywalking.apm.collector.agentstream.worker.register.application; + +import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorker; +import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerException; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.ForeverFirstSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class ApplicationRegisterRemoteWorker extends AbstractRemoteWorker { + + private final Logger logger = LoggerFactory.getLogger(ApplicationRegisterRemoteWorker.class); + + protected ApplicationRegisterRemoteWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + } + + @Override protected void onWork(Object message) throws WorkerException { + ApplicationDataDefine.Application application = (ApplicationDataDefine.Application)message; + logger.debug("application code: {}", application.getApplicationCode()); + getClusterContext().lookup(ApplicationRegisterSerialWorker.WorkerRole.INSTANCE).tell(application); + } + + public static class Factory extends AbstractRemoteWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public ApplicationRegisterRemoteWorker workerInstance(ClusterWorkerContext clusterContext) { + return new ApplicationRegisterRemoteWorker(role(), clusterContext); + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return ApplicationRegisterRemoteWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new ForeverFirstSelector(); + } + + @Override public DataDefine dataDefine() { + return new ApplicationDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java new file mode 100644 index 000000000..66c6c0e2a --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java @@ -0,0 +1,86 @@ +package org.skywalking.apm.collector.agentstream.worker.register.application; + +import org.skywalking.apm.collector.agentstream.worker.register.IdAutoIncrement; +import org.skywalking.apm.collector.agentstream.worker.register.application.dao.IApplicationDAO; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorker; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerException; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.ForeverFirstSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class ApplicationRegisterSerialWorker extends AbstractLocalAsyncWorker { + + private final Logger logger = LoggerFactory.getLogger(ApplicationRegisterSerialWorker.class); + + public ApplicationRegisterSerialWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + super.preStart(); + } + + @Override protected void onWork(Object message) throws WorkerException { + if (message instanceof ApplicationDataDefine.Application) { + ApplicationDataDefine.Application application = (ApplicationDataDefine.Application)message; + logger.debug("register application, application code: {}", application.getApplicationCode()); + + IApplicationDAO dao = (IApplicationDAO)DAOContainer.INSTANCE.get(IApplicationDAO.class.getName()); + int min = dao.getMinApplicationId(); + if (min == 0) { + application.setApplicationId(1); + application.setId("1"); + } else { + int max = dao.getMaxApplicationId(); + int applicationId = IdAutoIncrement.INSTANCE.increment(min, max); + application.setApplicationId(applicationId); + application.setId(String.valueOf(applicationId)); + } + dao.save(application); + } + } + + public static class Factory extends AbstractLocalAsyncWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public ApplicationRegisterSerialWorker workerInstance(ClusterWorkerContext clusterContext) { + return new ApplicationRegisterSerialWorker(role(), clusterContext); + } + + @Override public int queueSize() { + return 256; + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return ApplicationRegisterSerialWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new ForeverFirstSelector(); + } + + @Override public DataDefine dataDefine() { + return new ApplicationDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationTable.java new file mode 100644 index 000000000..797b8d80a --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationTable.java @@ -0,0 +1,10 @@ +package org.skywalking.apm.collector.agentstream.worker.register.application; + +/** + * @author pengys5 + */ +public class ApplicationTable { + public static final String TABLE = "application"; + public static final String COLUMN_APPLICATION_CODE = "application_code"; + public static final String COLUMN_APPLICATION_ID = "application_id"; +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/ApplicationEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/ApplicationEsDAO.java new file mode 100644 index 000000000..ebe4360a8 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/ApplicationEsDAO.java @@ -0,0 +1,62 @@ +package org.skywalking.apm.collector.agentstream.worker.register.application.dao; + +import java.util.HashMap; +import java.util.Map; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine; +import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationTable; +import org.skywalking.apm.collector.client.elasticsearch.ElasticSearchClient; +import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class ApplicationEsDAO extends EsDAO implements IApplicationDAO { + + private final Logger logger = LoggerFactory.getLogger(ApplicationEsDAO.class); + + @Override public int getApplicationId(String applicationCode) { + ElasticSearchClient client = getClient(); + + SearchRequestBuilder searchRequestBuilder = client.prepareSearch(ApplicationTable.TABLE); + searchRequestBuilder.setTypes("type"); + searchRequestBuilder.setSearchType(SearchType.QUERY_THEN_FETCH); + searchRequestBuilder.setQuery(QueryBuilders.termQuery(ApplicationTable.COLUMN_APPLICATION_CODE, applicationCode)); + searchRequestBuilder.setSize(1); + + SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); + if (searchResponse.getHits().totalHits > 0) { + SearchHit searchHit = searchResponse.getHits().iterator().next(); + int applicationId = (int)searchHit.getSource().get(ApplicationTable.COLUMN_APPLICATION_ID); + return applicationId; + } + return 0; + } + + @Override public int getMaxApplicationId() { + return getMaxId(ApplicationTable.TABLE, ApplicationTable.COLUMN_APPLICATION_ID); + } + + @Override public int getMinApplicationId() { + return getMinId(ApplicationTable.TABLE, ApplicationTable.COLUMN_APPLICATION_ID); + } + + @Override public void save(ApplicationDataDefine.Application application) { + logger.debug("save application register info, application id: {}, application code: {}", application.getApplicationId(), application.getApplicationCode()); + ElasticSearchClient client = getClient(); + Map source = new HashMap(); + source.put(ApplicationTable.COLUMN_APPLICATION_CODE, application.getApplicationCode()); + source.put(ApplicationTable.COLUMN_APPLICATION_ID, application.getApplicationId()); + + IndexResponse response = client.prepareIndex(ApplicationTable.TABLE, application.getId()).setSource(source).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + logger.debug("save application register info, application id: {}, application code: {}, status: {}", application.getApplicationId(), application.getApplicationCode(), response.status().name()); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/ApplicationH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/ApplicationH2DAO.java new file mode 100644 index 000000000..3c6a03e9e --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/ApplicationH2DAO.java @@ -0,0 +1,28 @@ +package org.skywalking.apm.collector.agentstream.worker.register.application.dao; + +import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine; +import org.skywalking.apm.collector.client.h2.H2Client; +import org.skywalking.apm.collector.storage.h2.dao.H2DAO; + +/** + * @author pengys5 + */ +public class ApplicationH2DAO extends H2DAO implements IApplicationDAO { + + @Override public int getApplicationId(String applicationCode) { + H2Client client = getClient(); + return 100; + } + + @Override public int getMaxApplicationId() { + return 0; + } + + @Override public int getMinApplicationId() { + return 0; + } + + @Override public void save(ApplicationDataDefine.Application application) { + + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/IApplicationDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/IApplicationDAO.java new file mode 100644 index 000000000..ad1083a00 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/dao/IApplicationDAO.java @@ -0,0 +1,16 @@ +package org.skywalking.apm.collector.agentstream.worker.register.application.dao; + +import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine; + +/** + * @author pengys5 + */ +public interface IApplicationDAO { + int getApplicationId(String applicationCode); + + int getMaxApplicationId(); + + int getMinApplicationId(); + + void save(ApplicationDataDefine.Application application); +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java new file mode 100644 index 000000000..e14afa7b2 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java @@ -0,0 +1,108 @@ +package org.skywalking.apm.collector.agentstream.worker.register.instance; + +import org.skywalking.apm.collector.remote.grpc.proto.RemoteData; +import org.skywalking.apm.collector.stream.worker.impl.data.Attribute; +import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation; + +/** + * @author pengys5 + */ +public class InstanceDataDefine extends DataDefine { + + public static final int DEFINE_ID = 102; + + @Override public int defineId() { + return DEFINE_ID; + } + + @Override protected int initialCapacity() { + return 6; + } + + @Override protected void attributeDefine() { + addAttribute(0, new Attribute("id", AttributeType.STRING, new NonOperation())); + addAttribute(1, new Attribute(InstanceTable.COLUMN_APPLICATION_ID, AttributeType.INTEGER, new CoverOperation())); + addAttribute(2, new Attribute(InstanceTable.COLUMN_AGENTUUID, AttributeType.STRING, new CoverOperation())); + addAttribute(3, new Attribute(InstanceTable.COLUMN_REGISTER_TIME, AttributeType.LONG, new CoverOperation())); + addAttribute(4, new Attribute(InstanceTable.COLUMN_INSTANCE_ID, AttributeType.INTEGER, new CoverOperation())); + addAttribute(5, new Attribute(InstanceTable.COLUMN_HEARTBEAT_TIME, AttributeType.LONG, new CoverOperation())); + } + + @Override public Object deserialize(RemoteData remoteData) { + String id = remoteData.getDataStrings(0); + int applicationId = remoteData.getDataIntegers(0); + String agentUUID = remoteData.getDataStrings(1); + int instanceId = remoteData.getDataIntegers(1); + long registerTime = remoteData.getDataLongs(0); + return new Instance(id, applicationId, agentUUID, registerTime, instanceId); + } + + @Override public RemoteData serialize(Object object) { + Instance instance = (Instance)object; + RemoteData.Builder builder = RemoteData.newBuilder(); + builder.addDataStrings(instance.getId()); + builder.addDataIntegers(instance.getApplicationId()); + builder.addDataStrings(instance.getAgentUUID()); + builder.addDataLongs(instance.getRegisterTime()); + return builder.build(); + } + + public static class Instance { + private String id; + private int applicationId; + private String agentUUID; + private long registerTime; + private int instanceId; + + public Instance(String id, int applicationId, String agentUUID, long registerTime, int instanceId) { + this.id = id; + this.applicationId = applicationId; + this.agentUUID = agentUUID; + this.registerTime = registerTime; + this.instanceId = instanceId; + } + + public String getId() { + return id; + } + + public int getApplicationId() { + return applicationId; + } + + public String getAgentUUID() { + return agentUUID; + } + + public long getRegisterTime() { + return registerTime; + } + + public int getInstanceId() { + return instanceId; + } + + public void setId(String id) { + this.id = id; + } + + public void setApplicationId(int applicationId) { + this.applicationId = applicationId; + } + + public void setAgentUUID(String agentUUID) { + this.agentUUID = agentUUID; + } + + public void setRegisterTime(long registerTime) { + this.registerTime = registerTime; + } + + public void setInstanceId(int instanceId) { + this.instanceId = instanceId; + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceEsTableDefine.java new file mode 100644 index 000000000..dffcfa9fb --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceEsTableDefine.java @@ -0,0 +1,34 @@ +package org.skywalking.apm.collector.agentstream.worker.register.instance; + +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine; +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine; + +/** + * @author pengys5 + */ +public class InstanceEsTableDefine extends ElasticSearchTableDefine { + + public InstanceEsTableDefine() { + super(InstanceTable.TABLE); + } + + @Override public int refreshInterval() { + return 0; + } + + @Override public int numberOfShards() { + return 2; + } + + @Override public int numberOfReplicas() { + return 0; + } + + @Override public void initialize() { + addColumn(new ElasticSearchColumnDefine(InstanceTable.COLUMN_APPLICATION_ID, ElasticSearchColumnDefine.Type.Integer.name())); + addColumn(new ElasticSearchColumnDefine(InstanceTable.COLUMN_AGENTUUID, ElasticSearchColumnDefine.Type.Keyword.name())); + addColumn(new ElasticSearchColumnDefine(InstanceTable.COLUMN_REGISTER_TIME, ElasticSearchColumnDefine.Type.Long.name())); + addColumn(new ElasticSearchColumnDefine(InstanceTable.COLUMN_INSTANCE_ID, ElasticSearchColumnDefine.Type.Integer.name())); + addColumn(new ElasticSearchColumnDefine(InstanceTable.COLUMN_HEARTBEAT_TIME, ElasticSearchColumnDefine.Type.Long.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceH2TableDefine.java new file mode 100644 index 000000000..58ca3e626 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceH2TableDefine.java @@ -0,0 +1,22 @@ +package org.skywalking.apm.collector.agentstream.worker.register.instance; + +import org.skywalking.apm.collector.storage.h2.define.H2ColumnDefine; +import org.skywalking.apm.collector.storage.h2.define.H2TableDefine; + +/** + * @author pengys5 + */ +public class InstanceH2TableDefine extends H2TableDefine { + + public InstanceH2TableDefine() { + super(InstanceTable.TABLE); + } + + @Override public void initialize() { + addColumn(new H2ColumnDefine(InstanceTable.COLUMN_APPLICATION_ID, H2ColumnDefine.Type.Int.name())); + addColumn(new H2ColumnDefine(InstanceTable.COLUMN_AGENTUUID, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(InstanceTable.COLUMN_REGISTER_TIME, H2ColumnDefine.Type.Bigint.name())); + addColumn(new H2ColumnDefine(InstanceTable.COLUMN_INSTANCE_ID, H2ColumnDefine.Type.Int.name())); + addColumn(new H2ColumnDefine(InstanceTable.COLUMN_HEARTBEAT_TIME, H2ColumnDefine.Type.Bigint.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceRegisterRemoteWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceRegisterRemoteWorker.java new file mode 100644 index 000000000..c6a85dd3e --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceRegisterRemoteWorker.java @@ -0,0 +1,64 @@ +package org.skywalking.apm.collector.agentstream.worker.register.instance; + +import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorker; +import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerException; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.ForeverFirstSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class InstanceRegisterRemoteWorker extends AbstractRemoteWorker { + + private final Logger logger = LoggerFactory.getLogger(InstanceRegisterRemoteWorker.class); + + protected InstanceRegisterRemoteWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + } + + @Override protected void onWork(Object message) throws WorkerException { + InstanceDataDefine.Instance instance = (InstanceDataDefine.Instance)message; + logger.debug("application id: {}, agentUUID: {}, register time: {}", instance.getApplicationId(), instance.getAgentUUID(), instance.getRegisterTime()); + getClusterContext().lookup(InstanceRegisterSerialWorker.WorkerRole.INSTANCE).tell(instance); + } + + public static class Factory extends AbstractRemoteWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public InstanceRegisterRemoteWorker workerInstance(ClusterWorkerContext clusterContext) { + return new InstanceRegisterRemoteWorker(role(), clusterContext); + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return InstanceRegisterRemoteWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new ForeverFirstSelector(); + } + + @Override public DataDefine dataDefine() { + return new InstanceDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceRegisterSerialWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceRegisterSerialWorker.java new file mode 100644 index 000000000..9bfabe44e --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceRegisterSerialWorker.java @@ -0,0 +1,87 @@ +package org.skywalking.apm.collector.agentstream.worker.register.instance; + +import org.skywalking.apm.collector.agentstream.worker.register.IdAutoIncrement; +import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine; +import org.skywalking.apm.collector.agentstream.worker.register.instance.dao.IInstanceDAO; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorker; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerException; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.ForeverFirstSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class InstanceRegisterSerialWorker extends AbstractLocalAsyncWorker { + + private final Logger logger = LoggerFactory.getLogger(InstanceRegisterSerialWorker.class); + + public InstanceRegisterSerialWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + super.preStart(); + } + + @Override protected void onWork(Object message) throws WorkerException { + if (message instanceof InstanceDataDefine.Instance) { + InstanceDataDefine.Instance instance = (InstanceDataDefine.Instance)message; + logger.debug("register instance, application id: {}, agentUUID: {}", instance.getApplicationId(), instance.getAgentUUID()); + + IInstanceDAO dao = (IInstanceDAO)DAOContainer.INSTANCE.get(IInstanceDAO.class.getName()); + int min = dao.getMinInstanceId(); + if (min == 0) { + instance.setId("1"); + instance.setInstanceId(1); + } else { + int max = dao.getMaxInstanceId(); + int instanceId = IdAutoIncrement.INSTANCE.increment(min, max); + instance.setId(String.valueOf(instanceId)); + instance.setInstanceId(instanceId); + } + dao.save(instance); + } + } + + public static class Factory extends AbstractLocalAsyncWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public InstanceRegisterSerialWorker workerInstance(ClusterWorkerContext clusterContext) { + return new InstanceRegisterSerialWorker(role(), clusterContext); + } + + @Override public int queueSize() { + return 256; + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return InstanceRegisterSerialWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new ForeverFirstSelector(); + } + + @Override public DataDefine dataDefine() { + return new ApplicationDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceTable.java new file mode 100644 index 000000000..5a26adb6e --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceTable.java @@ -0,0 +1,13 @@ +package org.skywalking.apm.collector.agentstream.worker.register.instance; + +/** + * @author pengys5 + */ +public class InstanceTable { + public static final String TABLE = "instance"; + public static final String COLUMN_APPLICATION_ID = "application_id"; + public static final String COLUMN_AGENTUUID = "agent_uuid"; + public static final String COLUMN_REGISTER_TIME = "register_time"; + public static final String COLUMN_INSTANCE_ID = "instance_id"; + public static final String COLUMN_HEARTBEAT_TIME = "heartbeatTime"; +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/IInstanceDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/IInstanceDAO.java new file mode 100644 index 000000000..f0c311146 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/IInstanceDAO.java @@ -0,0 +1,18 @@ +package org.skywalking.apm.collector.agentstream.worker.register.instance.dao; + +import org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceDataDefine; + +/** + * @author pengys5 + */ +public interface IInstanceDAO { + int getInstanceId(int applicationId, String agentUUID); + + int getMaxInstanceId(); + + int getMinInstanceId(); + + void save(InstanceDataDefine.Instance instance); + + void updateHeartbeatTime(int instanceId, long heartbeatTime); +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceEsDAO.java new file mode 100644 index 000000000..237f50009 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceEsDAO.java @@ -0,0 +1,84 @@ +package org.skywalking.apm.collector.agentstream.worker.register.instance.dao; + +import java.util.HashMap; +import java.util.Map; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceDataDefine; +import org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceTable; +import org.skywalking.apm.collector.client.elasticsearch.ElasticSearchClient; +import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class InstanceEsDAO extends EsDAO implements IInstanceDAO { + + private final Logger logger = LoggerFactory.getLogger(InstanceEsDAO.class); + + @Override public int getInstanceId(int applicationId, String agentUUID) { + ElasticSearchClient client = getClient(); + + SearchRequestBuilder searchRequestBuilder = client.prepareSearch(InstanceTable.TABLE); + searchRequestBuilder.setTypes("type"); + searchRequestBuilder.setSearchType(SearchType.QUERY_THEN_FETCH); + BoolQueryBuilder builder = QueryBuilders.boolQuery(); + builder.must().add(QueryBuilders.termQuery(InstanceTable.COLUMN_APPLICATION_ID, applicationId)); + builder.must().add(QueryBuilders.termQuery(InstanceTable.COLUMN_AGENTUUID, agentUUID)); + searchRequestBuilder.setQuery(builder); + searchRequestBuilder.setSize(1); + + SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); + if (searchResponse.getHits().totalHits > 0) { + SearchHit searchHit = searchResponse.getHits().iterator().next(); + int instanceId = (int)searchHit.getSource().get(InstanceTable.COLUMN_INSTANCE_ID); + return instanceId; + } + return 0; + } + + @Override public int getMaxInstanceId() { + return getMaxId(InstanceTable.TABLE, InstanceTable.COLUMN_INSTANCE_ID); + } + + @Override public int getMinInstanceId() { + return getMinId(InstanceTable.TABLE, InstanceTable.COLUMN_INSTANCE_ID); + } + + @Override public void save(InstanceDataDefine.Instance instance) { + logger.debug("save instance register info, application id: {}, agentUUID: {}", instance.getApplicationId(), instance.getAgentUUID()); + ElasticSearchClient client = getClient(); + Map source = new HashMap(); + source.put(InstanceTable.COLUMN_INSTANCE_ID, instance.getInstanceId()); + source.put(InstanceTable.COLUMN_APPLICATION_ID, instance.getApplicationId()); + source.put(InstanceTable.COLUMN_AGENTUUID, instance.getAgentUUID()); + source.put(InstanceTable.COLUMN_REGISTER_TIME, instance.getRegisterTime()); + + IndexResponse response = client.prepareIndex(InstanceTable.TABLE, instance.getId()).setSource(source).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + logger.debug("save instance register info, application id: {}, agentUUID: {}, status: {}", instance.getApplicationId(), instance.getAgentUUID(), response.status().name()); + } + + @Override public void updateHeartbeatTime(int instanceId, long heartbeatTime) { + ElasticSearchClient client = getClient(); + UpdateRequest updateRequest = new UpdateRequest(); + updateRequest.index(InstanceTable.TABLE); + updateRequest.type("type"); + updateRequest.id(String.valueOf(instanceId)); + updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + Map source = new HashMap(); + source.put(InstanceTable.COLUMN_HEARTBEAT_TIME, heartbeatTime); + + updateRequest.doc(source); + client.update(updateRequest); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceH2DAO.java new file mode 100644 index 000000000..64070090e --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceH2DAO.java @@ -0,0 +1,29 @@ +package org.skywalking.apm.collector.agentstream.worker.register.instance.dao; + +import org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceDataDefine; +import org.skywalking.apm.collector.storage.h2.dao.H2DAO; + +/** + * @author pengys5 + */ +public class InstanceH2DAO extends H2DAO implements IInstanceDAO { + @Override public int getInstanceId(int applicationId, String agentUUID) { + return 0; + } + + @Override public int getMaxInstanceId() { + return 0; + } + + @Override public int getMinInstanceId() { + return 0; + } + + @Override public void save(InstanceDataDefine.Instance instance) { + + } + + @Override public void updateHeartbeatTime(int instanceId, long heartbeatTime) { + + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameDataDefine.java new file mode 100644 index 000000000..d30a15240 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameDataDefine.java @@ -0,0 +1,95 @@ +package org.skywalking.apm.collector.agentstream.worker.register.servicename; + +import org.skywalking.apm.collector.remote.grpc.proto.RemoteData; +import org.skywalking.apm.collector.stream.worker.impl.data.Attribute; +import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation; + +/** + * @author pengys5 + */ +public class ServiceNameDataDefine extends DataDefine { + + public static final int DEFINE_ID = 103; + + @Override public int defineId() { + return DEFINE_ID; + } + + @Override protected int initialCapacity() { + return 4; + } + + @Override protected void attributeDefine() { + addAttribute(0, new Attribute("id", AttributeType.STRING, new NonOperation())); + addAttribute(1, new Attribute(ServiceNameTable.COLUMN_SERVICE_NAME, AttributeType.STRING, new CoverOperation())); + addAttribute(2, new Attribute(ServiceNameTable.COLUMN_APPLICATION_ID, AttributeType.INTEGER, new CoverOperation())); + addAttribute(3, new Attribute(ServiceNameTable.COLUMN_SERVICE_ID, AttributeType.INTEGER, new CoverOperation())); + } + + @Override public Object deserialize(RemoteData remoteData) { + String id = remoteData.getDataStrings(0); + String serviceName = remoteData.getDataStrings(1); + int applicationId = remoteData.getDataIntegers(0); + int serviceId = remoteData.getDataIntegers(1); + return new ServiceName(id, serviceName, applicationId, serviceId); + } + + @Override public RemoteData serialize(Object object) { + ServiceName serviceName = (ServiceName)object; + RemoteData.Builder builder = RemoteData.newBuilder(); + builder.addDataStrings(serviceName.getId()); + builder.addDataStrings(serviceName.getServiceName()); + builder.addDataIntegers(serviceName.getApplicationId()); + builder.addDataIntegers(serviceName.getServiceId()); + return builder.build(); + } + + public static class ServiceName { + private String id; + private String serviceName; + private int applicationId; + private int serviceId; + + public ServiceName(String id, String serviceName, int applicationId, int serviceId) { + this.id = id; + this.serviceName = serviceName; + this.applicationId = applicationId; + this.serviceId = serviceId; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getServiceName() { + return serviceName; + } + + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + public int getApplicationId() { + return applicationId; + } + + public void setApplicationId(int applicationId) { + this.applicationId = applicationId; + } + + public int getServiceId() { + return serviceId; + } + + public void setServiceId(int serviceId) { + this.serviceId = serviceId; + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameEsTableDefine.java new file mode 100644 index 000000000..9d3a33384 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameEsTableDefine.java @@ -0,0 +1,32 @@ +package org.skywalking.apm.collector.agentstream.worker.register.servicename; + +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine; +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine; + +/** + * @author pengys5 + */ +public class ServiceNameEsTableDefine extends ElasticSearchTableDefine { + + public ServiceNameEsTableDefine() { + super(ServiceNameTable.TABLE); + } + + @Override public int refreshInterval() { + return 0; + } + + @Override public int numberOfShards() { + return 2; + } + + @Override public int numberOfReplicas() { + return 0; + } + + @Override public void initialize() { + addColumn(new ElasticSearchColumnDefine(ServiceNameTable.COLUMN_APPLICATION_ID, ElasticSearchColumnDefine.Type.Integer.name())); + addColumn(new ElasticSearchColumnDefine(ServiceNameTable.COLUMN_SERVICE_NAME, ElasticSearchColumnDefine.Type.Keyword.name())); + addColumn(new ElasticSearchColumnDefine(ServiceNameTable.COLUMN_SERVICE_ID, ElasticSearchColumnDefine.Type.Integer.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameH2TableDefine.java new file mode 100644 index 000000000..1ead6dc16 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameH2TableDefine.java @@ -0,0 +1,20 @@ +package org.skywalking.apm.collector.agentstream.worker.register.servicename; + +import org.skywalking.apm.collector.storage.h2.define.H2ColumnDefine; +import org.skywalking.apm.collector.storage.h2.define.H2TableDefine; + +/** + * @author pengys5 + */ +public class ServiceNameH2TableDefine extends H2TableDefine { + + public ServiceNameH2TableDefine() { + super(ServiceNameTable.TABLE); + } + + @Override public void initialize() { + addColumn(new H2ColumnDefine(ServiceNameTable.COLUMN_APPLICATION_ID, H2ColumnDefine.Type.Int.name())); + addColumn(new H2ColumnDefine(ServiceNameTable.COLUMN_SERVICE_NAME, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(ServiceNameTable.COLUMN_SERVICE_ID, H2ColumnDefine.Type.Int.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameRegisterRemoteWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameRegisterRemoteWorker.java new file mode 100644 index 000000000..397eb475b --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameRegisterRemoteWorker.java @@ -0,0 +1,64 @@ +package org.skywalking.apm.collector.agentstream.worker.register.servicename; + +import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorker; +import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerException; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.ForeverFirstSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class ServiceNameRegisterRemoteWorker extends AbstractRemoteWorker { + + private final Logger logger = LoggerFactory.getLogger(ServiceNameRegisterRemoteWorker.class); + + protected ServiceNameRegisterRemoteWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + } + + @Override protected void onWork(Object message) throws WorkerException { + ServiceNameDataDefine.ServiceName serviceName = (ServiceNameDataDefine.ServiceName)message; + logger.debug("service name: {}", serviceName.getServiceName()); + getClusterContext().lookup(ServiceNameRegisterSerialWorker.WorkerRole.INSTANCE).tell(serviceName); + } + + public static class Factory extends AbstractRemoteWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public ServiceNameRegisterRemoteWorker workerInstance(ClusterWorkerContext clusterContext) { + return new ServiceNameRegisterRemoteWorker(role(), clusterContext); + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return ServiceNameRegisterRemoteWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new ForeverFirstSelector(); + } + + @Override public DataDefine dataDefine() { + return new ServiceNameDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameRegisterSerialWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameRegisterSerialWorker.java new file mode 100644 index 000000000..b0d235759 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameRegisterSerialWorker.java @@ -0,0 +1,86 @@ +package org.skywalking.apm.collector.agentstream.worker.register.servicename; + +import org.skywalking.apm.collector.agentstream.worker.register.IdAutoIncrement; +import org.skywalking.apm.collector.agentstream.worker.register.servicename.dao.IServiceNameDAO; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorker; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerException; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.ForeverFirstSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class ServiceNameRegisterSerialWorker extends AbstractLocalAsyncWorker { + + private final Logger logger = LoggerFactory.getLogger(ServiceNameRegisterSerialWorker.class); + + public ServiceNameRegisterSerialWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + super.preStart(); + } + + @Override protected void onWork(Object message) throws WorkerException { + if (message instanceof ServiceNameDataDefine.ServiceName) { + ServiceNameDataDefine.ServiceName serviceName = (ServiceNameDataDefine.ServiceName)message; + logger.debug("register service name: {}, application id: {}", serviceName.getServiceName(), serviceName.getApplicationId()); + + IServiceNameDAO dao = (IServiceNameDAO)DAOContainer.INSTANCE.get(IServiceNameDAO.class.getName()); + int min = dao.getMinServiceId(); + if (min == 0) { + serviceName.setServiceId(1); + serviceName.setId("1"); + } else { + int max = dao.getMaxServiceId(); + int serviceId = IdAutoIncrement.INSTANCE.increment(min, max); + serviceName.setApplicationId(serviceId); + serviceName.setId(String.valueOf(serviceId)); + } + dao.save(serviceName); + } + } + + public static class Factory extends AbstractLocalAsyncWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public ServiceNameRegisterSerialWorker workerInstance(ClusterWorkerContext clusterContext) { + return new ServiceNameRegisterSerialWorker(role(), clusterContext); + } + + @Override public int queueSize() { + return 256; + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return ServiceNameRegisterSerialWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new ForeverFirstSelector(); + } + + @Override public DataDefine dataDefine() { + return new ServiceNameDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameTable.java new file mode 100644 index 000000000..f15fd6075 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameTable.java @@ -0,0 +1,11 @@ +package org.skywalking.apm.collector.agentstream.worker.register.servicename; + +/** + * @author pengys5 + */ +public class ServiceNameTable { + public static final String TABLE = "service_name"; + public static final String COLUMN_SERVICE_NAME = "service_name"; + public static final String COLUMN_APPLICATION_ID = "application_id"; + public static final String COLUMN_SERVICE_ID = "service_id"; +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/dao/IServiceNameDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/dao/IServiceNameDAO.java new file mode 100644 index 000000000..a8752e4cc --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/dao/IServiceNameDAO.java @@ -0,0 +1,16 @@ +package org.skywalking.apm.collector.agentstream.worker.register.servicename.dao; + +import org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameDataDefine; + +/** + * @author pengys5 + */ +public interface IServiceNameDAO { + int getServiceId(int applicationId, String serviceName); + + int getMaxServiceId(); + + int getMinServiceId(); + + void save(ServiceNameDataDefine.ServiceName serviceName); +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/dao/ServiceNameEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/dao/ServiceNameEsDAO.java new file mode 100644 index 000000000..e3cd72202 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/dao/ServiceNameEsDAO.java @@ -0,0 +1,67 @@ +package org.skywalking.apm.collector.agentstream.worker.register.servicename.dao; + +import java.util.HashMap; +import java.util.Map; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameDataDefine; +import org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameTable; +import org.skywalking.apm.collector.client.elasticsearch.ElasticSearchClient; +import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class ServiceNameEsDAO extends EsDAO implements IServiceNameDAO { + + private final Logger logger = LoggerFactory.getLogger(ServiceNameEsDAO.class); + + @Override public int getServiceId(int applicationId, String serviceName) { + ElasticSearchClient client = getClient(); + + SearchRequestBuilder searchRequestBuilder = client.prepareSearch(ServiceNameTable.TABLE); + searchRequestBuilder.setTypes("type"); + searchRequestBuilder.setSearchType(SearchType.QUERY_THEN_FETCH); + BoolQueryBuilder builder = QueryBuilders.boolQuery(); + builder.must().add(QueryBuilders.termQuery(ServiceNameTable.COLUMN_APPLICATION_ID, applicationId)); + builder.must().add(QueryBuilders.termQuery(ServiceNameTable.COLUMN_SERVICE_NAME, serviceName)); + searchRequestBuilder.setQuery(builder); + searchRequestBuilder.setSize(1); + + SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); + if (searchResponse.getHits().totalHits > 0) { + SearchHit searchHit = searchResponse.getHits().iterator().next(); + int serviceId = (int)searchHit.getSource().get(ServiceNameTable.COLUMN_SERVICE_ID); + return serviceId; + } + return 0; + } + + @Override public int getMaxServiceId() { + return getMaxId(ServiceNameTable.TABLE, ServiceNameTable.COLUMN_SERVICE_ID); + } + + @Override public int getMinServiceId() { + return getMinId(ServiceNameTable.TABLE, ServiceNameTable.COLUMN_SERVICE_ID); + } + + @Override public void save(ServiceNameDataDefine.ServiceName serviceName) { + logger.debug("save service name register info, application id: {}, service name: {}", serviceName.getApplicationId(), serviceName.getServiceName()); + ElasticSearchClient client = getClient(); + Map source = new HashMap(); + source.put(ServiceNameTable.COLUMN_SERVICE_ID, serviceName.getServiceId()); + source.put(ServiceNameTable.COLUMN_APPLICATION_ID, serviceName.getApplicationId()); + source.put(ServiceNameTable.COLUMN_SERVICE_NAME, serviceName.getServiceName()); + + IndexResponse response = client.prepareIndex(ServiceNameTable.TABLE, serviceName.getId()).setSource(source).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + logger.debug("save service name register info, application id: {}, service name: {}, status: {}", serviceName.getApplicationId(), serviceName.getServiceName(), response.status().name()); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/dao/ServiceNameH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/dao/ServiceNameH2DAO.java new file mode 100644 index 000000000..f0e2c359f --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/dao/ServiceNameH2DAO.java @@ -0,0 +1,26 @@ +package org.skywalking.apm.collector.agentstream.worker.register.servicename.dao; + +import org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameDataDefine; +import org.skywalking.apm.collector.storage.h2.dao.H2DAO; + +/** + * @author pengys5 + */ +public class ServiceNameH2DAO extends H2DAO implements IServiceNameDAO { + + @Override public int getServiceId(int applicationId, String serviceName) { + return 0; + } + + @Override public int getMaxServiceId() { + return 0; + } + + @Override public int getMinServiceId() { + return 0; + } + + @Override public void save(ServiceNameDataDefine.ServiceName serviceName) { + + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/EntrySpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/EntrySpanListener.java new file mode 100644 index 000000000..6c2ae8211 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/EntrySpanListener.java @@ -0,0 +1,10 @@ +package org.skywalking.apm.collector.agentstream.worker.segment; + +import org.skywalking.apm.network.proto.SpanObject; + +/** + * @author pengys5 + */ +public interface EntrySpanListener extends SpanListener { + void parseEntry(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId); +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/ExitSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/ExitSpanListener.java new file mode 100644 index 000000000..16cd6e727 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/ExitSpanListener.java @@ -0,0 +1,10 @@ +package org.skywalking.apm.collector.agentstream.worker.segment; + +import org.skywalking.apm.network.proto.SpanObject; + +/** + * @author pengys5 + */ +public interface ExitSpanListener extends SpanListener { + void parseExit(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId); +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/FirstSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/FirstSpanListener.java new file mode 100644 index 000000000..b7a7ce957 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/FirstSpanListener.java @@ -0,0 +1,10 @@ +package org.skywalking.apm.collector.agentstream.worker.segment; + +import org.skywalking.apm.network.proto.SpanObject; + +/** + * @author pengys5 + */ +public interface FirstSpanListener extends SpanListener { + void parseFirst(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId); +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/GlobalTraceIdsListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/GlobalTraceIdsListener.java new file mode 100644 index 000000000..2c4c31f8d --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/GlobalTraceIdsListener.java @@ -0,0 +1,10 @@ +package org.skywalking.apm.collector.agentstream.worker.segment; + +import org.skywalking.apm.network.proto.UniqueId; + +/** + * @author pengys5 + */ +public interface GlobalTraceIdsListener extends SpanListener { + void parseGlobalTraceId(UniqueId uniqueId); +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/LocalSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/LocalSpanListener.java new file mode 100644 index 000000000..f605642d1 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/LocalSpanListener.java @@ -0,0 +1,10 @@ +package org.skywalking.apm.collector.agentstream.worker.segment; + +import org.skywalking.apm.network.proto.SpanObject; + +/** + * @author pengys5 + */ +public interface LocalSpanListener extends SpanListener { + void parseLocal(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId); +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/RefsListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/RefsListener.java new file mode 100644 index 000000000..39dc25669 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/RefsListener.java @@ -0,0 +1,10 @@ +package org.skywalking.apm.collector.agentstream.worker.segment; + +import org.skywalking.apm.network.proto.TraceSegmentReference; + +/** + * @author pengys5 + */ +public interface RefsListener extends SpanListener { + void parseRef(TraceSegmentReference reference, int applicationId, int applicationInstanceId, String segmentId); +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java new file mode 100644 index 000000000..dadec0762 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java @@ -0,0 +1,158 @@ +package org.skywalking.apm.collector.agentstream.worker.segment; + +import java.util.ArrayList; +import java.util.List; +import org.skywalking.apm.collector.agentstream.worker.global.GlobalTraceSpanListener; +import org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentSpanListener; +import org.skywalking.apm.collector.agentstream.worker.node.mapping.NodeMappingSpanListener; +import org.skywalking.apm.collector.agentstream.worker.noderef.reference.NodeRefSpanListener; +import org.skywalking.apm.collector.agentstream.worker.noderef.summary.NodeRefSumSpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.cost.SegmentCostSpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.origin.SegmentPersistenceWorker; +import org.skywalking.apm.collector.agentstream.worker.segment.origin.define.SegmentDataDefine; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.core.util.CollectionUtils; +import org.skywalking.apm.collector.stream.StreamModuleContext; +import org.skywalking.apm.collector.stream.StreamModuleGroupDefine; +import org.skywalking.apm.collector.stream.worker.WorkerInvokeException; +import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; +import org.skywalking.apm.network.proto.SpanObject; +import org.skywalking.apm.network.proto.SpanType; +import org.skywalking.apm.network.proto.TraceSegmentObject; +import org.skywalking.apm.network.proto.TraceSegmentReference; +import org.skywalking.apm.network.proto.UniqueId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class SegmentParse { + + private final Logger logger = LoggerFactory.getLogger(SegmentParse.class); + + private List spanListeners; + + public SegmentParse() { + spanListeners = new ArrayList<>(); + spanListeners.add(new NodeRefSpanListener()); + spanListeners.add(new NodeComponentSpanListener()); + spanListeners.add(new NodeMappingSpanListener()); + spanListeners.add(new NodeRefSpanListener()); + spanListeners.add(new NodeRefSumSpanListener()); + spanListeners.add(new SegmentCostSpanListener()); + spanListeners.add(new GlobalTraceSpanListener()); + } + + public void parse(List traceIds, TraceSegmentObject segmentObject) { + StringBuilder segmentIdBuilder = new StringBuilder(); + segmentObject.getTraceSegmentId().getIdPartsList().forEach(part -> { + segmentIdBuilder.append(part); + }); + String segmentId = segmentIdBuilder.toString(); + + for (UniqueId uniqueId : traceIds) { + notifyGlobalsListener(uniqueId); + } + + int applicationId = segmentObject.getApplicationId(); + int applicationInstanceId = segmentObject.getApplicationInstanceId(); + + for (TraceSegmentReference reference : segmentObject.getRefsList()) { + notifyRefsListener(reference, applicationId, applicationInstanceId, segmentId); + } + + List spans = segmentObject.getSpansList(); + if (CollectionUtils.isNotEmpty(spans)) { + for (SpanObject spanObject : spans) { + if (spanObject.getSpanId() == 0) { + notifyFirstListener(spanObject, applicationId, applicationInstanceId, segmentId); + } + + if (SpanType.Exit.equals(spanObject.getSpanType())) { + notifyExitListener(spanObject, applicationId, applicationInstanceId, segmentId); + } else if (SpanType.Entry.equals(spanObject.getSpanType())) { + notifyEntryListener(spanObject, applicationId, applicationInstanceId, segmentId); + } else if (SpanType.Local.equals(spanObject.getSpanType())) { + notifyLocalListener(spanObject, applicationId, applicationInstanceId, segmentId); + } else { + logger.error("span type error, span type: {}", spanObject.getSpanType().name()); + } + } + } + + notifyListenerToBuild(); + buildSegment(segmentId, segmentObject.toByteArray()); + } + + public void buildSegment(String id, byte[] dataBinary) { + StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); + SegmentDataDefine.Segment segment = new SegmentDataDefine.Segment(); + segment.setId(id); + segment.setDataBinary(dataBinary); + + try { + logger.debug("send to segment persistence worker, id: {}, dataBinary length: {}", segment.getId(), dataBinary.length); + context.getClusterWorkerContext().lookup(SegmentPersistenceWorker.WorkerRole.INSTANCE).tell(segment.transform()); + } catch (WorkerInvokeException | WorkerNotFoundException e) { + logger.error(e.getMessage(), e); + } + } + + private void notifyListenerToBuild() { + spanListeners.forEach(listener -> listener.build()); + } + + private void notifyExitListener(SpanObject spanObject, int applicationId, int applicationInstanceId, + String segmentId) { + for (SpanListener listener : spanListeners) { + if (listener instanceof ExitSpanListener) { + ((ExitSpanListener)listener).parseExit(spanObject, applicationId, applicationInstanceId, segmentId); + } + } + } + + private void notifyEntryListener(SpanObject spanObject, int applicationId, int applicationInstanceId, + String segmentId) { + for (SpanListener listener : spanListeners) { + if (listener instanceof EntrySpanListener) { + ((EntrySpanListener)listener).parseEntry(spanObject, applicationId, applicationInstanceId, segmentId); + } + } + } + + private void notifyLocalListener(SpanObject spanObject, int applicationId, int applicationInstanceId, + String segmentId) { + for (SpanListener listener : spanListeners) { + if (listener instanceof LocalSpanListener) { + ((LocalSpanListener)listener).parseLocal(spanObject, applicationId, applicationInstanceId, segmentId); + } + } + } + + private void notifyFirstListener(SpanObject spanObject, int applicationId, int applicationInstanceId, + String segmentId) { + for (SpanListener listener : spanListeners) { + if (listener instanceof FirstSpanListener) { + ((FirstSpanListener)listener).parseFirst(spanObject, applicationId, applicationInstanceId, segmentId); + } + } + } + + private void notifyRefsListener(TraceSegmentReference reference, int applicationId, int applicationInstanceId, + String segmentId) { + for (SpanListener listener : spanListeners) { + if (listener instanceof RefsListener) { + ((RefsListener)listener).parseRef(reference, applicationId, applicationInstanceId, segmentId); + } + } + } + + private void notifyGlobalsListener(UniqueId uniqueId) { + for (SpanListener listener : spanListeners) { + if (listener instanceof GlobalTraceIdsListener) { + ((GlobalTraceIdsListener)listener).parseGlobalTraceId(uniqueId); + } + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SpanListener.java new file mode 100644 index 000000000..3dde04717 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SpanListener.java @@ -0,0 +1,8 @@ +package org.skywalking.apm.collector.agentstream.worker.segment; + +/** + * @author pengys5 + */ +public interface SpanListener { + void build(); +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/SegmentCostPersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/SegmentCostPersistenceWorker.java new file mode 100644 index 000000000..d6b72f8a2 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/SegmentCostPersistenceWorker.java @@ -0,0 +1,70 @@ +package org.skywalking.apm.collector.agentstream.worker.segment.cost; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.agentstream.worker.segment.cost.dao.ISegmentCostDAO; +import org.skywalking.apm.collector.agentstream.worker.segment.cost.define.SegmentCostDataDefine; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.RollingSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class SegmentCostPersistenceWorker extends PersistenceWorker { + + public SegmentCostPersistenceWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + super.preStart(); + } + + @Override protected List prepareBatch(Map dataMap) { + ISegmentCostDAO dao = (ISegmentCostDAO)DAOContainer.INSTANCE.get(ISegmentCostDAO.class.getName()); + return dao.prepareBatch(dataMap); + } + + public static class Factory extends AbstractLocalAsyncWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public SegmentCostPersistenceWorker workerInstance(ClusterWorkerContext clusterContext) { + return new SegmentCostPersistenceWorker(role(), clusterContext); + } + + @Override + public int queueSize() { + return 1024; + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return SegmentCostPersistenceWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new RollingSelector(); + } + + @Override public DataDefine dataDefine() { + return new SegmentCostDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/SegmentCostSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/SegmentCostSpanListener.java new file mode 100644 index 000000000..a8508cd3d --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/SegmentCostSpanListener.java @@ -0,0 +1,75 @@ +package org.skywalking.apm.collector.agentstream.worker.segment.cost; + +import java.util.ArrayList; +import java.util.List; +import org.skywalking.apm.collector.agentstream.worker.segment.EntrySpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.ExitSpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.LocalSpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.cost.define.SegmentCostDataDefine; +import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.stream.StreamModuleContext; +import org.skywalking.apm.collector.stream.StreamModuleGroupDefine; +import org.skywalking.apm.collector.stream.worker.WorkerInvokeException; +import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; +import org.skywalking.apm.network.proto.SpanObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class SegmentCostSpanListener implements EntrySpanListener, ExitSpanListener, LocalSpanListener, FirstSpanListener { + + private final Logger logger = LoggerFactory.getLogger(SegmentCostSpanListener.class); + + private List segmentCosts = new ArrayList<>(); + private boolean isError = false; + private long timeBucket; + + @Override + public void parseFirst(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + timeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(spanObject.getStartTime()); + } + + @Override + public void parseEntry(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + SegmentCostDataDefine.SegmentCost segmentCost = new SegmentCostDataDefine.SegmentCost(); + segmentCost.setSegmentId(segmentId); + segmentCost.setCost(spanObject.getEndTime() - spanObject.getStartTime()); + segmentCost.setStartTime(spanObject.getStartTime()); + segmentCost.setEndTime(spanObject.getEndTime()); + segmentCost.setOperationName(spanObject.getOperationName()); + segmentCost.setId(segmentId); + segmentCosts.add(segmentCost); + + isError = isError || spanObject.getIsError(); + } + + @Override + public void parseExit(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + isError = isError || spanObject.getIsError(); + } + + @Override + public void parseLocal(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + isError = isError || spanObject.getIsError(); + } + + @Override public void build() { + logger.debug("segment cost listener build"); + StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); + + for (SegmentCostDataDefine.SegmentCost segmentCost : segmentCosts) { + segmentCost.setError(isError); + segmentCost.setTimeBucket(timeBucket); + try { + logger.debug("send to segment cost persistence worker, id: {}", segmentCost.getId()); + context.getClusterWorkerContext().lookup(SegmentCostPersistenceWorker.WorkerRole.INSTANCE).tell(segmentCost.transform()); + } catch (WorkerInvokeException | WorkerNotFoundException e) { + logger.error(e.getMessage(), e); + } + } + } +} \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/ISegmentCostDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/ISegmentCostDAO.java new file mode 100644 index 000000000..03787ca00 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/ISegmentCostDAO.java @@ -0,0 +1,12 @@ +package org.skywalking.apm.collector.agentstream.worker.segment.cost.dao; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; + +/** + * @author pengys5 + */ +public interface ISegmentCostDAO { + List prepareBatch(Map dataMap); +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/SegmentCostEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/SegmentCostEsDAO.java new file mode 100644 index 000000000..d1b9e19b2 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/SegmentCostEsDAO.java @@ -0,0 +1,39 @@ +package org.skywalking.apm.collector.agentstream.worker.segment.cost.dao; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.skywalking.apm.collector.agentstream.worker.segment.cost.define.SegmentCostTable; +import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class SegmentCostEsDAO extends EsDAO implements ISegmentCostDAO { + + private final Logger logger = LoggerFactory.getLogger(SegmentCostEsDAO.class); + + @Override public List prepareBatch(Map dataMap) { + List indexRequestBuilders = new ArrayList<>(); + dataMap.forEach((id, data) -> { + logger.debug("segment cost prepareBatch, id: {}", id); + Map source = new HashMap(); + source.put(SegmentCostTable.COLUMN_SEGMENT_ID, data.getDataString(1)); + source.put(SegmentCostTable.COLUMN_OPERATION_NAME, data.getDataString(2)); + source.put(SegmentCostTable.COLUMN_COST, data.getDataLong(0)); + source.put(SegmentCostTable.COLUMN_START_TIME, data.getDataLong(1)); + source.put(SegmentCostTable.COLUMN_END_TIME, data.getDataLong(2)); + source.put(SegmentCostTable.COLUMN_IS_ERROR, data.getDataBoolean(0)); + source.put(SegmentCostTable.COLUMN_TIME_BUCKET, data.getDataLong(3)); + logger.debug("segment cost source: {}", source.toString()); + IndexRequestBuilder builder = getClient().prepareIndex(SegmentCostTable.TABLE, id).setSource(source); + indexRequestBuilders.add(builder); + }); + return indexRequestBuilders; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/SegmentCostH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/SegmentCostH2DAO.java new file mode 100644 index 000000000..90fd044b9 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/SegmentCostH2DAO.java @@ -0,0 +1,15 @@ +package org.skywalking.apm.collector.agentstream.worker.segment.cost.dao; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.storage.h2.dao.H2DAO; + +/** + * @author pengys5 + */ +public class SegmentCostH2DAO extends H2DAO implements ISegmentCostDAO { + + @Override public List prepareBatch(Map map) { + return null; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostDataDefine.java new file mode 100644 index 000000000..4c59a1f28 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostDataDefine.java @@ -0,0 +1,165 @@ +package org.skywalking.apm.collector.agentstream.worker.segment.cost.define; + +import org.skywalking.apm.collector.remote.grpc.proto.RemoteData; +import org.skywalking.apm.collector.stream.worker.impl.data.Attribute; +import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.impl.data.TransformToData; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation; + +/** + * @author pengys5 + */ +public class SegmentCostDataDefine extends DataDefine { + + @Override public int defineId() { + return 402; + } + + @Override protected int initialCapacity() { + return 8; + } + + @Override protected void attributeDefine() { + addAttribute(0, new Attribute(SegmentCostTable.COLUMN_ID, AttributeType.STRING, new NonOperation())); + addAttribute(1, new Attribute(SegmentCostTable.COLUMN_SEGMENT_ID, AttributeType.STRING, new CoverOperation())); + addAttribute(2, new Attribute(SegmentCostTable.COLUMN_OPERATION_NAME, AttributeType.STRING, new CoverOperation())); + addAttribute(3, new Attribute(SegmentCostTable.COLUMN_COST, AttributeType.LONG, new CoverOperation())); + addAttribute(4, new Attribute(SegmentCostTable.COLUMN_START_TIME, AttributeType.LONG, new CoverOperation())); + addAttribute(5, new Attribute(SegmentCostTable.COLUMN_END_TIME, AttributeType.LONG, new CoverOperation())); + addAttribute(6, new Attribute(SegmentCostTable.COLUMN_IS_ERROR, AttributeType.BOOLEAN, new CoverOperation())); + addAttribute(7, new Attribute(SegmentCostTable.COLUMN_TIME_BUCKET, AttributeType.LONG, new NonOperation())); + } + + @Override public Object deserialize(RemoteData remoteData) { + String id = remoteData.getDataStrings(0); + String segmentId = remoteData.getDataStrings(1); + String operationName = remoteData.getDataStrings(2); + Long cost = remoteData.getDataLongs(0); + Long startTime = remoteData.getDataLongs(1); + Long endTime = remoteData.getDataLongs(2); + Boolean isError = remoteData.getDataBooleans(0); + Long timeBucket = remoteData.getDataLongs(2); + return new SegmentCost(id, segmentId, operationName, cost, startTime, endTime, isError, timeBucket); + } + + @Override public RemoteData serialize(Object object) { + SegmentCost segmentCost = (SegmentCost)object; + RemoteData.Builder builder = RemoteData.newBuilder(); + builder.addDataStrings(segmentCost.getId()); + builder.addDataStrings(segmentCost.getSegmentId()); + builder.addDataStrings(segmentCost.getOperationName()); + builder.addDataLongs(segmentCost.getCost()); + builder.addDataLongs(segmentCost.getStartTime()); + builder.addDataLongs(segmentCost.getEndTime()); + builder.addDataBooleans(segmentCost.isError()); + builder.addDataLongs(segmentCost.getTimeBucket()); + return builder.build(); + } + + public static class SegmentCost implements TransformToData { + private String id; + private String segmentId; + private String operationName; + private Long cost; + private Long startTime; + private Long endTime; + private boolean isError; + private long timeBucket; + + SegmentCost(String id, String segmentId, String operationName, Long cost, + Long startTime, Long endTime, boolean isError, long timeBucket) { + this.id = id; + this.segmentId = segmentId; + this.operationName = operationName; + this.cost = cost; + this.startTime = startTime; + this.endTime = endTime; + this.isError = isError; + this.timeBucket = timeBucket; + } + + public SegmentCost() { + } + + @Override public Data transform() { + SegmentCostDataDefine define = new SegmentCostDataDefine(); + Data data = define.build(id); + data.setDataString(0, this.id); + data.setDataString(1, this.segmentId); + data.setDataString(2, this.operationName); + data.setDataLong(0, this.cost); + data.setDataLong(1, this.startTime); + data.setDataLong(2, this.endTime); + data.setDataBoolean(0, this.isError); + data.setDataLong(3, this.timeBucket); + return data; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getSegmentId() { + return segmentId; + } + + public void setSegmentId(String segmentId) { + this.segmentId = segmentId; + } + + public String getOperationName() { + return operationName; + } + + public void setOperationName(String operationName) { + this.operationName = operationName; + } + + public Long getCost() { + return cost; + } + + public void setCost(Long cost) { + this.cost = cost; + } + + public Long getStartTime() { + return startTime; + } + + public void setStartTime(Long startTime) { + this.startTime = startTime; + } + + public Long getEndTime() { + return endTime; + } + + public void setEndTime(Long endTime) { + this.endTime = endTime; + } + + public boolean isError() { + return isError; + } + + public void setError(boolean error) { + isError = error; + } + + public long getTimeBucket() { + return timeBucket; + } + + public void setTimeBucket(long timeBucket) { + this.timeBucket = timeBucket; + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostEsTableDefine.java new file mode 100644 index 000000000..4f493da99 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostEsTableDefine.java @@ -0,0 +1,36 @@ +package org.skywalking.apm.collector.agentstream.worker.segment.cost.define; + +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine; +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine; + +/** + * @author pengys5 + */ +public class SegmentCostEsTableDefine extends ElasticSearchTableDefine { + + public SegmentCostEsTableDefine() { + super(SegmentCostTable.TABLE); + } + + @Override public int refreshInterval() { + return 2; + } + + @Override public int numberOfShards() { + return 2; + } + + @Override public int numberOfReplicas() { + return 0; + } + + @Override public void initialize() { + addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_SEGMENT_ID, ElasticSearchColumnDefine.Type.Keyword.name())); + addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_OPERATION_NAME, ElasticSearchColumnDefine.Type.Keyword.name())); + addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_COST, ElasticSearchColumnDefine.Type.Long.name())); + addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_START_TIME, ElasticSearchColumnDefine.Type.Long.name())); + addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_END_TIME, ElasticSearchColumnDefine.Type.Long.name())); + addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_IS_ERROR, ElasticSearchColumnDefine.Type.Boolean.name())); + addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_TIME_BUCKET, ElasticSearchColumnDefine.Type.Long.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostH2TableDefine.java new file mode 100644 index 000000000..c4ca90226 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostH2TableDefine.java @@ -0,0 +1,25 @@ +package org.skywalking.apm.collector.agentstream.worker.segment.cost.define; + +import org.skywalking.apm.collector.storage.h2.define.H2ColumnDefine; +import org.skywalking.apm.collector.storage.h2.define.H2TableDefine; + +/** + * @author pengys5 + */ +public class SegmentCostH2TableDefine extends H2TableDefine { + + public SegmentCostH2TableDefine() { + super(SegmentCostTable.TABLE); + } + + @Override public void initialize() { + addColumn(new H2ColumnDefine(SegmentCostTable.COLUMN_ID, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(SegmentCostTable.COLUMN_SEGMENT_ID, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(SegmentCostTable.COLUMN_OPERATION_NAME, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(SegmentCostTable.COLUMN_COST, H2ColumnDefine.Type.Bigint.name())); + addColumn(new H2ColumnDefine(SegmentCostTable.COLUMN_START_TIME, H2ColumnDefine.Type.Bigint.name())); + addColumn(new H2ColumnDefine(SegmentCostTable.COLUMN_END_TIME, H2ColumnDefine.Type.Bigint.name())); + addColumn(new H2ColumnDefine(SegmentCostTable.COLUMN_IS_ERROR, H2ColumnDefine.Type.Boolean.name())); + addColumn(new H2ColumnDefine(SegmentCostTable.COLUMN_TIME_BUCKET, H2ColumnDefine.Type.Bigint.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostTable.java new file mode 100644 index 000000000..0bc0ee947 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostTable.java @@ -0,0 +1,16 @@ +package org.skywalking.apm.collector.agentstream.worker.segment.cost.define; + +import org.skywalking.apm.collector.agentstream.worker.CommonTable; + +/** + * @author pengys5 + */ +public class SegmentCostTable extends CommonTable { + public static final String TABLE = "segment_cost"; + public static final String COLUMN_SEGMENT_ID = "segment_id"; + public static final String COLUMN_START_TIME = "start_time"; + public static final String COLUMN_END_TIME = "end_time"; + public static final String COLUMN_OPERATION_NAME = "operation_name"; + public static final String COLUMN_COST = "cost"; + public static final String COLUMN_IS_ERROR = "is_error"; +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/SegmentPersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/SegmentPersistenceWorker.java new file mode 100644 index 000000000..a76fbd11d --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/SegmentPersistenceWorker.java @@ -0,0 +1,70 @@ +package org.skywalking.apm.collector.agentstream.worker.segment.origin; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.agentstream.worker.segment.origin.dao.ISegmentDAO; +import org.skywalking.apm.collector.agentstream.worker.segment.origin.define.SegmentDataDefine; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.RollingSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class SegmentPersistenceWorker extends PersistenceWorker { + + public SegmentPersistenceWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + super.preStart(); + } + + @Override protected List prepareBatch(Map dataMap) { + ISegmentDAO dao = (ISegmentDAO)DAOContainer.INSTANCE.get(ISegmentDAO.class.getName()); + return dao.prepareBatch(dataMap); + } + + public static class Factory extends AbstractLocalAsyncWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public SegmentPersistenceWorker workerInstance(ClusterWorkerContext clusterContext) { + return new SegmentPersistenceWorker(role(), clusterContext); + } + + @Override + public int queueSize() { + return 1024; + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return SegmentPersistenceWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new RollingSelector(); + } + + @Override public DataDefine dataDefine() { + return new SegmentDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/ISegmentDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/ISegmentDAO.java new file mode 100644 index 000000000..1c1d67e03 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/ISegmentDAO.java @@ -0,0 +1,12 @@ +package org.skywalking.apm.collector.agentstream.worker.segment.origin.dao; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; + +/** + * @author pengys5 + */ +public interface ISegmentDAO { + List prepareBatch(Map dataMap); +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/SegmentEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/SegmentEsDAO.java new file mode 100644 index 000000000..11ced7431 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/SegmentEsDAO.java @@ -0,0 +1,34 @@ +package org.skywalking.apm.collector.agentstream.worker.segment.origin.dao; + +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.skywalking.apm.collector.agentstream.worker.segment.origin.define.SegmentTable; +import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class SegmentEsDAO extends EsDAO implements ISegmentDAO { + + private final Logger logger = LoggerFactory.getLogger(SegmentEsDAO.class); + + @Override public List prepareBatch(Map dataMap) { + List indexRequestBuilders = new ArrayList<>(); + dataMap.forEach((id, data) -> { + logger.debug("segment prepareBatch, id: {}", id); + Map source = new HashMap(); + source.put(SegmentTable.COLUMN_DATA_BINARY, new String(Base64.getEncoder().encode(data.getDataBytes(0)))); + logger.debug("segment source: {}", source.toString()); + IndexRequestBuilder builder = getClient().prepareIndex(SegmentTable.TABLE, id).setSource(source); + indexRequestBuilders.add(builder); + }); + return indexRequestBuilders; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/SegmentH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/SegmentH2DAO.java new file mode 100644 index 000000000..5e9918fe8 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/SegmentH2DAO.java @@ -0,0 +1,15 @@ +package org.skywalking.apm.collector.agentstream.worker.segment.origin.dao; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.storage.h2.dao.H2DAO; + +/** + * @author pengys5 + */ +public class SegmentH2DAO extends H2DAO implements ISegmentDAO { + + @Override public List prepareBatch(Map map) { + return null; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/define/SegmentDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/define/SegmentDataDefine.java new file mode 100644 index 000000000..e3cfdb0a4 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/define/SegmentDataDefine.java @@ -0,0 +1,83 @@ +package org.skywalking.apm.collector.agentstream.worker.segment.origin.define; + +import com.google.protobuf.ByteString; +import org.skywalking.apm.collector.remote.grpc.proto.RemoteData; +import org.skywalking.apm.collector.stream.worker.impl.data.Attribute; +import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.impl.data.TransformToData; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation; + +/** + * @author pengys5 + */ +public class SegmentDataDefine extends DataDefine { + + public static final int DEFINE_ID = 401; + + @Override public int defineId() { + return DEFINE_ID; + } + + @Override protected int initialCapacity() { + return 2; + } + + @Override protected void attributeDefine() { + addAttribute(0, new Attribute(SegmentTable.COLUMN_ID, AttributeType.STRING, new NonOperation())); + addAttribute(1, new Attribute(SegmentTable.COLUMN_DATA_BINARY, AttributeType.BYTE, new CoverOperation())); + } + + @Override public Object deserialize(RemoteData remoteData) { + String id = remoteData.getDataStrings(0); + byte[] dataBinary = remoteData.getDataBytes(0).toByteArray(); + return new Segment(id, dataBinary); + } + + @Override public RemoteData serialize(Object object) { + Segment segment = (Segment)object; + RemoteData.Builder builder = RemoteData.newBuilder(); + builder.addDataStrings(segment.getId()); + builder.addDataBytes(ByteString.copyFrom(segment.getDataBinary())); + return builder.build(); + } + + public static class Segment implements TransformToData { + private String id; + private byte[] dataBinary; + + public Segment(String id, byte[] dataBinary) { + this.id = id; + this.dataBinary = dataBinary; + } + + public Segment() { + } + + @Override public Data transform() { + SegmentDataDefine define = new SegmentDataDefine(); + Data data = define.build(id); + data.setDataString(0, this.id); + data.setDataBytes(0, this.dataBinary); + return data; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public byte[] getDataBinary() { + return dataBinary; + } + + public void setDataBinary(byte[] dataBinary) { + this.dataBinary = dataBinary; + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/define/SegmentEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/define/SegmentEsTableDefine.java new file mode 100644 index 000000000..92d58a956 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/define/SegmentEsTableDefine.java @@ -0,0 +1,30 @@ +package org.skywalking.apm.collector.agentstream.worker.segment.origin.define; + +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine; +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine; + +/** + * @author pengys5 + */ +public class SegmentEsTableDefine extends ElasticSearchTableDefine { + + public SegmentEsTableDefine() { + super(SegmentTable.TABLE); + } + + @Override public int refreshInterval() { + return 10; + } + + @Override public int numberOfShards() { + return 2; + } + + @Override public int numberOfReplicas() { + return 0; + } + + @Override public void initialize() { + addColumn(new ElasticSearchColumnDefine(SegmentTable.COLUMN_DATA_BINARY, ElasticSearchColumnDefine.Type.Binary.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/define/SegmentH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/define/SegmentH2TableDefine.java new file mode 100644 index 000000000..cfcf378bc --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/define/SegmentH2TableDefine.java @@ -0,0 +1,19 @@ +package org.skywalking.apm.collector.agentstream.worker.segment.origin.define; + +import org.skywalking.apm.collector.storage.h2.define.H2ColumnDefine; +import org.skywalking.apm.collector.storage.h2.define.H2TableDefine; + +/** + * @author pengys5 + */ +public class SegmentH2TableDefine extends H2TableDefine { + + public SegmentH2TableDefine() { + super(SegmentTable.TABLE); + } + + @Override public void initialize() { + addColumn(new H2ColumnDefine(SegmentTable.COLUMN_ID, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(SegmentTable.COLUMN_DATA_BINARY, H2ColumnDefine.Type.BINARY.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/define/SegmentTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/define/SegmentTable.java new file mode 100644 index 000000000..54577fa09 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/define/SegmentTable.java @@ -0,0 +1,11 @@ +package org.skywalking.apm.collector.agentstream.worker.segment.origin.define; + +import org.skywalking.apm.collector.agentstream.worker.CommonTable; + +/** + * @author pengys5 + */ +public class SegmentTable extends CommonTable { + public static final String TABLE = "segment"; + public static final String COLUMN_DATA_BINARY = "data_binary"; +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/ServiceRefSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/ServiceRefSpanListener.java new file mode 100644 index 000000000..12986db71 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/ServiceRefSpanListener.java @@ -0,0 +1,83 @@ +package org.skywalking.apm.collector.agentstream.worker.serviceref.reference; + +import java.util.ArrayList; +import java.util.List; +import org.skywalking.apm.collector.agentstream.worker.Const; +import org.skywalking.apm.collector.agentstream.worker.segment.EntrySpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.ExitSpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.RefsListener; +import org.skywalking.apm.collector.agentstream.worker.serviceref.reference.define.ServiceRefDataDefine; +import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils; +import org.skywalking.apm.network.proto.SpanObject; +import org.skywalking.apm.network.proto.TraceSegmentReference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class ServiceRefSpanListener implements FirstSpanListener, EntrySpanListener, ExitSpanListener, RefsListener { + + private final Logger logger = LoggerFactory.getLogger(ServiceRefSpanListener.class); + + private String front; + private List behinds = new ArrayList<>(); + private List fronts = new ArrayList<>(); + private long timeBucket; + + @Override + public void parseFirst(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + timeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(spanObject.getStartTime()); + } + + @Override public void parseRef(TraceSegmentReference reference, int applicationId, int applicationInstanceId, + String segmentId) { + String entryService = String.valueOf(reference.getEntryServiceId()); + if (reference.getEntryServiceId() == 0) { + entryService = reference.getEntryServiceName(); + } + String parentService = String.valueOf(reference.getParentServiceId()); + if (reference.getParentServiceId() == 0) { + parentService = reference.getParentServiceName(); + } + fronts.add(new ServiceTemp(entryService, parentService)); + } + + @Override + public void parseEntry(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + front = String.valueOf(spanObject.getOperationNameId()); + if (spanObject.getOperationNameId() == 0) { + front = spanObject.getOperationName(); + } + } + + @Override + public void parseExit(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + String behind = String.valueOf(spanObject.getOperationNameId()); + if (spanObject.getOperationNameId() == 0) { + behind = spanObject.getOperationName(); + } + behinds.add(behind); + } + + @Override public void build() { + for (String behind : behinds) { + String agg = front + Const.ID_SPLIT + behind; + ServiceRefDataDefine.ServiceReference serviceReference = new ServiceRefDataDefine.ServiceReference(); + serviceReference.setId(timeBucket + Const.ID_SPLIT + agg); + serviceReference.setAgg(agg); + serviceReference.setTimeBucket(timeBucket); + } + } + + class ServiceTemp { + private final String entryService; + private final String parentService; + + public ServiceTemp(String entryService, String parentService) { + this.entryService = entryService; + this.parentService = parentService; + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefDataDefine.java new file mode 100644 index 000000000..00ff6b140 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefDataDefine.java @@ -0,0 +1,98 @@ +package org.skywalking.apm.collector.agentstream.worker.serviceref.reference.define; + +import org.skywalking.apm.collector.remote.grpc.proto.RemoteData; +import org.skywalking.apm.collector.stream.worker.impl.data.Attribute; +import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation; + +/** + * @author pengys5 + */ +public class ServiceRefDataDefine extends DataDefine { + + public static final int DEFINE_ID = 501; + + @Override public int defineId() { + return DEFINE_ID; + } + + @Override protected int initialCapacity() { + return 4; + } + + @Override protected void attributeDefine() { + addAttribute(0, new Attribute(ServiceRefTable.COLUMN_ID, AttributeType.STRING, new NonOperation())); + addAttribute(1, new Attribute(ServiceRefTable.COLUMN_ENTRY_SERVICE, AttributeType.STRING, new NonOperation())); + addAttribute(2, new Attribute(ServiceRefTable.COLUMN_AGG, AttributeType.STRING, new CoverOperation())); + addAttribute(3, new Attribute(ServiceRefTable.COLUMN_TIME_BUCKET, AttributeType.LONG, new CoverOperation())); + } + + @Override public Object deserialize(RemoteData remoteData) { + String id = remoteData.getDataStrings(0); + String entryService = remoteData.getDataStrings(1); + String agg = remoteData.getDataStrings(2); + long timeBucket = remoteData.getDataLongs(0); + return new ServiceReference(id, entryService, agg, timeBucket); + } + + @Override public RemoteData serialize(Object object) { + ServiceReference serviceReference = (ServiceReference)object; + RemoteData.Builder builder = RemoteData.newBuilder(); + builder.addDataStrings(serviceReference.getId()); + builder.addDataStrings(serviceReference.getEntryService()); + builder.addDataStrings(serviceReference.getAgg()); + builder.addDataLongs(serviceReference.getTimeBucket()); + return builder.build(); + } + + public static class ServiceReference { + private String id; + private String entryService; + private String agg; + private long timeBucket; + + public ServiceReference(String id, String entryService, String agg, long timeBucket) { + this.id = id; + this.entryService = entryService; + this.agg = agg; + this.timeBucket = timeBucket; + } + + public ServiceReference() { + } + + public String getId() { + return id; + } + + public String getAgg() { + return agg; + } + + public long getTimeBucket() { + return timeBucket; + } + + public void setId(String id) { + this.id = id; + } + + public void setAgg(String agg) { + this.agg = agg; + } + + public void setTimeBucket(long timeBucket) { + this.timeBucket = timeBucket; + } + + public String getEntryService() { + return entryService; + } + + public void setEntryService(String entryService) { + this.entryService = entryService; + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefEsTableDefine.java new file mode 100644 index 000000000..067624ed5 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefEsTableDefine.java @@ -0,0 +1,32 @@ +package org.skywalking.apm.collector.agentstream.worker.serviceref.reference.define; + +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine; +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine; + +/** + * @author pengys5 + */ +public class ServiceRefEsTableDefine extends ElasticSearchTableDefine { + + public ServiceRefEsTableDefine() { + super(ServiceRefTable.TABLE); + } + + @Override public int refreshInterval() { + return 0; + } + + @Override public int numberOfShards() { + return 2; + } + + @Override public int numberOfReplicas() { + return 0; + } + + @Override public void initialize() { + addColumn(new ElasticSearchColumnDefine(ServiceRefTable.COLUMN_AGG, ElasticSearchColumnDefine.Type.Keyword.name())); + addColumn(new ElasticSearchColumnDefine(ServiceRefTable.COLUMN_ENTRY_SERVICE, ElasticSearchColumnDefine.Type.Keyword.name())); + addColumn(new ElasticSearchColumnDefine(ServiceRefTable.COLUMN_TIME_BUCKET, ElasticSearchColumnDefine.Type.Long.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefH2TableDefine.java new file mode 100644 index 000000000..c6756cab5 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefH2TableDefine.java @@ -0,0 +1,20 @@ +package org.skywalking.apm.collector.agentstream.worker.serviceref.reference.define; + +import org.skywalking.apm.collector.storage.h2.define.H2ColumnDefine; +import org.skywalking.apm.collector.storage.h2.define.H2TableDefine; + +/** + * @author pengys5 + */ +public class ServiceRefH2TableDefine extends H2TableDefine { + + public ServiceRefH2TableDefine() { + super(ServiceRefTable.TABLE); + } + + @Override public void initialize() { + addColumn(new H2ColumnDefine(ServiceRefTable.COLUMN_AGG, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(ServiceRefTable.COLUMN_ENTRY_SERVICE, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(ServiceRefTable.COLUMN_TIME_BUCKET, H2ColumnDefine.Type.Bigint.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefTable.java new file mode 100644 index 000000000..d145b93f5 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefTable.java @@ -0,0 +1,11 @@ +package org.skywalking.apm.collector.agentstream.worker.serviceref.reference.define; + +import org.skywalking.apm.collector.agentstream.worker.CommonTable; + +/** + * @author pengys5 + */ +public class ServiceRefTable extends CommonTable { + public static final String TABLE = "service_reference"; + public static final String COLUMN_ENTRY_SERVICE = "entry_service"; +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java new file mode 100644 index 000000000..8052525f9 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java @@ -0,0 +1,58 @@ +package org.skywalking.apm.collector.agentstream.worker.storage; + +import java.util.ArrayList; +import java.util.List; +import org.skywalking.apm.collector.core.framework.Starter; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.storage.dao.IBatchDAO; +import org.skywalking.apm.collector.stream.worker.WorkerException; +import org.skywalking.apm.collector.stream.worker.impl.FlushAndSwitch; +import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; +import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorkerContainer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class PersistenceTimer implements Starter { + + private final Logger logger = LoggerFactory.getLogger(PersistenceTimer.class); + + public void start() { + logger.info("persistence timer start"); + //TODO timer value config +// final long timeInterval = EsConfig.Es.Persistence.Timer.VALUE * 1000; + final long timeInterval = 3 * 1000; + + Thread persistenceThread = new Thread(() -> { + while (true) { + try { + extractDataAndSave(); + Thread.sleep(timeInterval); + } catch (Throwable e) { + logger.error(e.getMessage(), e); + } + } + }); + persistenceThread.setName("timerPersistence"); + persistenceThread.start(); + } + + private void extractDataAndSave() { + List workers = PersistenceWorkerContainer.INSTANCE.getPersistenceWorkers(); + List batchAllCollection = new ArrayList<>(); + workers.forEach((PersistenceWorker worker) -> { + try { + worker.allocateJob(new FlushAndSwitch()); + List batchCollection = worker.buildBatchCollection(); + batchAllCollection.addAll(batchCollection); + } catch (WorkerException e) { + logger.error(e.getMessage(), e); + } + }); + + IBatchDAO dao = (IBatchDAO)DAOContainer.INSTANCE.get(IBatchDAO.class.getName()); + dao.batchPersistence(batchAllCollection); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/util/TimeBucketUtils.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/util/TimeBucketUtils.java new file mode 100644 index 000000000..de8390297 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/util/TimeBucketUtils.java @@ -0,0 +1,47 @@ +package org.skywalking.apm.collector.agentstream.worker.util; + +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.TimeZone; + +/** + * @author pengys5 + */ +public enum TimeBucketUtils { + INSTANCE; + + private final SimpleDateFormat dayDateFormat = new SimpleDateFormat("yyyyMMdd"); + private final SimpleDateFormat hourDateFormat = new SimpleDateFormat("yyyyMMddHH"); + private final SimpleDateFormat minuteDateFormat = new SimpleDateFormat("yyyyMMddHHmm"); + + public long getMinuteTimeBucket(long time) { + Calendar calendar = Calendar.getInstance(); + calendar.setTimeInMillis(time); + String timeStr = minuteDateFormat.format(calendar.getTime()); + return Long.valueOf(timeStr); + } + + public long getHourTimeBucket(long time) { + Calendar calendar = Calendar.getInstance(); + calendar.setTimeInMillis(time); + String timeStr = hourDateFormat.format(calendar.getTime()) + "00"; + return Long.valueOf(timeStr); + } + + public long getDayTimeBucket(long time) { + Calendar calendar = Calendar.getInstance(); + calendar.setTimeInMillis(time); + String timeStr = dayDateFormat.format(calendar.getTime()) + "0000"; + return Long.valueOf(timeStr); + } + + public long changeToUTCTimeBucket(long timeBucket) { + String timeBucketStr = String.valueOf(timeBucket); + + if (TimeZone.getDefault().getID().equals("GMT+08:00") || timeBucketStr.endsWith("0000")) { + return timeBucket; + } else { + return timeBucket - 800; + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define new file mode 100644 index 000000000..c8abe9d72 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define @@ -0,0 +1,4 @@ +org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentDataDefine +org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine +org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceDataDefine +org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameDataDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define new file mode 100644 index 000000000..dd2e4e545 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define @@ -0,0 +1,10 @@ +org.skywalking.apm.collector.agentstream.worker.register.application.dao.ApplicationEsDAO +org.skywalking.apm.collector.agentstream.worker.register.instance.dao.InstanceEsDAO +org.skywalking.apm.collector.agentstream.worker.register.servicename.dao.ServiceNameEsDAO +org.skywalking.apm.collector.agentstream.worker.node.component.dao.NodeComponentEsDAO +org.skywalking.apm.collector.agentstream.worker.node.mapping.dao.NodeMappingEsDAO +org.skywalking.apm.collector.agentstream.worker.noderef.reference.dao.NodeReferenceEsDAO +org.skywalking.apm.collector.agentstream.worker.segment.origin.dao.SegmentEsDAO +org.skywalking.apm.collector.agentstream.worker.noderef.summary.dao.NodeRefSumEsDAO +org.skywalking.apm.collector.agentstream.worker.segment.cost.dao.SegmentCostEsDAO +org.skywalking.apm.collector.agentstream.worker.global.dao.GlobalTraceEsDAO \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/group.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/group.define new file mode 100644 index 000000000..2c200d484 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/group.define @@ -0,0 +1 @@ +org.skywalking.apm.collector.agentstream.AgentStreamModuleGroupDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define new file mode 100644 index 000000000..da2352e37 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define @@ -0,0 +1,10 @@ +org.skywalking.apm.collector.agentstream.worker.register.application.dao.ApplicationH2DAO +org.skywalking.apm.collector.agentstream.worker.register.instance.dao.InstanceH2DAO +org.skywalking.apm.collector.agentstream.worker.register.servicename.dao.ServiceNameH2DAO +org.skywalking.apm.collector.agentstream.worker.node.component.dao.NodeComponentH2DAO +org.skywalking.apm.collector.agentstream.worker.node.mapping.dao.NodeMappingH2DAO +org.skywalking.apm.collector.agentstream.worker.noderef.reference.dao.NodeReferenceH2DAO +org.skywalking.apm.collector.agentstream.worker.segment.origin.dao.SegmentH2DAO +org.skywalking.apm.collector.agentstream.worker.noderef.summary.dao.NodeRefSumH2DAO +org.skywalking.apm.collector.agentstream.worker.segment.cost.dao.SegmentCostH2DAO +org.skywalking.apm.collector.agentstream.worker.global.dao.GlobalTraceH2DAO \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define new file mode 100644 index 000000000..fb133fbe0 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define @@ -0,0 +1,19 @@ +org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentAggregationWorker$Factory +org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentPersistenceWorker$Factory + +org.skywalking.apm.collector.agentstream.worker.node.mapping.NodeMappingAggregationWorker$Factory +org.skywalking.apm.collector.agentstream.worker.node.mapping.NodeMappingPersistenceWorker$Factory + +org.skywalking.apm.collector.agentstream.worker.noderef.reference.NodeRefAggregationWorker$Factory +org.skywalking.apm.collector.agentstream.worker.noderef.reference.NodeRefPersistenceWorker$Factory + +org.skywalking.apm.collector.agentstream.worker.noderef.summary.NodeRefSumAggregationWorker$Factory +org.skywalking.apm.collector.agentstream.worker.noderef.summary.NodeRefSumPersistenceWorker$Factory + +org.skywalking.apm.collector.agentstream.worker.segment.origin.SegmentPersistenceWorker$Factory +org.skywalking.apm.collector.agentstream.worker.segment.cost.SegmentCostPersistenceWorker$Factory +org.skywalking.apm.collector.agentstream.worker.global.GlobalTracePersistenceWorker$Factory + +org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterSerialWorker$Factory +org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceRegisterSerialWorker$Factory +org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameRegisterSerialWorker$Factory \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/module.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/module.define index 6891cf0e2..7103f4219 100644 --- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/module.define +++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/module.define @@ -1,2 +1,2 @@ -org.skywalking.apm.collector.agent.stream.server.grpc.AgentStreamGRPCModuleDefine -org.skywalking.apm.collector.agent.stream.server.jetty.AgentStreamJettyModuleDefine \ No newline at end of file +org.skywalking.apm.collector.agentstream.grpc.AgentStreamGRPCModuleDefine +org.skywalking.apm.collector.agentstream.jetty.AgentStreamJettyModuleDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define new file mode 100644 index 000000000..b82aef6bd --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define @@ -0,0 +1,8 @@ +org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterRemoteWorker$Factory +org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceRegisterRemoteWorker$Factory +org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameRegisterRemoteWorker$Factory + +org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentRemoteWorker$Factory +org.skywalking.apm.collector.agentstream.worker.node.mapping.NodeMappingRemoteWorker$Factory +org.skywalking.apm.collector.agentstream.worker.noderef.reference.NodeRefRemoteWorker$Factory +org.skywalking.apm.collector.agentstream.worker.noderef.summary.NodeRefSumRemoteWorker$Factory \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define new file mode 100644 index 000000000..d046f4613 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define @@ -0,0 +1,29 @@ +org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentEsTableDefine +org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentH2TableDefine + +org.skywalking.apm.collector.agentstream.worker.node.mapping.define.NodeMappingEsTableDefine +org.skywalking.apm.collector.agentstream.worker.node.mapping.define.NodeMappingH2TableDefine + +org.skywalking.apm.collector.agentstream.worker.noderef.reference.define.NodeRefEsTableDefine +org.skywalking.apm.collector.agentstream.worker.noderef.reference.define.NodeRefH2TableDefine + +org.skywalking.apm.collector.agentstream.worker.noderef.summary.define.NodeRefSumEsTableDefine +org.skywalking.apm.collector.agentstream.worker.noderef.summary.define.NodeRefSumH2TableDefine + +org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationEsTableDefine +org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationH2TableDefine + +org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceEsTableDefine +org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceH2TableDefine + +org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameEsTableDefine +org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameH2TableDefine + +org.skywalking.apm.collector.agentstream.worker.segment.origin.define.SegmentEsTableDefine +org.skywalking.apm.collector.agentstream.worker.segment.origin.define.SegmentH2TableDefine + +org.skywalking.apm.collector.agentstream.worker.segment.cost.define.SegmentCostEsTableDefine +org.skywalking.apm.collector.agentstream.worker.segment.cost.define.SegmentCostH2TableDefine + +org.skywalking.apm.collector.agentstream.worker.global.define.GlobalTraceEsTableDefine +org.skywalking.apm.collector.agentstream.worker.global.define.GlobalTraceH2TableDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandlerTestCase.java b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandlerTestCase.java new file mode 100644 index 000000000..fc988f521 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandlerTestCase.java @@ -0,0 +1,116 @@ +package org.skywalking.apm.collector.agentstream.grpc.handler; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.stub.StreamObserver; +import org.skywalking.apm.network.proto.Downstream; +import org.skywalking.apm.network.proto.KeyWithStringValue; +import org.skywalking.apm.network.proto.LogMessage; +import org.skywalking.apm.network.proto.SpanLayer; +import org.skywalking.apm.network.proto.SpanObject; +import org.skywalking.apm.network.proto.SpanType; +import org.skywalking.apm.network.proto.TraceSegmentObject; +import org.skywalking.apm.network.proto.TraceSegmentReference; +import org.skywalking.apm.network.proto.TraceSegmentServiceGrpc; +import org.skywalking.apm.network.proto.UniqueId; +import org.skywalking.apm.network.proto.UpstreamSegment; +import org.skywalking.apm.network.trace.component.ComponentsDefine; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class TraceSegmentServiceHandlerTestCase { + + private final Logger logger = LoggerFactory.getLogger(TraceSegmentServiceHandlerTestCase.class); + + private TraceSegmentServiceGrpc.TraceSegmentServiceStub stub; + + public void testCollect() { + ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", 11800).usePlaintext(true).build(); + stub = TraceSegmentServiceGrpc.newStub(channel); + + StreamObserver streamObserver = stub.collect(new StreamObserver() { + @Override public void onNext(Downstream downstream) { + } + + @Override public void onError(Throwable throwable) { + logger.error(throwable.getMessage(), throwable); + } + + @Override public void onCompleted() { + + } + }); + + UpstreamSegment.Builder builder = UpstreamSegment.newBuilder(); + buildGlobalTraceIds(builder); + buildSegment(builder); + + streamObserver.onNext(builder.build()); + streamObserver.onCompleted(); + + try { + Thread.sleep(30000); + } catch (InterruptedException e) { + } + } + + private void buildGlobalTraceIds(UpstreamSegment.Builder builder) { + UniqueId.Builder builder1 = UniqueId.newBuilder(); + builder1.addIdParts(100); + builder1.addIdParts(100); + builder1.addIdParts(100); + builder.addGlobalTraceIds(builder1.build()); + } + + private void buildSegment(UpstreamSegment.Builder builder) { + long now = System.currentTimeMillis(); + + TraceSegmentObject.Builder segmentBuilder = TraceSegmentObject.newBuilder(); + segmentBuilder.setApplicationId(2); + segmentBuilder.setApplicationInstanceId(2); + segmentBuilder.setTraceSegmentId(UniqueId.newBuilder().addIdParts(200).addIdParts(200).addIdParts(200).build()); + + SpanObject.Builder span_0 = SpanObject.newBuilder(); + span_0.setSpanId(0); + span_0.setOperationName("/dubbox-case/case/dubbox-rest"); + span_0.setOperationNameId(0); + span_0.setParentSpanId(-1); + span_0.setSpanLayer(SpanLayer.Http); + span_0.setStartTime(now); + span_0.setEndTime(now + 100000); + span_0.setComponentId(ComponentsDefine.TOMCAT.getId()); + span_0.setIsError(false); + span_0.setSpanType(SpanType.Entry); + span_0.setPeerId(2); + span_0.setPeer("localhost:8082"); + + LogMessage.Builder log_0 = LogMessage.newBuilder(); + log_0.setTime(now); + log_0.addData(KeyWithStringValue.newBuilder().setKey("log1").setValue("value1")); + log_0.addData(KeyWithStringValue.newBuilder().setKey("log2").setValue("value2")); + log_0.addData(KeyWithStringValue.newBuilder().setKey("log3").setValue("value3")); + span_0.addLogs(log_0.build()); + + span_0.addTags(KeyWithStringValue.newBuilder().setKey("tag1").setValue("value1")); + span_0.addTags(KeyWithStringValue.newBuilder().setKey("tag2").setValue("value2")); + span_0.addTags(KeyWithStringValue.newBuilder().setKey("tag3").setValue("value3")); + segmentBuilder.addSpans(span_0); + + TraceSegmentReference.Builder ref_0 = TraceSegmentReference.newBuilder(); + ref_0.setEntryServiceId(1); + ref_0.setEntryServiceName("ServiceName"); + ref_0.setNetworkAddress("localhost:8081"); + ref_0.setNetworkAddressId(1); + ref_0.setParentApplicationInstanceId(1); + ref_0.setParentServiceId(1); + ref_0.setParentServiceName(""); + ref_0.setParentSpanId(2); + ref_0.setParentTraceSegmentId(UniqueId.newBuilder().addIdParts(100).addIdParts(100).addIdParts(100).build()); + segmentBuilder.addRefs(ref_0); + + builder.setSegment(segmentBuilder.build().toByteString()); + } +} diff --git a/apm-collector/apm-collector-boot/pom.xml b/apm-collector/apm-collector-boot/pom.xml index 2c66d9874..8f089d54a 100644 --- a/apm-collector/apm-collector-boot/pom.xml +++ b/apm-collector/apm-collector-boot/pom.xml @@ -23,15 +23,35 @@ apm-collector-queue ${project.version} + + org.skywalking + apm-collector-storage + ${project.version} + org.skywalking apm-collector-ui ${project.version} + + org.skywalking + apm-collector-agentserver + ${project.version} + org.skywalking apm-collector-agentstream ${project.version} + + org.skywalking + apm-collector-agentregister + ${project.version} + + + org.skywalking + apm-collector-agentjvm + ${project.version} + \ No newline at end of file diff --git a/apm-collector/apm-collector-boot/src/main/java/org/skywalking/apm/collector/boot/CollectorBootStartUp.java b/apm-collector/apm-collector-boot/src/main/java/org/skywalking/apm/collector/boot/CollectorBootStartUp.java index 46acc6cc5..ef2dc90df 100644 --- a/apm-collector/apm-collector-boot/src/main/java/org/skywalking/apm/collector/boot/CollectorBootStartUp.java +++ b/apm-collector/apm-collector-boot/src/main/java/org/skywalking/apm/collector/boot/CollectorBootStartUp.java @@ -2,7 +2,6 @@ package org.skywalking.apm.collector.boot; import org.skywalking.apm.collector.core.client.ClientException; import org.skywalking.apm.collector.core.config.ConfigException; -import org.skywalking.apm.collector.core.framework.CollectorStarter; import org.skywalking.apm.collector.core.framework.DefineException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/CollectorStarter.java b/apm-collector/apm-collector-boot/src/main/java/org/skywalking/apm/collector/boot/CollectorStarter.java similarity index 50% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/CollectorStarter.java rename to apm-collector/apm-collector-boot/src/main/java/org/skywalking/apm/collector/boot/CollectorStarter.java index 44534172d..74d041c27 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/CollectorStarter.java +++ b/apm-collector/apm-collector-boot/src/main/java/org/skywalking/apm/collector/boot/CollectorStarter.java @@ -1,16 +1,18 @@ -package org.skywalking.apm.collector.core.framework; +package org.skywalking.apm.collector.boot; import java.util.Map; +import org.skywalking.apm.collector.cluster.ClusterModuleGroupDefine; import org.skywalking.apm.collector.core.client.ClientException; import org.skywalking.apm.collector.core.config.ConfigException; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.framework.Starter; import org.skywalking.apm.collector.core.module.ModuleConfigLoader; import org.skywalking.apm.collector.core.module.ModuleDefine; import org.skywalking.apm.collector.core.module.ModuleDefineLoader; -import org.skywalking.apm.collector.core.module.ModuleGroup; import org.skywalking.apm.collector.core.module.ModuleGroupDefine; import org.skywalking.apm.collector.core.module.ModuleGroupDefineLoader; -import org.skywalking.apm.collector.core.module.ModuleInstallerAdapter; -import org.skywalking.apm.collector.core.remote.SerializedDefineLoader; +import org.skywalking.apm.collector.core.server.ServerException; +import org.skywalking.apm.collector.core.server.ServerHolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,29 +24,29 @@ public class CollectorStarter implements Starter { private final Logger logger = LoggerFactory.getLogger(CollectorStarter.class); @Override public void start() throws ConfigException, DefineException, ClientException { - Context context = new Context(); ModuleConfigLoader configLoader = new ModuleConfigLoader(); Map configuration = configLoader.load(); - SerializedDefineLoader serializedDefineLoader = new SerializedDefineLoader(); - serializedDefineLoader.load(); - - ModuleDefineLoader defineLoader = new ModuleDefineLoader(); - Map> moduleDefineMap = defineLoader.load(); - ModuleGroupDefineLoader groupDefineLoader = new ModuleGroupDefineLoader(); Map moduleGroupDefineMap = groupDefineLoader.load(); - ModuleInstallerAdapter moduleInstallerAdapter = new ModuleInstallerAdapter(ModuleGroup.Cluster); - moduleInstallerAdapter.install(configuration.get(ModuleGroup.Cluster.name().toLowerCase()), moduleDefineMap.get(ModuleGroup.Cluster.name().toLowerCase())); + ModuleDefineLoader defineLoader = new ModuleDefineLoader(); + Map> moduleDefineMap = defineLoader.load(); - ModuleGroup[] moduleGroups = ModuleGroup.values(); - for (ModuleGroup moduleGroup : moduleGroups) { - if (!ModuleGroup.Cluster.equals(moduleGroup)) { - moduleInstallerAdapter = new ModuleInstallerAdapter(moduleGroup); - logger.info("module group {}, configuration {}", moduleGroup.name().toLowerCase(), configuration.get(moduleGroup.name().toLowerCase())); - moduleInstallerAdapter.install(configuration.get(moduleGroup.name().toLowerCase()), moduleDefineMap.get(moduleGroup.name().toLowerCase())); - } + ServerHolder serverHolder = new ServerHolder(); + moduleGroupDefineMap.get(ClusterModuleGroupDefine.GROUP_NAME).moduleInstaller().install(configuration.get(ClusterModuleGroupDefine.GROUP_NAME), moduleDefineMap.get(ClusterModuleGroupDefine.GROUP_NAME), serverHolder); + moduleGroupDefineMap.remove(ClusterModuleGroupDefine.GROUP_NAME); + + for (ModuleGroupDefine moduleGroupDefine : moduleGroupDefineMap.values()) { + moduleGroupDefine.moduleInstaller().install(configuration.get(moduleGroupDefine.name()), moduleDefineMap.get(moduleGroupDefine.name()), serverHolder); } + + serverHolder.getServers().forEach(server -> { + try { + server.start(); + } catch (ServerException e) { + logger.error(e.getMessage(), e); + } + }); } } diff --git a/apm-collector/apm-collector-boot/src/main/resources/application.yml b/apm-collector/apm-collector-boot/src/main/resources/application.yml index 7406f217b..1a84f5740 100644 --- a/apm-collector/apm-collector-boot/src/main/resources/application.yml +++ b/apm-collector/apm-collector-boot/src/main/resources/application.yml @@ -1,13 +1,10 @@ cluster: -# zookeeper: -# hostPort: localhost:2181 -# sessionTimeout: 1000 + zookeeper: + hostPort: localhost:2181 + sessionTimeout: 100000 # redis: # host: localhost # port: 6379 -queue: - disruptor: on - data_carrier: off agentstream: grpc: host: localhost @@ -23,4 +20,11 @@ ui: jetty: host: localhost port: 12800 + context_path: / +storage: + elasticsearch: + cluster_name: CollectorDBCluster + cluster_transport_sniffer: true + cluster_nodes: 127.0.0.1:9300 + diff --git a/apm-collector/apm-collector-client/pom.xml b/apm-collector/apm-collector-client/pom.xml index 1de50be53..1f6b17431 100644 --- a/apm-collector/apm-collector-client/pom.xml +++ b/apm-collector/apm-collector-client/pom.xml @@ -28,10 +28,36 @@ jedis 2.9.0 + + org.elasticsearch.client + transport + 5.5.0 + + + snakeyaml + org.yaml + + + org.apache.zookeeper zookeeper 3.4.10 + + + slf4j-api + org.slf4j + + + slf4j-log4j12 + org.slf4j + + + + + io.grpc + grpc-core + 1.4.0 \ No newline at end of file diff --git a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java new file mode 100644 index 000000000..58cf145ef --- /dev/null +++ b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java @@ -0,0 +1,132 @@ +package org.skywalking.apm.collector.client.elasticsearch; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.get.GetRequestBuilder; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.IndicesAdminClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.transport.client.PreBuiltTransportClient; +import org.skywalking.apm.collector.core.client.Client; +import org.skywalking.apm.collector.core.client.ClientException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class ElasticSearchClient implements Client { + + private final Logger logger = LoggerFactory.getLogger(ElasticSearchClient.class); + + private org.elasticsearch.client.Client client; + + private final String clusterName; + + private final Boolean clusterTransportSniffer; + + private final String clusterNodes; + + public ElasticSearchClient(String clusterName, Boolean clusterTransportSniffer, String clusterNodes) { + this.clusterName = clusterName; + this.clusterTransportSniffer = clusterTransportSniffer; + this.clusterNodes = clusterNodes; + } + + @Override public void initialize() throws ClientException { + Settings settings = Settings.builder() + .put("cluster.name", clusterName) + .put("client.transport.sniff", clusterTransportSniffer) + .build(); + + client = new PreBuiltTransportClient(settings); + + List pairsList = parseClusterNodes(clusterNodes); + for (AddressPairs pairs : pairsList) { + try { + ((PreBuiltTransportClient)client).addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(pairs.host), pairs.port)); + } catch (UnknownHostException e) { + throw new ElasticSearchClientException(e.getMessage(), e); + } + } + } + + private List parseClusterNodes(String nodes) { + List pairsList = new LinkedList<>(); + logger.info("elasticsearch cluster nodes: {}", nodes); + String[] nodesSplit = nodes.split(","); + for (int i = 0; i < nodesSplit.length; i++) { + String node = nodesSplit[i]; + String host = node.split(":")[0]; + String port = node.split(":")[1]; + pairsList.add(new AddressPairs(host, Integer.valueOf(port))); + } + + return pairsList; + } + + class AddressPairs { + private String host; + private Integer port; + + public AddressPairs(String host, Integer port) { + this.host = host; + this.port = port; + } + } + + public boolean createIndex(String indexName, String indexType, Settings settings, XContentBuilder mappingBuilder) { + IndicesAdminClient adminClient = client.admin().indices(); + CreateIndexResponse response = adminClient.prepareCreate(indexName).setSettings(settings).addMapping(indexType, mappingBuilder).get(); + logger.info("create {} index with type of {} finished, isAcknowledged: {}", indexName, indexType, response.isAcknowledged()); + return response.isShardsAcked(); + } + + public boolean deleteIndex(String indexName) { + IndicesAdminClient adminClient = client.admin().indices(); + DeleteIndexResponse response = adminClient.prepareDelete(indexName).get(); + logger.info("delete {} index finished, isAcknowledged: {}", indexName, response.isAcknowledged()); + return response.isAcknowledged(); + } + + public boolean isExistsIndex(String indexName) { + IndicesAdminClient adminClient = client.admin().indices(); + IndicesExistsResponse response = adminClient.prepareExists(indexName).get(); + return response.isExists(); + } + + public SearchRequestBuilder prepareSearch(String indexName) { + return client.prepareSearch(indexName); + } + + public IndexRequestBuilder prepareIndex(String indexName, String id) { + return client.prepareIndex(indexName, "type", id); + } + + public GetRequestBuilder prepareGet(String indexName, String id) { + return client.prepareGet(indexName, "type", id); + } + + public BulkRequestBuilder prepareBulk() { + return client.prepareBulk(); + } + + public void update(UpdateRequest updateRequest) { + try { + client.update(updateRequest).get(); + } catch (InterruptedException | ExecutionException e) { + logger.error(e.getMessage(), e); + } + } +} diff --git a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClientException.java b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClientException.java new file mode 100644 index 000000000..02fa609b7 --- /dev/null +++ b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClientException.java @@ -0,0 +1,16 @@ +package org.skywalking.apm.collector.client.elasticsearch; + +import org.skywalking.apm.collector.core.client.ClientException; + +/** + * @author pengys5 + */ +public class ElasticSearchClientException extends ClientException { + public ElasticSearchClientException(String message) { + super(message); + } + + public ElasticSearchClientException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/grpc/GRPCClient.java b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/grpc/GRPCClient.java new file mode 100644 index 000000000..8788a5472 --- /dev/null +++ b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/grpc/GRPCClient.java @@ -0,0 +1,35 @@ +package org.skywalking.apm.collector.client.grpc; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import org.skywalking.apm.collector.core.client.Client; +import org.skywalking.apm.collector.core.client.ClientException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class GRPCClient implements Client { + + private final Logger logger = LoggerFactory.getLogger(GRPCClient.class); + + private final String host; + + private final int port; + + private ManagedChannel channel; + + public GRPCClient(String host, int port) { + this.host = host; + this.port = port; + } + + @Override public void initialize() throws ClientException { + channel = ManagedChannelBuilder.forAddress(host, port).usePlaintext(true).build(); + } + + public ManagedChannel getChannel() { + return channel; + } +} diff --git a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/grpc/GRPCClientException.java b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/grpc/GRPCClientException.java new file mode 100644 index 000000000..ec2a018c6 --- /dev/null +++ b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/grpc/GRPCClientException.java @@ -0,0 +1,17 @@ +package org.skywalking.apm.collector.client.grpc; + +import org.skywalking.apm.collector.core.client.ClientException; + +/** + * @author pengys5 + */ +public class GRPCClientException extends ClientException { + + public GRPCClientException(String message) { + super(message); + } + + public GRPCClientException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/zookeeper/ZookeeperClient.java b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/zookeeper/ZookeeperClient.java index 8191e1da7..89dc9174a 100644 --- a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/zookeeper/ZookeeperClient.java +++ b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/zookeeper/ZookeeperClient.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.util.List; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Stat; @@ -22,15 +23,17 @@ public class ZookeeperClient implements Client { private final String hostPort; private final int sessionTimeout; + private final Watcher watcher; - public ZookeeperClient(String hostPort, int sessionTimeout) { + public ZookeeperClient(String hostPort, int sessionTimeout, Watcher watcher) { this.hostPort = hostPort; this.sessionTimeout = sessionTimeout; + this.watcher = watcher; } @Override public void initialize() throws ZookeeperClientException { try { - zk = new ZooKeeper(hostPort, sessionTimeout, new ZookeeperDataListener(this)); + zk = new ZooKeeper(hostPort, sessionTimeout, watcher); } catch (IOException e) { throw new ZookeeperClientException(e.getMessage(), e); } @@ -68,4 +71,12 @@ public class ZookeeperClient implements Client { throw new ZookeeperClientException(e.getMessage(), e); } } + + public List getChildren(final String path, boolean watch) throws ZookeeperClientException { + try { + return zk.getChildren(path, watch); + } catch (KeeperException | InterruptedException e) { + throw new ZookeeperClientException(e.getMessage(), e); + } + } } diff --git a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/zookeeper/ZookeeperDataListener.java b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/zookeeper/ZookeeperDataListener.java deleted file mode 100644 index cd5ff1e19..000000000 --- a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/zookeeper/ZookeeperDataListener.java +++ /dev/null @@ -1,59 +0,0 @@ -package org.skywalking.apm.collector.client.zookeeper; - -import java.util.LinkedList; -import java.util.List; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.skywalking.apm.collector.core.client.Client; -import org.skywalking.apm.collector.core.client.ClientException; -import org.skywalking.apm.collector.core.client.DataListener; -import org.skywalking.apm.collector.core.cluster.ClusterDataInitializer; -import org.skywalking.apm.collector.core.util.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author pengys5 - */ -public class ZookeeperDataListener implements DataListener, Watcher { - - private final Logger logger = LoggerFactory.getLogger(ZookeeperDataListener.class); - - private ZookeeperClient client; - - public ZookeeperDataListener(Client client) { - this.client = (ZookeeperClient)client; - } - - @Override public void process(WatchedEvent event) { - logger.debug("path {}", event.getPath()); - if (StringUtils.isEmpty(event.getPath())) { - return; - } - - try { - String data = String.valueOf(client.getData(event.getPath(), false, null)); - logger.debug("data {}", data); - } catch (ClientException e) { - logger.error(e.getMessage(), e); - } - } - - @Override public void listen() throws ClientException { - for (String itemKey : items()) { - String[] catalogs = itemKey.split("\\."); - StringBuilder pathBuilder = new StringBuilder(); - for (String catalog : catalogs) { - pathBuilder.append("/").append(catalog); - } - client.exists(pathBuilder.toString(), true); - } - } - - @Override public List items() { - List items = new LinkedList<>(); - items.add(ClusterDataInitializer.FOR_AGENT_CATALOG); - items.add(ClusterDataInitializer.FOR_UI_CATALOG); - return items; - } -} diff --git a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/ClusterModuleDefine.java b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/ClusterModuleDefine.java new file mode 100644 index 000000000..900cf9b3d --- /dev/null +++ b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/ClusterModuleDefine.java @@ -0,0 +1,59 @@ +package org.skywalking.apm.collector.cluster; + +import java.util.Map; +import org.skywalking.apm.collector.core.client.Client; +import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.client.DataMonitor; +import org.skywalking.apm.collector.core.cluster.ClusterModuleContext; +import org.skywalking.apm.collector.core.cluster.ClusterModuleException; +import org.skywalking.apm.collector.core.cluster.ClusterModuleRegistrationReader; +import org.skywalking.apm.collector.core.config.ConfigParseException; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.core.module.ModuleDefine; +import org.skywalking.apm.collector.core.module.ModuleRegistration; +import org.skywalking.apm.collector.core.server.Server; +import org.skywalking.apm.collector.core.server.ServerHolder; + +/** + * @author pengys5 + */ +public abstract class ClusterModuleDefine extends ModuleDefine { + + public static final String BASE_CATALOG = "skywalking"; + + private Client client; + + @Override public final void initialize(Map config, ServerHolder serverHolder) throws ClusterModuleException { + try { + configParser().parse(config); + + DataMonitor dataMonitor = dataMonitor(); + client = createClient(dataMonitor); + client.initialize(); + dataMonitor.setClient(client); + + ClusterModuleRegistrationReader reader = registrationReader(dataMonitor); + + ((ClusterModuleContext)CollectorContextHelper.INSTANCE.getContext(group())).setDataMonitor(dataMonitor); + ((ClusterModuleContext)CollectorContextHelper.INSTANCE.getContext(group())).setReader(reader); + } catch (ConfigParseException | ClientException e) { + throw new ClusterModuleException(e.getMessage(), e); + } + } + + public final Client getClient() { + return this.client; + } + + @Override public final Server server() { + throw new UnsupportedOperationException(""); + } + + @Override protected final ModuleRegistration registration() { + throw new UnsupportedOperationException("Cluster module do not need module registration."); + } + + public abstract DataMonitor dataMonitor(); + + public abstract ClusterModuleRegistrationReader registrationReader(DataMonitor dataMonitor); +} diff --git a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/ClusterModuleGroupDefine.java b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/ClusterModuleGroupDefine.java index 4f5246442..d51023d8a 100644 --- a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/ClusterModuleGroupDefine.java +++ b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/ClusterModuleGroupDefine.java @@ -1,17 +1,26 @@ package org.skywalking.apm.collector.cluster; +import org.skywalking.apm.collector.core.cluster.ClusterModuleContext; +import org.skywalking.apm.collector.core.framework.Context; import org.skywalking.apm.collector.core.module.ModuleGroupDefine; -import org.skywalking.apm.collector.core.module.ModuleInstallMode; +import org.skywalking.apm.collector.core.module.ModuleInstaller; /** * @author pengys5 */ public class ClusterModuleGroupDefine implements ModuleGroupDefine { + + public static final String GROUP_NAME = "cluster"; + @Override public String name() { - return "cluster"; + return GROUP_NAME; } - @Override public ModuleInstallMode mode() { - return ModuleInstallMode.Single; + @Override public Context groupContext() { + return new ClusterModuleContext(GROUP_NAME); + } + + @Override public ModuleInstaller moduleInstaller() { + return new ClusterModuleInstaller(); } } diff --git a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/ClusterModuleInstaller.java b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/ClusterModuleInstaller.java new file mode 100644 index 000000000..f8992796c --- /dev/null +++ b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/ClusterModuleInstaller.java @@ -0,0 +1,30 @@ +package org.skywalking.apm.collector.cluster; + +import java.util.Map; +import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.cluster.ClusterModuleContext; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.module.ModuleDefine; +import org.skywalking.apm.collector.core.module.SingleModuleInstaller; +import org.skywalking.apm.collector.core.server.ServerHolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class ClusterModuleInstaller extends SingleModuleInstaller { + + private final Logger logger = LoggerFactory.getLogger(ClusterModuleInstaller.class); + + @Override public void install(Map moduleConfig, + Map moduleDefineMap, ServerHolder serverHolder) throws DefineException, ClientException { + logger.info("beginning cluster module install"); + + ClusterModuleContext context = new ClusterModuleContext(ClusterModuleGroupDefine.GROUP_NAME); + CollectorContextHelper.INSTANCE.putContext(context); + + installSingle(moduleConfig, moduleDefineMap, serverHolder); + } +} diff --git a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/redis/ClusterRedisDataInitializer.java b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/redis/ClusterRedisDataInitializer.java deleted file mode 100644 index 9ea528867..000000000 --- a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/redis/ClusterRedisDataInitializer.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.skywalking.apm.collector.cluster.redis; - -import org.skywalking.apm.collector.core.client.Client; -import org.skywalking.apm.collector.core.client.ClientException; -import org.skywalking.apm.collector.core.cluster.ClusterDataInitializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author pengys5 - */ -public class ClusterRedisDataInitializer extends ClusterDataInitializer { - - private final Logger logger = LoggerFactory.getLogger(ClusterRedisDataInitializer.class); - - @Override public void addItem(Client client, String itemKey) throws ClientException { - logger.info("add the redis item key \"{}\" exist", itemKey); - } - - @Override public boolean existItem(Client client, String itemKey) throws ClientException { - logger.info("assess the redis item key \"{}\" exist", itemKey); - return false; - } -} diff --git a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/redis/ClusterRedisModuleDefine.java b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/redis/ClusterRedisModuleDefine.java index cc7e9c71f..b946cd709 100644 --- a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/redis/ClusterRedisModuleDefine.java +++ b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/redis/ClusterRedisModuleDefine.java @@ -1,25 +1,26 @@ package org.skywalking.apm.collector.cluster.redis; import org.skywalking.apm.collector.client.redis.RedisClient; +import org.skywalking.apm.collector.cluster.ClusterModuleDefine; +import org.skywalking.apm.collector.cluster.ClusterModuleGroupDefine; import org.skywalking.apm.collector.core.client.Client; -import org.skywalking.apm.collector.core.cluster.ClusterModuleDefine; +import org.skywalking.apm.collector.core.client.DataMonitor; import org.skywalking.apm.collector.core.cluster.ClusterModuleRegistrationReader; -import org.skywalking.apm.collector.core.cluster.ClusterModuleRegistrationWriter; -import org.skywalking.apm.collector.core.framework.DataInitializer; import org.skywalking.apm.collector.core.module.ModuleConfigParser; -import org.skywalking.apm.collector.core.module.ModuleGroup; /** * @author pengys5 */ public class ClusterRedisModuleDefine extends ClusterModuleDefine { - @Override public ModuleGroup group() { - return ModuleGroup.Cluster; + public static final String MODULE_NAME = "redis"; + + @Override public String group() { + return ClusterModuleGroupDefine.GROUP_NAME; } @Override public String name() { - return "redis"; + return MODULE_NAME; } @Override public boolean defaultModule() { @@ -30,19 +31,15 @@ public class ClusterRedisModuleDefine extends ClusterModuleDefine { return new ClusterRedisConfigParser(); } - @Override protected Client createClient() { + @Override public DataMonitor dataMonitor() { + return null; + } + + @Override protected Client createClient(DataMonitor dataMonitor) { return new RedisClient(ClusterRedisConfig.HOST, ClusterRedisConfig.PORT); } - @Override protected DataInitializer dataInitializer() { - return new ClusterRedisDataInitializer(); - } - - @Override protected ClusterModuleRegistrationWriter registrationWriter() { - return new ClusterRedisModuleRegistrationWriter(getClient()); - } - - @Override protected ClusterModuleRegistrationReader registrationReader() { - return null; + @Override public ClusterModuleRegistrationReader registrationReader(DataMonitor dataMonitor) { + return new ClusterRedisModuleRegistrationReader(dataMonitor); } } diff --git a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/redis/ClusterRedisModuleRegistrationReader.java b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/redis/ClusterRedisModuleRegistrationReader.java new file mode 100644 index 000000000..6f7672323 --- /dev/null +++ b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/redis/ClusterRedisModuleRegistrationReader.java @@ -0,0 +1,14 @@ +package org.skywalking.apm.collector.cluster.redis; + +import org.skywalking.apm.collector.core.client.DataMonitor; +import org.skywalking.apm.collector.core.cluster.ClusterModuleRegistrationReader; + +/** + * @author pengys5 + */ +public class ClusterRedisModuleRegistrationReader extends ClusterModuleRegistrationReader { + + public ClusterRedisModuleRegistrationReader(DataMonitor dataMonitor) { + super(dataMonitor); + } +} diff --git a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/redis/ClusterRedisModuleRegistrationWriter.java b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/redis/ClusterRedisModuleRegistrationWriter.java deleted file mode 100644 index f94a614fa..000000000 --- a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/redis/ClusterRedisModuleRegistrationWriter.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.skywalking.apm.collector.cluster.redis; - -import org.skywalking.apm.collector.client.redis.RedisClient; -import org.skywalking.apm.collector.core.client.Client; -import org.skywalking.apm.collector.core.cluster.ClusterModuleRegistrationWriter; -import org.skywalking.apm.collector.core.module.ModuleRegistration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author pengys5 - */ -public class ClusterRedisModuleRegistrationWriter extends ClusterModuleRegistrationWriter { - - private final Logger logger = LoggerFactory.getLogger(ClusterRedisModuleRegistrationWriter.class); - - public ClusterRedisModuleRegistrationWriter(Client client) { - super(client); - } - - @Override public void write(String key, ModuleRegistration.Value value) { - logger.debug("key {}, value {}", key, value.getHost()); - key = key + "." + value.getHost() + ":" + value.getPort(); - value.getData().addProperty("host", value.getHost()); - value.getData().addProperty("port", value.getPort()); - ((RedisClient)client).setex(key, 120, value.getData().toString()); - } -} diff --git a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/standalone/ClusterStandaloneDataInitializer.java b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/standalone/ClusterStandaloneDataInitializer.java deleted file mode 100644 index 6221ec2bb..000000000 --- a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/standalone/ClusterStandaloneDataInitializer.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.skywalking.apm.collector.cluster.standalone; - -import org.skywalking.apm.collector.client.h2.H2Client; -import org.skywalking.apm.collector.client.h2.H2ClientException; -import org.skywalking.apm.collector.core.client.Client; -import org.skywalking.apm.collector.core.client.ClientException; -import org.skywalking.apm.collector.core.cluster.ClusterDataInitializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author pengys5 - */ -public class ClusterStandaloneDataInitializer extends ClusterDataInitializer { - - private final Logger logger = LoggerFactory.getLogger(ClusterStandaloneDataInitializer.class); - - @Override public void addItem(Client client, String itemKey) throws ClientException { - logger.info("add the h2 item key \"{}\" exist", itemKey); - itemKey = itemKey.replaceAll("\\.", "_"); - String sql = "CREATE TABLE " + itemKey + "(ADDRESS VARCHAR(100) PRIMARY KEY,DATA VARCHAR(255));"; - try { - ((H2Client)client).execute(sql); - } catch (H2ClientException e) { - logger.error(e.getMessage(), e); - } - } - - @Override public boolean existItem(Client client, String itemKey) throws ClientException { - return false; - } -} diff --git a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/standalone/ClusterStandaloneModuleDefine.java b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/standalone/ClusterStandaloneModuleDefine.java index d7802e216..1c95e0642 100644 --- a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/standalone/ClusterStandaloneModuleDefine.java +++ b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/standalone/ClusterStandaloneModuleDefine.java @@ -1,25 +1,26 @@ package org.skywalking.apm.collector.cluster.standalone; import org.skywalking.apm.collector.client.h2.H2Client; +import org.skywalking.apm.collector.cluster.ClusterModuleDefine; +import org.skywalking.apm.collector.cluster.ClusterModuleGroupDefine; import org.skywalking.apm.collector.core.client.Client; -import org.skywalking.apm.collector.core.cluster.ClusterModuleDefine; +import org.skywalking.apm.collector.core.client.DataMonitor; import org.skywalking.apm.collector.core.cluster.ClusterModuleRegistrationReader; -import org.skywalking.apm.collector.core.cluster.ClusterModuleRegistrationWriter; -import org.skywalking.apm.collector.core.framework.DataInitializer; import org.skywalking.apm.collector.core.module.ModuleConfigParser; -import org.skywalking.apm.collector.core.module.ModuleGroup; /** * @author pengys5 */ public class ClusterStandaloneModuleDefine extends ClusterModuleDefine { - @Override public ModuleGroup group() { - return ModuleGroup.Cluster; + public static final String MODULE_NAME = "standalone"; + + @Override public String group() { + return ClusterModuleGroupDefine.GROUP_NAME; } @Override public String name() { - return "standalone"; + return MODULE_NAME; } @Override public boolean defaultModule() { @@ -30,19 +31,15 @@ public class ClusterStandaloneModuleDefine extends ClusterModuleDefine { return new ClusterStandaloneConfigParser(); } - @Override public Client createClient() { + @Override public DataMonitor dataMonitor() { + return null; + } + + @Override protected Client createClient(DataMonitor dataMonitor) { return new H2Client(); } - @Override protected DataInitializer dataInitializer() { - return new ClusterStandaloneDataInitializer(); - } - - @Override protected ClusterModuleRegistrationWriter registrationWriter() { - return new ClusterStandaloneModuleRegistrationWriter(getClient()); - } - - @Override protected ClusterModuleRegistrationReader registrationReader() { - return null; + @Override public ClusterModuleRegistrationReader registrationReader(DataMonitor dataMonitor) { + return new ClusterStandaloneModuleRegistrationReader(dataMonitor); } } diff --git a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/standalone/ClusterStandaloneModuleRegistrationReader.java b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/standalone/ClusterStandaloneModuleRegistrationReader.java new file mode 100644 index 000000000..f08f821be --- /dev/null +++ b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/standalone/ClusterStandaloneModuleRegistrationReader.java @@ -0,0 +1,14 @@ +package org.skywalking.apm.collector.cluster.standalone; + +import org.skywalking.apm.collector.core.client.DataMonitor; +import org.skywalking.apm.collector.core.cluster.ClusterModuleRegistrationReader; + +/** + * @author pengys5 + */ +public class ClusterStandaloneModuleRegistrationReader extends ClusterModuleRegistrationReader { + + public ClusterStandaloneModuleRegistrationReader(DataMonitor dataMonitor) { + super(dataMonitor); + } +} diff --git a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/standalone/ClusterStandaloneModuleRegistrationWriter.java b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/standalone/ClusterStandaloneModuleRegistrationWriter.java deleted file mode 100644 index de811f165..000000000 --- a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/standalone/ClusterStandaloneModuleRegistrationWriter.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.skywalking.apm.collector.cluster.standalone; - -import org.skywalking.apm.collector.client.h2.H2Client; -import org.skywalking.apm.collector.client.h2.H2ClientException; -import org.skywalking.apm.collector.core.client.Client; -import org.skywalking.apm.collector.core.cluster.ClusterModuleRegistrationWriter; -import org.skywalking.apm.collector.core.module.ModuleRegistration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author pengys5 - */ -public class ClusterStandaloneModuleRegistrationWriter extends ClusterModuleRegistrationWriter { - - private final Logger logger = LoggerFactory.getLogger(ClusterStandaloneModuleRegistrationWriter.class); - - public ClusterStandaloneModuleRegistrationWriter(Client client) { - super(client); - } - - @Override public void write(String key, ModuleRegistration.Value value) { - key = key.replaceAll("\\.", "_"); - String hostPort = value.getHost() + ":" + value.getPort(); - String sql = "INSERT INTO " + key + " VALUES('" + hostPort + "', '" + value.getData().toString() + "');"; - String sql2 = "SELECT * FROM " + key; - try { - ((H2Client)client).execute(sql); - ((H2Client)client).executeQuery(sql2); - } catch (H2ClientException e) { - logger.error(e.getMessage(), e); - } - } -} diff --git a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKConfigParser.java b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKConfigParser.java index 9d0d62871..2555b61cb 100644 --- a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKConfigParser.java +++ b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKConfigParser.java @@ -15,7 +15,7 @@ public class ClusterZKConfigParser implements ModuleConfigParser { @Override public void parse(Map config) throws ConfigParseException { ClusterZKConfig.HOST_PORT = (String)config.get(HOST_PORT); - ClusterZKConfig.SESSION_TIMEOUT = 1000; + ClusterZKConfig.SESSION_TIMEOUT = 3000; if (StringUtils.isEmpty(ClusterZKConfig.HOST_PORT)) { throw new ConfigParseException(""); diff --git a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKDataInitializer.java b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKDataInitializer.java deleted file mode 100644 index 4f741245f..000000000 --- a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKDataInitializer.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.skywalking.apm.collector.cluster.zookeeper; - -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.ZooDefs; -import org.skywalking.apm.collector.client.zookeeper.ZookeeperClient; -import org.skywalking.apm.collector.core.client.Client; -import org.skywalking.apm.collector.core.client.ClientException; -import org.skywalking.apm.collector.core.cluster.ClusterDataInitializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author pengys5 - */ -public class ClusterZKDataInitializer extends ClusterDataInitializer { - - private final Logger logger = LoggerFactory.getLogger(ClusterZKDataInitializer.class); - - @Override public void addItem(Client client, String itemKey) throws ClientException { - logger.info("add the zookeeper item key \"{}\" exist", itemKey); - ZookeeperClient zkClient = (ZookeeperClient)client; - - String[] catalogs = itemKey.split("\\."); - StringBuilder pathBuilder = new StringBuilder(); - for (String catalog : catalogs) { - pathBuilder.append("/").append(catalog); - if (zkClient.exists(pathBuilder.toString(), false) == null) { - zkClient.create(pathBuilder.toString(), null, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - } - } - } - - @Override public boolean existItem(Client client, String itemKey) throws ClientException { - logger.info("assess the zookeeper item key \"{}\" exist", itemKey); - ZookeeperClient zkClient = (ZookeeperClient)client; - - String[] catalogs = itemKey.split("\\."); - StringBuilder pathBuilder = new StringBuilder(); - for (String catalog : catalogs) { - pathBuilder.append("/").append(catalog); - } - -// if (zkClient.exists(pathBuilder.toString(), false) == null) { -// return false; -// } else { - return true; -// } - } -} diff --git a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKDataMonitor.java b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKDataMonitor.java new file mode 100644 index 000000000..7e973716a --- /dev/null +++ b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKDataMonitor.java @@ -0,0 +1,104 @@ +package org.skywalking.apm.collector.cluster.zookeeper; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooDefs; +import org.skywalking.apm.collector.client.zookeeper.ZookeeperClient; +import org.skywalking.apm.collector.client.zookeeper.ZookeeperClientException; +import org.skywalking.apm.collector.client.zookeeper.util.PathUtils; +import org.skywalking.apm.collector.core.client.Client; +import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.client.DataMonitor; +import org.skywalking.apm.collector.core.cluster.ClusterDataListener; +import org.skywalking.apm.collector.core.module.ModuleRegistration; +import org.skywalking.apm.collector.core.util.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class ClusterZKDataMonitor implements DataMonitor, Watcher { + + private final Logger logger = LoggerFactory.getLogger(ClusterZKDataMonitor.class); + + private ZookeeperClient client; + + private Map listeners; + + public ClusterZKDataMonitor() { + listeners = new LinkedHashMap<>(); + } + + @Override public void process(WatchedEvent event) { + logger.debug("changed path {}", event.getPath()); + if (listeners.containsKey(event.getPath())) { + List paths = null; + try { + paths = client.getChildren(event.getPath(), true); + listeners.get(event.getPath()).clearData(); + if (CollectionUtils.isNotEmpty(paths)) { + for (String serverPath : paths) { + byte[] data = client.getData(event.getPath() + "/" + serverPath, false, null); + String dataStr = new String(data); + logger.debug("path children has been changed, path: {}, data: {}", event.getPath() + "/" + serverPath, dataStr); + listeners.get(event.getPath()).addAddress(serverPath + dataStr); + listeners.get(event.getPath()).addressChangedNotify(); + } + } + } catch (ZookeeperClientException e) { + logger.error(e.getMessage(), e); + } + } + } + + @Override public void setClient(Client client) { + this.client = (ZookeeperClient)client; + } + + @Override + public void addListener(ClusterDataListener listener, ModuleRegistration registration) throws ClientException { + String path = PathUtils.convertKey2Path(listener.path()); + logger.info("listener path: {}", path); + listeners.put(path, listener); + createPath(path); + + ModuleRegistration.Value value = registration.buildValue(); + String contextPath = value.getContextPath() == null ? "" : value.getContextPath(); + + client.getChildren(path, true); + String serverPath = path + "/" + value.getHostPort(); + listener.addAddress(value.getHostPort() + contextPath); + + setData(serverPath, contextPath); + } + + @Override public ClusterDataListener getListener(String path) { + path = PathUtils.convertKey2Path(path); + return listeners.get(path); + } + + @Override public void createPath(String path) throws ClientException { + String[] paths = path.replaceFirst("/", "").split("/"); + + StringBuilder pathBuilder = new StringBuilder(); + for (String subPath : paths) { + pathBuilder.append("/").append(subPath); + if (client.exists(pathBuilder.toString(), false) == null) { + client.create(pathBuilder.toString(), null, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + } + } + } + + @Override public void setData(String path, String value) throws ClientException { + if (client.exists(path, false) == null) { + client.create(path, value.getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL); + } else { + client.setData(path, value.getBytes(), -1); + } + } +} diff --git a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKModuleDefine.java b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKModuleDefine.java index b555ac458..c53676407 100644 --- a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKModuleDefine.java +++ b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKModuleDefine.java @@ -1,25 +1,27 @@ package org.skywalking.apm.collector.cluster.zookeeper; +import org.apache.zookeeper.Watcher; import org.skywalking.apm.collector.client.zookeeper.ZookeeperClient; +import org.skywalking.apm.collector.cluster.ClusterModuleDefine; +import org.skywalking.apm.collector.cluster.ClusterModuleGroupDefine; import org.skywalking.apm.collector.core.client.Client; -import org.skywalking.apm.collector.core.cluster.ClusterDataInitializer; -import org.skywalking.apm.collector.core.cluster.ClusterModuleDefine; +import org.skywalking.apm.collector.core.client.DataMonitor; import org.skywalking.apm.collector.core.cluster.ClusterModuleRegistrationReader; -import org.skywalking.apm.collector.core.cluster.ClusterModuleRegistrationWriter; import org.skywalking.apm.collector.core.module.ModuleConfigParser; -import org.skywalking.apm.collector.core.module.ModuleGroup; /** * @author pengys5 */ public class ClusterZKModuleDefine extends ClusterModuleDefine { - @Override protected ModuleGroup group() { - return ModuleGroup.Cluster; + public static final String MODULE_NAME = "zookeeper"; + + @Override protected String group() { + return ClusterModuleGroupDefine.GROUP_NAME; } @Override public String name() { - return "zookeeper"; + return MODULE_NAME; } @Override public boolean defaultModule() { @@ -30,19 +32,15 @@ public class ClusterZKModuleDefine extends ClusterModuleDefine { return new ClusterZKConfigParser(); } - @Override protected Client createClient() { - return new ZookeeperClient(ClusterZKConfig.HOST_PORT, ClusterZKConfig.SESSION_TIMEOUT); + @Override public DataMonitor dataMonitor() { + return new ClusterZKDataMonitor(); } - @Override protected ClusterDataInitializer dataInitializer() { - return new ClusterZKDataInitializer(); + @Override protected Client createClient(DataMonitor dataMonitor) { + return new ZookeeperClient(ClusterZKConfig.HOST_PORT, ClusterZKConfig.SESSION_TIMEOUT, (Watcher)dataMonitor); } - @Override protected ClusterModuleRegistrationWriter registrationWriter() { - return new ClusterZKModuleRegistrationWriter(getClient()); - } - - @Override protected ClusterModuleRegistrationReader registrationReader() { - return null; + @Override public ClusterModuleRegistrationReader registrationReader(DataMonitor dataMonitor) { + return new ClusterZKModuleRegistrationReader(dataMonitor); } } diff --git a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKModuleRegistrationReader.java b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKModuleRegistrationReader.java index 44dec4861..123611687 100644 --- a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKModuleRegistrationReader.java +++ b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKModuleRegistrationReader.java @@ -1,13 +1,14 @@ package org.skywalking.apm.collector.cluster.zookeeper; -import java.util.List; +import org.skywalking.apm.collector.core.client.DataMonitor; import org.skywalking.apm.collector.core.cluster.ClusterModuleRegistrationReader; /** * @author pengys5 */ -public class ClusterZKModuleRegistrationReader implements ClusterModuleRegistrationReader { - @Override public List read(String key) { - return null; +public class ClusterZKModuleRegistrationReader extends ClusterModuleRegistrationReader { + + public ClusterZKModuleRegistrationReader(DataMonitor dataMonitor) { + super(dataMonitor); } } diff --git a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKModuleRegistrationWriter.java b/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKModuleRegistrationWriter.java deleted file mode 100644 index 622379560..000000000 --- a/apm-collector/apm-collector-cluster/src/main/java/org/skywalking/apm/collector/cluster/zookeeper/ClusterZKModuleRegistrationWriter.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.skywalking.apm.collector.cluster.zookeeper; - -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.data.Stat; -import org.skywalking.apm.collector.client.zookeeper.ZookeeperClient; -import org.skywalking.apm.collector.client.zookeeper.util.PathUtils; -import org.skywalking.apm.collector.core.client.Client; -import org.skywalking.apm.collector.core.client.ClientException; -import org.skywalking.apm.collector.core.cluster.ClusterModuleRegistrationWriter; -import org.skywalking.apm.collector.core.module.ModuleRegistration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author pengys5 - */ -public class ClusterZKModuleRegistrationWriter extends ClusterModuleRegistrationWriter { - - private final Logger logger = LoggerFactory.getLogger(ClusterZKModuleRegistrationWriter.class); - - public ClusterZKModuleRegistrationWriter(Client client) { - super(client); - } - - @Override public void write(String key, ModuleRegistration.Value value) throws ClientException { - logger.info("cluster zookeeper register key: {}, value: {}", key, value); - String workerUIPath = PathUtils.convertKey2Path(key) + "/" + value.getHost() + ":" + value.getPort(); - - Stat stat = ((ZookeeperClient)client).exists(workerUIPath, false); - if (stat == null) { - ((ZookeeperClient)client).create(workerUIPath, value.getData() == null ? null : value.getData().toString().getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL); - } else { - ((ZookeeperClient)client).setData(workerUIPath, value.getData() == null ? null : value.getData().toString().getBytes(), -1); - } - } -} diff --git a/apm-collector/apm-collector-core/pom.xml b/apm-collector/apm-collector-core/pom.xml index 5c21a41b7..8c3d9f630 100644 --- a/apm-collector/apm-collector-core/pom.xml +++ b/apm-collector/apm-collector-core/pom.xml @@ -18,11 +18,6 @@ snakeyaml 1.18 - - ch.qos.logback - logback-classic - 1.2.3 - com.google.code.gson gson diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/agentstream/AgentStreamModuleDefine.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/agentstream/AgentStreamModuleDefine.java deleted file mode 100644 index 97e377ba6..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/agentstream/AgentStreamModuleDefine.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.skywalking.apm.collector.core.agentstream; - -import java.util.Map; -import org.skywalking.apm.collector.core.client.Client; -import org.skywalking.apm.collector.core.client.ClientException; -import org.skywalking.apm.collector.core.cluster.ClusterDataInitializer; -import org.skywalking.apm.collector.core.cluster.ClusterModuleContext; -import org.skywalking.apm.collector.core.config.ConfigParseException; -import org.skywalking.apm.collector.core.framework.DataInitializer; -import org.skywalking.apm.collector.core.framework.DefineException; -import org.skywalking.apm.collector.core.module.ModuleDefine; -import org.skywalking.apm.collector.core.server.Server; -import org.skywalking.apm.collector.core.server.ServerException; - -/** - * @author pengys5 - */ -public abstract class AgentStreamModuleDefine extends ModuleDefine { - - @Override public final void initialize(Map config) throws DefineException, ClientException { - try { - configParser().parse(config); - Server server = server(); - server.initialize(); - - String key = ClusterDataInitializer.BASE_CATALOG + "." + name(); - ClusterModuleContext.WRITER.write(key, registration().buildValue()); - } catch (ConfigParseException | ServerException e) { - throw new AgentStreamModuleException(e.getMessage(), e); - } - } - - @Override protected final DataInitializer dataInitializer() { - throw new UnsupportedOperationException(""); - } - - @Override protected final Client createClient() { - throw new UnsupportedOperationException(""); - } -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/client/DataListener.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/client/DataListener.java deleted file mode 100644 index 1022d39aa..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/client/DataListener.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.skywalking.apm.collector.core.client; - -import java.util.List; - -/** - * @author pengys5 - */ -public interface DataListener { - List items(); - - void listen() throws ClientException; -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/client/DataMonitor.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/client/DataMonitor.java new file mode 100644 index 000000000..c5e8d537d --- /dev/null +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/client/DataMonitor.java @@ -0,0 +1,19 @@ +package org.skywalking.apm.collector.core.client; + +import org.skywalking.apm.collector.core.cluster.ClusterDataListener; +import org.skywalking.apm.collector.core.module.ModuleRegistration; + +/** + * @author pengys5 + */ +public interface DataMonitor { + void setClient(Client client); + + void addListener(ClusterDataListener listener, ModuleRegistration registration) throws ClientException; + + ClusterDataListener getListener(String path); + + void createPath(String path) throws ClientException; + + void setData(String path, String value) throws ClientException; +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterDataInitializer.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterDataInitializer.java deleted file mode 100644 index 62ff30d7e..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterDataInitializer.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.skywalking.apm.collector.core.cluster; - -import org.skywalking.apm.collector.core.client.Client; -import org.skywalking.apm.collector.core.client.ClientException; -import org.skywalking.apm.collector.core.framework.DataInitializer; - -/** - * @author pengys5 - */ -public abstract class ClusterDataInitializer implements DataInitializer { - - public static final String BASE_CATALOG = "skywalking"; - public static final String FOR_UI_CATALOG = BASE_CATALOG + ".ui"; - public static final String FOR_AGENT_CATALOG = BASE_CATALOG + ".agent"; - - @Override public final void initialize(Client client) throws ClientException { - if (!existItem(client, FOR_UI_CATALOG)) { - addItem(client, FOR_UI_CATALOG); - } - if (!existItem(client, FOR_AGENT_CATALOG)) { - addItem(client, FOR_AGENT_CATALOG); - } - } -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterDataListener.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterDataListener.java new file mode 100644 index 000000000..463a33adf --- /dev/null +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterDataListener.java @@ -0,0 +1,33 @@ +package org.skywalking.apm.collector.core.cluster; + +import java.util.LinkedList; +import java.util.List; +import org.skywalking.apm.collector.core.framework.Listener; + +/** + * @author pengys5 + */ +public abstract class ClusterDataListener implements Listener { + + private List addresses; + + public ClusterDataListener() { + addresses = new LinkedList<>(); + } + + public abstract String path(); + + public final void addAddress(String address) { + addresses.add(address); + } + + public final List getAddresses() { + return addresses; + } + + public final void clearData() { + addresses.clear(); + } + + public abstract void addressChangedNotify(); +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterDataListenerDefine.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterDataListenerDefine.java new file mode 100644 index 000000000..9f9600579 --- /dev/null +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterDataListenerDefine.java @@ -0,0 +1,8 @@ +package org.skywalking.apm.collector.core.cluster; + +/** + * @author pengys5 + */ +public interface ClusterDataListenerDefine { + ClusterDataListener listener(); +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterModuleContext.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterModuleContext.java index 41a6a2c52..17ab2971e 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterModuleContext.java +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterModuleContext.java @@ -1,10 +1,34 @@ package org.skywalking.apm.collector.core.cluster; +import org.skywalking.apm.collector.core.client.DataMonitor; +import org.skywalking.apm.collector.core.framework.Context; + /** * @author pengys5 */ -public class ClusterModuleContext { - public static ClusterModuleRegistrationWriter WRITER; +public class ClusterModuleContext extends Context { - public static ClusterModuleRegistrationReader READER; + public ClusterModuleContext(String groupName) { + super(groupName); + } + + private ClusterModuleRegistrationReader reader; + + private DataMonitor dataMonitor; + + public ClusterModuleRegistrationReader getReader() { + return reader; + } + + public void setReader(ClusterModuleRegistrationReader reader) { + this.reader = reader; + } + + public DataMonitor getDataMonitor() { + return dataMonitor; + } + + public void setDataMonitor(DataMonitor dataMonitor) { + this.dataMonitor = dataMonitor; + } } diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterModuleDefine.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterModuleDefine.java deleted file mode 100644 index 42e4016c3..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterModuleDefine.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.skywalking.apm.collector.core.cluster; - -import java.util.Map; -import org.skywalking.apm.collector.core.client.Client; -import org.skywalking.apm.collector.core.client.ClientException; -import org.skywalking.apm.collector.core.config.ConfigParseException; -import org.skywalking.apm.collector.core.module.ModuleDefine; -import org.skywalking.apm.collector.core.module.ModuleRegistration; -import org.skywalking.apm.collector.core.server.Server; - -/** - * @author pengys5 - */ -public abstract class ClusterModuleDefine extends ModuleDefine { - - private Client client; - - @Override public final void initialize(Map config) throws ClusterModuleException { - try { - configParser().parse(config); - client = createClient(); - client.initialize(); - dataInitializer().initialize(client); - } catch (ConfigParseException | ClientException e) { - throw new ClusterModuleException(e.getMessage(), e); - } - } - - public final Client getClient() { - return this.client; - } - - @Override public final Server server() { - throw new UnsupportedOperationException(""); - } - - @Override protected final ModuleRegistration registration() { - throw new UnsupportedOperationException("Cluster module do not need module registration."); - } - - protected abstract ClusterModuleRegistrationWriter registrationWriter(); - - protected abstract ClusterModuleRegistrationReader registrationReader(); -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterModuleInstaller.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterModuleInstaller.java deleted file mode 100644 index d632bace9..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterModuleInstaller.java +++ /dev/null @@ -1,43 +0,0 @@ -package org.skywalking.apm.collector.core.cluster; - -import java.util.Iterator; -import java.util.Map; -import org.skywalking.apm.collector.core.client.ClientException; -import org.skywalking.apm.collector.core.framework.DefineException; -import org.skywalking.apm.collector.core.module.ModuleDefine; -import org.skywalking.apm.collector.core.module.ModuleInstaller; -import org.skywalking.apm.collector.core.util.CollectionUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author pengys5 - */ -public class ClusterModuleInstaller implements ModuleInstaller { - - private final Logger logger = LoggerFactory.getLogger(ClusterModuleInstaller.class); - - @Override public void install(Map moduleConfig, - Map moduleDefineMap) throws DefineException, ClientException { - logger.info("beginning cluster module install"); - - ModuleDefine moduleDefine = null; - if (CollectionUtils.isEmpty(moduleConfig)) { - logger.info("could not configure cluster module, use the default"); - Iterator> moduleDefineEntry = moduleDefineMap.entrySet().iterator(); - while (moduleDefineEntry.hasNext()) { - moduleDefine = moduleDefineEntry.next().getValue(); - if (moduleDefine.defaultModule()) { - logger.info("module {} initialize", moduleDefine.getClass().getName()); - moduleDefine.initialize(null); - break; - } - } - } else { - Map.Entry clusterConfigEntry = moduleConfig.entrySet().iterator().next(); - moduleDefine = moduleDefineMap.get(clusterConfigEntry.getKey()); - moduleDefine.initialize(clusterConfigEntry.getValue()); - } - ClusterModuleContext.WRITER = ((ClusterModuleDefine)moduleDefine).registrationWriter(); - } -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterModuleRegistrationReader.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterModuleRegistrationReader.java index d9ad04e02..ddfcdd56a 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterModuleRegistrationReader.java +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterModuleRegistrationReader.java @@ -1,10 +1,20 @@ package org.skywalking.apm.collector.core.cluster; import java.util.List; +import org.skywalking.apm.collector.core.client.DataMonitor; /** * @author pengys5 */ -public interface ClusterModuleRegistrationReader { - List read(String key); +public abstract class ClusterModuleRegistrationReader { + + private final DataMonitor dataMonitor; + + public ClusterModuleRegistrationReader(DataMonitor dataMonitor) { + this.dataMonitor = dataMonitor; + } + + public final List read(String path) { + return dataMonitor.getListener(path).getAddresses(); + } } diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterModuleRegistrationWriter.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterModuleRegistrationWriter.java deleted file mode 100644 index f60eca9df..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterModuleRegistrationWriter.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.skywalking.apm.collector.core.cluster; - -import org.skywalking.apm.collector.core.client.Client; -import org.skywalking.apm.collector.core.client.ClientException; -import org.skywalking.apm.collector.core.module.ModuleRegistration; - -/** - * @author pengys5 - */ -public abstract class ClusterModuleRegistrationWriter { - - protected final Client client; - - public ClusterModuleRegistrationWriter(Client client) { - this.client = client; - } - - public abstract void write(String key, ModuleRegistration.Value value) throws ClientException; -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/CollectorContextHelper.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/CollectorContextHelper.java new file mode 100644 index 000000000..b0d6e96cf --- /dev/null +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/CollectorContextHelper.java @@ -0,0 +1,25 @@ +package org.skywalking.apm.collector.core.framework; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * @author pengys5 + */ +public enum CollectorContextHelper { + INSTANCE; + + private Map contexts = new LinkedHashMap(); + + public Context getContext(String moduleGroupName) { + return contexts.get(moduleGroupName); + } + + public void putContext(Context context) { + if (contexts.containsKey(context.getGroupName())) { + throw new UnsupportedOperationException("This module context was put, do not allow put a new one"); + } else { + contexts.put(context.getGroupName(), context); + } + } +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Context.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Context.java index db34bb29a..9be638427 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Context.java +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Context.java @@ -3,5 +3,14 @@ package org.skywalking.apm.collector.core.framework; /** * @author pengys5 */ -public class Context { +public abstract class Context { + private final String groupName; + + public Context(String groupName) { + this.groupName = groupName; + } + + public final String getGroupName() { + return groupName; + } } diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Define.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Define.java index 542d72b7e..69e97e294 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Define.java +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Define.java @@ -2,13 +2,14 @@ package org.skywalking.apm.collector.core.framework; import java.util.Map; import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.server.ServerHolder; /** * @author pengys5 */ public interface Define { - void initialize(Map config) throws DefineException, ClientException; + void initialize(Map config, ServerHolder serverHolder) throws DefineException, ClientException; String name(); } diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Executor.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Executor.java new file mode 100644 index 000000000..b09717518 --- /dev/null +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Executor.java @@ -0,0 +1,8 @@ +package org.skywalking.apm.collector.core.framework; + +/** + * @author pengys5 + */ +public interface Executor { + void execute(Object message); +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Writer.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Handler.java similarity index 75% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Writer.java rename to apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Handler.java index 68c40c065..0ae17b083 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Writer.java +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Handler.java @@ -3,5 +3,5 @@ package org.skywalking.apm.collector.core.framework; /** * @author pengys5 */ -public interface Writer { +public interface Handler { } diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Listener.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Listener.java new file mode 100644 index 000000000..d486408eb --- /dev/null +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Listener.java @@ -0,0 +1,7 @@ +package org.skywalking.apm.collector.core.framework; + +/** + * @author pengys5 + */ +public interface Listener { +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Loader.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Loader.java index 10c4be934..485925c4c 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Loader.java +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/framework/Loader.java @@ -1,10 +1,8 @@ package org.skywalking.apm.collector.core.framework; -import org.skywalking.apm.collector.core.config.ConfigException; - /** * @author pengys5 */ public interface Loader { - T load() throws ConfigException; + T load() throws DefineException; } diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleConfigLoader.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleConfigLoader.java index 55c8a0e78..2dee03f20 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleConfigLoader.java +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleConfigLoader.java @@ -1,8 +1,10 @@ package org.skywalking.apm.collector.core.module; import java.io.FileNotFoundException; +import java.io.FileReader; import java.util.Map; import org.skywalking.apm.collector.core.config.ConfigLoader; +import org.skywalking.apm.collector.core.framework.DefineException; import org.skywalking.apm.collector.core.util.ResourceUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -15,12 +17,18 @@ public class ModuleConfigLoader implements ConfigLoader> { private final Logger logger = LoggerFactory.getLogger(ModuleConfigLoader.class); - @Override public Map load() throws ModuleConfigLoaderException { + @Override public Map load() throws DefineException { Yaml yaml = new Yaml(); try { - return (Map)yaml.load(ResourceUtils.read("application.yml")); + try { + FileReader applicationFileReader = ResourceUtils.read("application.yml"); + return (Map)yaml.load(applicationFileReader); + } catch (FileNotFoundException e) { + logger.info("Could not found application.yml file, use default"); + return (Map)yaml.load(ResourceUtils.read("application-default.yml")); + } } catch (FileNotFoundException e) { - throw new ModuleConfigLoaderException(e.getMessage(), e); + throw new ModuleDefineException(e.getMessage(), e); } } } diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleConfigLoaderException.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleConfigLoaderException.java deleted file mode 100644 index 9143d0197..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleConfigLoaderException.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.skywalking.apm.collector.core.module; - -import org.skywalking.apm.collector.core.config.ConfigLoaderException; - -/** - * @author pengys5 - */ -public class ModuleConfigLoaderException extends ConfigLoaderException { - public ModuleConfigLoaderException(String message) { - super(message); - } - - public ModuleConfigLoaderException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleDefine.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleDefine.java index 1ead094ea..49cc7704a 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleDefine.java +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleDefine.java @@ -1,7 +1,7 @@ package org.skywalking.apm.collector.core.module; import org.skywalking.apm.collector.core.client.Client; -import org.skywalking.apm.collector.core.framework.DataInitializer; +import org.skywalking.apm.collector.core.client.DataMonitor; import org.skywalking.apm.collector.core.framework.Define; import org.skywalking.apm.collector.core.server.Server; @@ -10,17 +10,15 @@ import org.skywalking.apm.collector.core.server.Server; */ public abstract class ModuleDefine implements Define { - protected abstract ModuleGroup group(); + protected abstract String group(); public abstract boolean defaultModule(); protected abstract ModuleConfigParser configParser(); - protected abstract Client createClient(); + protected abstract Client createClient(DataMonitor dataMonitor); protected abstract Server server(); - protected abstract DataInitializer dataInitializer(); - protected abstract ModuleRegistration registration(); } diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleDefineException.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleDefineException.java new file mode 100644 index 000000000..773f465ec --- /dev/null +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleDefineException.java @@ -0,0 +1,16 @@ +package org.skywalking.apm.collector.core.module; + +import org.skywalking.apm.collector.core.framework.DefineException; + +/** + * @author pengys5 + */ +public class ModuleDefineException extends DefineException { + public ModuleDefineException(String message) { + super(message); + } + + public ModuleDefineException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleDefineLoader.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleDefineLoader.java index 407bc13ae..5808ce0df 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleDefineLoader.java +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleDefineLoader.java @@ -2,7 +2,7 @@ package org.skywalking.apm.collector.core.module; import java.util.LinkedHashMap; import java.util.Map; -import org.skywalking.apm.collector.core.config.ConfigException; +import org.skywalking.apm.collector.core.framework.DefineException; import org.skywalking.apm.collector.core.framework.Loader; import org.skywalking.apm.collector.core.util.DefinitionLoader; import org.slf4j.Logger; @@ -15,7 +15,7 @@ public class ModuleDefineLoader implements Loader> load() throws ConfigException { + @Override public Map> load() throws DefineException { Map> moduleDefineMap = new LinkedHashMap<>(); ModuleDefinitionFile definitionFile = new ModuleDefinitionFile(); @@ -24,7 +24,7 @@ public class ModuleDefineLoader implements Loader()); } diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleGroup.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleGroup.java deleted file mode 100644 index 31d9e4aed..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleGroup.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.skywalking.apm.collector.core.module; - -/** - * @author pengys5 - */ -public enum ModuleGroup { - Cluster, Queue, AgentStream -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleGroupDefine.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleGroupDefine.java index 8bca7831e..50783f9d3 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleGroupDefine.java +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleGroupDefine.java @@ -1,10 +1,14 @@ package org.skywalking.apm.collector.core.module; +import org.skywalking.apm.collector.core.framework.Context; + /** * @author pengys5 */ public interface ModuleGroupDefine { String name(); - ModuleInstallMode mode(); + Context groupContext(); + + ModuleInstaller moduleInstaller(); } diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleGroupDefineLoader.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleGroupDefineLoader.java index 6f4cb58aa..6a4e1387c 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleGroupDefineLoader.java +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleGroupDefineLoader.java @@ -2,7 +2,7 @@ package org.skywalking.apm.collector.core.module; import java.util.LinkedHashMap; import java.util.Map; -import org.skywalking.apm.collector.core.config.ConfigException; +import org.skywalking.apm.collector.core.framework.DefineException; import org.skywalking.apm.collector.core.framework.Loader; import org.skywalking.apm.collector.core.util.DefinitionLoader; import org.slf4j.Logger; @@ -15,7 +15,7 @@ public class ModuleGroupDefineLoader implements Loader load() throws ConfigException { + @Override public Map load() throws DefineException { Map moduleGroupDefineMap = new LinkedHashMap<>(); ModuleGroupDefineFile definitionFile = new ModuleGroupDefineFile(); diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleInstallMode.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleInstallMode.java deleted file mode 100644 index 3b2976797..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleInstallMode.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.skywalking.apm.collector.core.module; - -/** - * @author pengys5 - */ -public enum ModuleInstallMode { - Single, Multiple -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleInstaller.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleInstaller.java index 40aa905c8..2ec52b90b 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleInstaller.java +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleInstaller.java @@ -3,11 +3,12 @@ package org.skywalking.apm.collector.core.module; import java.util.Map; import org.skywalking.apm.collector.core.client.ClientException; import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.server.ServerHolder; /** * @author pengys5 */ public interface ModuleInstaller { void install(Map moduleConfig, - Map moduleDefineMap) throws DefineException, ClientException; + Map moduleDefineMap, ServerHolder serverHolder) throws DefineException, ClientException; } diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleInstallerAdapter.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleInstallerAdapter.java deleted file mode 100644 index 48c6c2d19..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleInstallerAdapter.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.skywalking.apm.collector.core.module; - -import java.util.Map; -import org.skywalking.apm.collector.core.client.ClientException; -import org.skywalking.apm.collector.core.cluster.ClusterModuleInstaller; -import org.skywalking.apm.collector.core.framework.DefineException; - -/** - * @author pengys5 - */ -public class ModuleInstallerAdapter implements ModuleInstaller { - - private ModuleInstaller moduleInstaller; - - public ModuleInstallerAdapter(ModuleGroup moduleGroup) { - if (ModuleGroup.Cluster.equals(moduleGroup)) { - moduleInstaller = new ClusterModuleInstaller(); - } - } - - @Override public void install(Map moduleConfig, - Map moduleDefineMap) throws DefineException, ClientException { - moduleInstaller.install(moduleConfig, moduleDefineMap); - } -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleRegistration.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleRegistration.java index 7242cbf2d..90dd9afe9 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleRegistration.java +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/ModuleRegistration.java @@ -1,7 +1,5 @@ package org.skywalking.apm.collector.core.module; -import com.google.gson.JsonObject; - /** * @author pengys5 */ @@ -12,12 +10,12 @@ public abstract class ModuleRegistration { public static class Value { private final String host; private final int port; - private final JsonObject data; + private final String contextPath; - public Value(String host, int port, JsonObject data) { + public Value(String host, int port, String contextPath) { this.host = host; this.port = port; - this.data = data; + this.contextPath = contextPath; } public String getHost() { @@ -28,8 +26,12 @@ public abstract class ModuleRegistration { return port; } - public JsonObject getData() { - return data; + public String getHostPort() { + return host + ":" + port; + } + + public String getContextPath() { + return contextPath; } } } \ No newline at end of file diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/SingleModuleInstaller.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/SingleModuleInstaller.java new file mode 100644 index 000000000..247282402 --- /dev/null +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/module/SingleModuleInstaller.java @@ -0,0 +1,39 @@ +package org.skywalking.apm.collector.core.module; + +import java.util.Iterator; +import java.util.Map; +import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.server.ServerHolder; +import org.skywalking.apm.collector.core.util.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public abstract class SingleModuleInstaller implements ModuleInstaller { + + private final Logger logger = LoggerFactory.getLogger(SingleModuleInstaller.class); + + protected void installSingle(Map moduleConfig, + Map moduleDefineMap, ServerHolder serverHolder) throws DefineException, ClientException { + ModuleDefine moduleDefine = null; + if (CollectionUtils.isEmpty(moduleConfig)) { + logger.info("could not configure module, use the default"); + Iterator> moduleDefineEntry = moduleDefineMap.entrySet().iterator(); + while (moduleDefineEntry.hasNext()) { + moduleDefine = moduleDefineEntry.next().getValue(); + if (moduleDefine.defaultModule()) { + logger.info("module {} initialize", moduleDefine.getClass().getName()); + moduleDefine.initialize(null, serverHolder); + break; + } + } + } else { + Map.Entry configEntry = moduleConfig.entrySet().iterator().next(); + moduleDefine = moduleDefineMap.get(configEntry.getKey()); + moduleDefine.initialize(configEntry.getValue(), serverHolder); + } + } +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/queue/Creator.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/queue/Creator.java deleted file mode 100644 index c2da5c469..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/queue/Creator.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.skywalking.apm.collector.core.queue; - -import org.skywalking.apm.collector.core.worker.AbstractLocalAsyncWorker; - -/** - * @author pengys5 - */ -public interface Creator { - QueueEventHandler create(int queueSize, AbstractLocalAsyncWorker localAsyncWorker); -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/queue/QueueCreator.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/queue/QueueCreator.java new file mode 100644 index 000000000..d404f23a6 --- /dev/null +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/queue/QueueCreator.java @@ -0,0 +1,8 @@ +package org.skywalking.apm.collector.core.queue; + +/** + * @author pengys5 + */ +public interface QueueCreator { + QueueEventHandler create(int queueSize, QueueExecutor executor); +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/queue/QueueExecutor.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/queue/QueueExecutor.java new file mode 100644 index 000000000..1576caf58 --- /dev/null +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/queue/QueueExecutor.java @@ -0,0 +1,9 @@ +package org.skywalking.apm.collector.core.queue; + +import org.skywalking.apm.collector.core.framework.Executor; + +/** + * @author pengys5 + */ +public interface QueueExecutor extends Executor { +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/queue/QueueModuleContext.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/queue/QueueModuleContext.java deleted file mode 100644 index 7825ca638..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/queue/QueueModuleContext.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.skywalking.apm.collector.core.queue; - -/** - * @author pengys5 - */ -public class QueueModuleContext { - public static Creator CREATOR; -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/RemoteModuleDefine.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/RemoteModuleDefine.java deleted file mode 100644 index b8e57b5a7..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/RemoteModuleDefine.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.skywalking.apm.collector.core.remote; - -import org.skywalking.apm.collector.core.module.ModuleDefine; - -/** - * @author pengys5 - */ -public abstract class RemoteModuleDefine extends ModuleDefine { -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/SerializedDefine.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/SerializedDefine.java deleted file mode 100644 index bafafd87e..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/SerializedDefine.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.skywalking.apm.collector.core.remote; - -/** - * @author pengys5 - */ -public interface SerializedDefine { - int ID(); - - Class clazz(); -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/SerializedDefineLoader.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/SerializedDefineLoader.java deleted file mode 100644 index c07386260..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/SerializedDefineLoader.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.skywalking.apm.collector.core.remote; - -import java.util.LinkedHashMap; -import java.util.Map; -import org.skywalking.apm.collector.core.config.ConfigException; -import org.skywalking.apm.collector.core.framework.Loader; -import org.skywalking.apm.collector.core.module.ModuleGroupDefineLoader; -import org.skywalking.apm.collector.core.util.DefinitionLoader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author pengys5 - */ -public class SerializedDefineLoader implements Loader> { - - private final Logger logger = LoggerFactory.getLogger(ModuleGroupDefineLoader.class); - - @Override public Map load() throws ConfigException { - Map serializedDefineMap = new LinkedHashMap<>(); - SerializedDefinitionFile definitionFile = new SerializedDefinitionFile(); - logger.info("serialized definition file name: {}", definitionFile.fileName()); - - DefinitionLoader definitionLoader = DefinitionLoader.load(SerializedDefine.class, definitionFile); - - int id = 1; - for (SerializedDefine serializedDefine : definitionLoader) { - logger.info("loaded serialized definition class: {}", serializedDefine.getClass().getName()); - serializedDefineMap.put(id, serializedDefine); - } - return serializedDefineMap; - } -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/SerializedDefinitionFile.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/SerializedDefinitionFile.java deleted file mode 100644 index a8ee1988a..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/remote/SerializedDefinitionFile.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.skywalking.apm.collector.core.remote; - -import org.skywalking.apm.collector.core.framework.DefinitionFile; - -/** - * @author pengys5 - */ -public class SerializedDefinitionFile extends DefinitionFile { - @Override protected String fileName() { - return "serialized.define"; - } -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/server/Server.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/server/Server.java index eede89b81..1d12d31e8 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/server/Server.java +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/server/Server.java @@ -1,9 +1,19 @@ package org.skywalking.apm.collector.core.server; +import org.skywalking.apm.collector.core.framework.Handler; + /** * @author pengys5 */ public interface Server { + String hostPort(); + + String serverClassify(); + void initialize() throws ServerException; + + void start() throws ServerException; + + void addHandler(Handler handler); } diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/server/ServerHolder.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/server/ServerHolder.java new file mode 100644 index 000000000..96edf8f02 --- /dev/null +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/server/ServerHolder.java @@ -0,0 +1,43 @@ +package org.skywalking.apm.collector.core.server; + +import java.util.LinkedList; +import java.util.List; +import org.skywalking.apm.collector.core.framework.Handler; +import org.skywalking.apm.collector.core.util.CollectionUtils; + +/** + * @author pengys5 + */ +public class ServerHolder { + + private List servers; + + public ServerHolder() { + servers = new LinkedList<>(); + } + + public void holdServer(Server newServer, List handlers) throws ServerException { + boolean isNewServer = true; + for (Server server : servers) { + if (server.hostPort().equals(newServer.hostPort()) && server.serverClassify().equals(newServer.serverClassify())) { + isNewServer = false; + addHandler(handlers, server); + } + } + if (isNewServer) { + newServer.initialize(); + servers.add(newServer); + addHandler(handlers, newServer); + } + } + + private void addHandler(List handlers, Server server) { + if (CollectionUtils.isNotEmpty(handlers)) { + handlers.forEach(handler -> server.addHandler(handler)); + } + } + + public List getServers() { + return servers; + } +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/Column.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/Column.java deleted file mode 100644 index b8eac2429..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/Column.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.skywalking.apm.collector.core.storage; - -/** - * @author pengys5 - */ -public abstract class Column { - - private String name; - - private T value; - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public T getValue() { - return value; - } - - public void setValue(T value) { - this.value = value; - } -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/ColumnDefine.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/ColumnDefine.java new file mode 100644 index 000000000..4e1f810db --- /dev/null +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/ColumnDefine.java @@ -0,0 +1,22 @@ +package org.skywalking.apm.collector.core.storage; + +/** + * @author pengys5 + */ +public abstract class ColumnDefine { + private final String name; + private final String type; + + public ColumnDefine(String name, String type) { + this.name = name; + this.type = type; + } + + public final String getName() { + return name; + } + + public String getType() { + return type; + } +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/Create.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/Create.java deleted file mode 100644 index a690635d2..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/Create.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.skywalking.apm.collector.core.storage; - -/** - * @author pengys5 - */ -public abstract class Create { -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/Insert.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/Insert.java deleted file mode 100644 index 993f51157..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/Insert.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.skywalking.apm.collector.core.storage; - -/** - * @author pengys5 - */ -public abstract class Insert { - -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/IntegerColumn.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/IntegerColumn.java deleted file mode 100644 index 1d1bff45d..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/IntegerColumn.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.skywalking.apm.collector.core.storage; - -/** - * @author pengys5 - */ -public class IntegerColumn extends Column { -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/LongColumn.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/LongColumn.java deleted file mode 100644 index a104e7336..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/LongColumn.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.skywalking.apm.collector.core.storage; - -/** - * @author pengys5 - */ -public class LongColumn extends Column { -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageBatchBuilder.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageBatchBuilder.java new file mode 100644 index 000000000..5f1665a76 --- /dev/null +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageBatchBuilder.java @@ -0,0 +1,11 @@ +package org.skywalking.apm.collector.core.storage; + +import java.util.List; +import java.util.Map; + +/** + * @author pengys5 + */ +public abstract class StorageBatchBuilder { + public abstract List build(C client, Map lastData); +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageDefineLoader.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageDefineLoader.java new file mode 100644 index 000000000..618ed1c6a --- /dev/null +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageDefineLoader.java @@ -0,0 +1,30 @@ +package org.skywalking.apm.collector.core.storage; + +import java.util.LinkedList; +import java.util.List; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.framework.Loader; +import org.skywalking.apm.collector.core.util.DefinitionLoader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class StorageDefineLoader implements Loader> { + + private final Logger logger = LoggerFactory.getLogger(StorageDefineLoader.class); + + @Override public List load() throws DefineException { + List tableDefines = new LinkedList<>(); + + StorageDefinitionFile definitionFile = new StorageDefinitionFile(); + logger.info("storage definition file name: {}", definitionFile.fileName()); + DefinitionLoader definitionLoader = DefinitionLoader.load(TableDefine.class, definitionFile); + for (TableDefine tableDefine : definitionLoader) { + logger.info("loaded storage definition class: {}", tableDefine.getClass().getName()); + tableDefines.add(tableDefine); + } + return tableDefines; + } +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageDefinitionFile.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageDefinitionFile.java new file mode 100644 index 000000000..c9c9fc09f --- /dev/null +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageDefinitionFile.java @@ -0,0 +1,12 @@ +package org.skywalking.apm.collector.core.storage; + +import org.skywalking.apm.collector.core.framework.DefinitionFile; + +/** + * @author pengys5 + */ +public class StorageDefinitionFile extends DefinitionFile { + @Override protected String fileName() { + return "storage.define"; + } +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageException.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageException.java new file mode 100644 index 000000000..22cda34e9 --- /dev/null +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageException.java @@ -0,0 +1,16 @@ +package org.skywalking.apm.collector.core.storage; + +import org.skywalking.apm.collector.core.CollectorException; + +/** + * @author pengys5 + */ +public abstract class StorageException extends CollectorException { + public StorageException(String message) { + super(message); + } + + public StorageException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageInstallException.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageInstallException.java new file mode 100644 index 000000000..bab04dcd9 --- /dev/null +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageInstallException.java @@ -0,0 +1,15 @@ +package org.skywalking.apm.collector.core.storage; + +/** + * @author pengys5 + */ +public class StorageInstallException extends StorageException { + + public StorageInstallException(String message) { + super(message); + } + + public StorageInstallException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageInstaller.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageInstaller.java new file mode 100644 index 000000000..0ae185794 --- /dev/null +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageInstaller.java @@ -0,0 +1,41 @@ +package org.skywalking.apm.collector.core.storage; + +import java.util.List; +import org.skywalking.apm.collector.core.client.Client; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public abstract class StorageInstaller { + + private final Logger logger = LoggerFactory.getLogger(StorageInstaller.class); + + public final void install(Client client) throws StorageException { + StorageDefineLoader defineLoader = new StorageDefineLoader(); + try { + List tableDefines = defineLoader.load(); + defineFilter(tableDefines); + + for (TableDefine tableDefine : tableDefines) { + if (!isExists(client, tableDefine)) { + logger.info("table: {} not exists", tableDefine.getName()); + tableDefine.initialize(); + createTable(client, tableDefine); + } + } + } catch (DefineException e) { + throw new StorageInstallException(e.getMessage(), e); + } + } + + protected abstract void defineFilter(List tableDefines); + + protected abstract boolean isExists(Client client, TableDefine tableDefine) throws StorageException; + + protected abstract boolean deleteIndex(Client client, TableDefine tableDefine) throws StorageException; + + protected abstract boolean createTable(Client client, TableDefine tableDefine) throws StorageException; +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StringColumn.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StringColumn.java deleted file mode 100644 index ac9facf11..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StringColumn.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.skywalking.apm.collector.core.storage; - -/** - * @author pengys5 - */ -public class StringColumn extends Column { -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/TableDefine.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/TableDefine.java new file mode 100644 index 000000000..1fa7d88f8 --- /dev/null +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/TableDefine.java @@ -0,0 +1,31 @@ +package org.skywalking.apm.collector.core.storage; + +import java.util.LinkedList; +import java.util.List; + +/** + * @author pengys5 + */ +public abstract class TableDefine { + private final String name; + private final List columnDefines; + + public TableDefine(String name) { + this.name = name; + this.columnDefines = new LinkedList<>(); + } + + public abstract void initialize(); + + public final void addColumn(ColumnDefine columnDefine) { + columnDefines.add(columnDefine); + } + + public final String getName() { + return name; + } + + public final List getColumnDefines() { + return columnDefines; + } +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/util/CollectionUtils.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/util/CollectionUtils.java index dd66a5ecb..22141d6a2 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/util/CollectionUtils.java +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/util/CollectionUtils.java @@ -1,5 +1,6 @@ package org.skywalking.apm.collector.core.util; +import java.util.List; import java.util.Map; /** @@ -10,4 +11,12 @@ public class CollectionUtils { public static boolean isEmpty(Map map) { return map == null || map.size() == 0; } + + public static boolean isEmpty(List list) { + return list == null || list.size() == 0; + } + + public static boolean isNotEmpty(List list) { + return !isEmpty(list); + } } diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/util/ObjectUtils.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/util/ObjectUtils.java index df264ab32..9300a103e 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/util/ObjectUtils.java +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/util/ObjectUtils.java @@ -7,4 +7,8 @@ public class ObjectUtils { public static boolean isEmpty(Object obj) { return obj == null; } + + public static boolean isNotEmpty(Object obj) { + return !isEmpty(obj); + } } diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/util/ResourceUtils.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/util/ResourceUtils.java index 31daeec6f..257f85dd2 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/util/ResourceUtils.java +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/util/ResourceUtils.java @@ -1,16 +1,21 @@ package org.skywalking.apm.collector.core.util; +import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; +import java.net.URL; /** * @author pengys5 */ public class ResourceUtils { - private static final String PATH = ResourceUtils.class.getResource("/").getPath(); - public static FileReader read(String fileName) throws FileNotFoundException { - return new FileReader(PATH + fileName); + URL url = ResourceUtils.class.getClassLoader().getResource(fileName); + if (url == null) { + throw new FileNotFoundException("file not found: " + fileName); + } + File file = new File(ResourceUtils.class.getClassLoader().getResource(fileName).getFile()); + return new FileReader(file); } } diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/util/StringUtils.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/util/StringUtils.java index 83413c178..58d260d7e 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/util/StringUtils.java +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/util/StringUtils.java @@ -10,4 +10,8 @@ public class StringUtils { public static boolean isEmpty(Object str) { return str == null || EMPTY_STRING.equals(str); } + + public static boolean isNotEmpty(Object str) { + return !isEmpty(str); + } } diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractClusterWorkerProvider.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractClusterWorkerProvider.java deleted file mode 100644 index 519689182..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractClusterWorkerProvider.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.skywalking.apm.collector.core.worker; - -/** - * The AbstractClusterWorkerProvider implementations represent providers, - * which create instance of cluster workers whose implemented {@link AbstractClusterWorker}. - *

- * - * @author pengys5 - * @since v3.0-2017 - */ -public abstract class AbstractClusterWorkerProvider extends AbstractWorkerProvider { - - /** - * Create how many worker instance of {@link AbstractClusterWorker} in one jvm. - * - * @return The worker instance number. - */ - public abstract int workerNum(); - - /** - * Create the worker instance into akka system, the akka system will control the cluster worker life cycle. - * - * @param localContext Not used, will be null. - * @return The created worker reference. See {@link ClusterWorkerRef} - * @throws ProviderNotFoundException This worker instance attempted to find a provider which use to create another - * worker instance, when the worker provider not find then Throw this Exception. - */ - @Override final public WorkerRef onCreate( - LocalWorkerContext localContext) throws ProviderNotFoundException { - T clusterWorker = workerInstance(getClusterContext()); - clusterWorker.preStart(); - - ClusterWorkerRef workerRef = new ClusterWorkerRef(role(), clusterWorker); - getClusterContext().put(workerRef); - return workerRef; - } -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractLocalAsyncWorkerProvider.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractLocalAsyncWorkerProvider.java deleted file mode 100644 index 22b3e9a58..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractLocalAsyncWorkerProvider.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.skywalking.apm.collector.core.worker; - -import org.skywalking.apm.collector.core.queue.QueueEventHandler; -import org.skywalking.apm.collector.core.queue.QueueModuleContext; - -/** - * @author pengys5 - */ -public abstract class AbstractLocalAsyncWorkerProvider extends AbstractLocalWorkerProvider { - - public abstract int queueSize(); - - @Override final public WorkerRef onCreate( - LocalWorkerContext localContext) throws ProviderNotFoundException { - T localAsyncWorker = workerInstance(getClusterContext()); - localAsyncWorker.preStart(); - - QueueEventHandler queueEventHandler = QueueModuleContext.CREATOR.create(queueSize(), localAsyncWorker); - - LocalAsyncWorkerRef workerRef = new LocalAsyncWorkerRef(role(), queueEventHandler); - - if (localContext != null) { - localContext.put(workerRef); - } - - return workerRef; - } -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractLocalSyncWorkerProvider.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractLocalSyncWorkerProvider.java deleted file mode 100644 index 056b724f6..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractLocalSyncWorkerProvider.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.skywalking.apm.collector.core.worker; - -/** - * @author pengys5 - */ -public abstract class AbstractLocalSyncWorkerProvider extends AbstractLocalWorkerProvider { - - @Override - final public WorkerRef onCreate( - LocalWorkerContext localContext) throws ProviderNotFoundException { - T localSyncWorker = (T) workerInstance(getClusterContext()); - localSyncWorker.preStart(); - - LocalSyncWorkerRef workerRef = new LocalSyncWorkerRef(role(), localSyncWorker); - - if (localContext != null) { - localContext.put(workerRef); - } - return workerRef; - } -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractWorker.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractWorker.java deleted file mode 100644 index cdfc839c7..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractWorker.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.skywalking.apm.collector.core.worker; - -/** - * @author pengys5 - */ -public abstract class AbstractWorker { - - private final LocalWorkerContext selfContext; - - private final Role role; - - private final ClusterWorkerContext clusterContext; - - public AbstractWorker(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) { - this.role = role; - this.clusterContext = clusterContext; - this.selfContext = selfContext; - } - - public abstract void preStart() throws ProviderNotFoundException; - - final public LookUp getSelfContext() { - return selfContext; - } - - final public LookUp getClusterContext() { - return clusterContext; - } - - final public Role getRole() { - return role; - } - - final public static AbstractWorker noOwner() { - return null; - } -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractWorkerProvider.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractWorkerProvider.java deleted file mode 100644 index bc87c19e4..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractWorkerProvider.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.skywalking.apm.collector.core.worker; - -/** - * @author pengys5 - */ -public abstract class AbstractWorkerProvider implements Provider { - - private ClusterWorkerContext clusterContext; - - public abstract Role role(); - - public abstract T workerInstance(ClusterWorkerContext clusterContext); - - public abstract WorkerRef onCreate( - LocalWorkerContext localContext) throws ProviderNotFoundException; - - final public void setClusterContext(ClusterWorkerContext clusterContext) { - this.clusterContext = clusterContext; - } - - final protected ClusterWorkerContext getClusterContext() { - return clusterContext; - } - - final public WorkerRef create( - AbstractWorker workerOwner) throws ProviderNotFoundException { - - if (workerOwner == null) { - return onCreate(null); - } else if (workerOwner.getSelfContext() instanceof LocalWorkerContext) { - return onCreate((LocalWorkerContext)workerOwner.getSelfContext()); - } else { - throw new IllegalArgumentException("the argument of workerOwner is Illegal"); - } - } -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/ClusterWorkerContext.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/ClusterWorkerContext.java deleted file mode 100644 index ebd29da62..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/ClusterWorkerContext.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.skywalking.apm.collector.core.worker; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author pengys5 - */ -public class ClusterWorkerContext extends WorkerContext { - private final Logger logger = LoggerFactory.getLogger(ClusterWorkerContext.class); - - private Map providers = new ConcurrentHashMap<>(); - - @Override - public AbstractWorkerProvider findProvider(Role role) throws ProviderNotFoundException { - logger.debug("find role of %s provider from ClusterWorkerContext", role.roleName()); - if (providers.containsKey(role.roleName())) { - return providers.get(role.roleName()); - } else { - throw new ProviderNotFoundException("role=" + role.roleName() + ", no available provider."); - } - } - - @Override - public void putProvider(AbstractWorkerProvider provider) throws UsedRoleNameException { - logger.debug("put role of %s provider into ClusterWorkerContext", provider.role().roleName()); - if (providers.containsKey(provider.role().roleName())) { - throw new UsedRoleNameException("provider with role=" + provider.role().roleName() + " duplicate each other."); - } else { - providers.put(provider.role().roleName(), provider); - } - } -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/ClusterWorkerRef.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/ClusterWorkerRef.java deleted file mode 100644 index dc2243f3a..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/ClusterWorkerRef.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.skywalking.apm.collector.core.worker; - -/** - * @author pengys5 - */ -public class ClusterWorkerRef extends WorkerRef { - - private AbstractClusterWorker clusterWorker; - - public ClusterWorkerRef(Role role, AbstractClusterWorker clusterWorker) { - super(role); - this.clusterWorker = clusterWorker; - } - - @Override - public void tell(Object message) throws WorkerInvokeException { - clusterWorker.allocateJob(message); - } -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/LocalWorkerContext.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/LocalWorkerContext.java deleted file mode 100644 index 3d70bbb56..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/LocalWorkerContext.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.skywalking.apm.collector.core.worker; - -/** - * @author pengys5 - */ -public class LocalWorkerContext extends WorkerContext { - - @Override - final public AbstractWorkerProvider findProvider(Role role) throws ProviderNotFoundException { - return null; - } - - @Override - final public void putProvider(AbstractWorkerProvider provider) throws UsedRoleNameException { - - } -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/Provider.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/Provider.java deleted file mode 100644 index d9ccf9769..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/Provider.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.skywalking.apm.collector.core.worker; - -/** - * @author pengys5 - */ -public interface Provider { - - WorkerRef create(AbstractWorker workerOwner) throws ProviderNotFoundException; -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/Role.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/Role.java deleted file mode 100644 index 8c59ff0ac..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/Role.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.skywalking.apm.collector.core.worker; - -import org.skywalking.apm.collector.core.worker.selector.WorkerSelector; - -/** - * @author pengys5 - */ -public interface Role { - - String roleName(); - - WorkerSelector workerSelector(); -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/WorkerContext.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/WorkerContext.java deleted file mode 100644 index 78ff5e231..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/WorkerContext.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.skywalking.apm.collector.core.worker; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -/** - * @author pengys5 - */ -public abstract class WorkerContext implements Context { - - private Map> roleWorkers; - - public WorkerContext() { - this.roleWorkers = new ConcurrentHashMap<>(); - } - - private Map> getRoleWorkers() { - return this.roleWorkers; - } - - @Override - final public WorkerRefs lookup(Role role) throws WorkerNotFoundException { - if (getRoleWorkers().containsKey(role.roleName())) { - WorkerRefs refs = new WorkerRefs(getRoleWorkers().get(role.roleName()), role.workerSelector()); - return refs; - } else { - throw new WorkerNotFoundException("role=" + role.roleName() + ", no available worker."); - } - } - - @Override - final public void put(WorkerRef workerRef) { - if (!getRoleWorkers().containsKey(workerRef.getRole().roleName())) { - getRoleWorkers().putIfAbsent(workerRef.getRole().roleName(), new ArrayList()); - } - getRoleWorkers().get(workerRef.getRole().roleName()).add(workerRef); - } - - @Override - final public void remove(WorkerRef workerRef) { - getRoleWorkers().remove(workerRef.getRole().roleName()); - } -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/WorkerModuleInstaller.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/WorkerModuleInstaller.java deleted file mode 100644 index 0b86aa90c..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/WorkerModuleInstaller.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.skywalking.apm.collector.core.worker; - -import java.util.Map; -import org.skywalking.apm.collector.core.client.ClientException; -import org.skywalking.apm.collector.core.framework.DefineException; -import org.skywalking.apm.collector.core.module.ModuleDefine; -import org.skywalking.apm.collector.core.module.ModuleInstaller; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author pengys5 - */ -public class WorkerModuleInstaller implements ModuleInstaller { - - private final Logger logger = LoggerFactory.getLogger(WorkerModuleInstaller.class); - - @Override public void install(Map moduleConfig, - Map moduleDefineMap) throws DefineException, ClientException { - logger.info("beginning worker module install"); - Map.Entry workerConfigEntry = moduleConfig.entrySet().iterator().next(); - ModuleDefine moduleDefine = moduleDefineMap.get(workerConfigEntry.getKey()); - moduleDefine.initialize(workerConfigEntry.getValue()); - } -} diff --git a/apm-collector/apm-collector-core/src/main/resources/application-default.yml b/apm-collector/apm-collector-core/src/main/resources/application-default.yml new file mode 100644 index 000000000..1e4e62547 --- /dev/null +++ b/apm-collector/apm-collector-core/src/main/resources/application-default.yml @@ -0,0 +1,26 @@ +cluster: + zookeeper: + hostPort: localhost:2181 + sessionTimeout: 100000 +# redis: +# host: localhost +# port: 6379 +queue: + disruptor: on + data_carrier: off +agentstream: + grpc: + host: localhost + port: 1000 + jetty: + host: localhost + port: 2000 + context_path: / +discovery: + grpc: localhost + port: 1000 +ui: + jetty: + host: localhost + port: 12800 + diff --git a/apm-collector/apm-collector-core/src/main/resources/logback.xml b/apm-collector/apm-collector-core/src/main/resources/logback.xml index b0df07af4..dc57f744d 100644 --- a/apm-collector/apm-collector-core/src/main/resources/logback.xml +++ b/apm-collector/apm-collector-core/src/main/resources/logback.xml @@ -2,11 +2,12 @@ - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{70} - %msg%n - + + diff --git a/apm-collector/apm-collector-core/src/test/java/org/skywalking/apm/collector/core/config/ModuleConfigLoaderTestCase.java b/apm-collector/apm-collector-core/src/test/java/org/skywalking/apm/collector/core/config/ModuleConfigLoaderTestCase.java deleted file mode 100644 index 85f802f23..000000000 --- a/apm-collector/apm-collector-core/src/test/java/org/skywalking/apm/collector/core/config/ModuleConfigLoaderTestCase.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.skywalking.apm.collector.core.config; - -import java.io.FileNotFoundException; -import org.junit.Test; -import org.skywalking.apm.collector.core.module.ModuleConfigLoader; -import org.skywalking.apm.collector.core.module.ModuleConfigLoaderException; - -/** - * @author pengys5 - */ -public class ModuleConfigLoaderTestCase { - - @Test - public void testLoad() throws ModuleConfigLoaderException { - ModuleConfigLoader loader = new ModuleConfigLoader(); - loader.load(); - } -} diff --git a/apm-collector/apm-collector-discovery/pom.xml b/apm-collector/apm-collector-discovery/pom.xml deleted file mode 100644 index 5aba21592..000000000 --- a/apm-collector/apm-collector-discovery/pom.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - apm-collector - org.skywalking - 3.2-2017 - - 4.0.0 - - apm-collector-discovery - jar - \ No newline at end of file diff --git a/apm-collector/apm-collector-discovery/src/main/java/org/skywalking/apm/collector/discovery/DiscoveryJettyModuleDefine.java b/apm-collector/apm-collector-discovery/src/main/java/org/skywalking/apm/collector/discovery/DiscoveryJettyModuleDefine.java deleted file mode 100644 index 3d696fa9e..000000000 --- a/apm-collector/apm-collector-discovery/src/main/java/org/skywalking/apm/collector/discovery/DiscoveryJettyModuleDefine.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.skywalking.apm.collector.discovery; - -/** - * @author pengys5 - */ -public class DiscoveryJettyModuleDefine { -} diff --git a/apm-collector/apm-collector-discovery/src/main/resources/META-INF/defines/module.define b/apm-collector/apm-collector-discovery/src/main/resources/META-INF/defines/module.define deleted file mode 100644 index 427dbc6e4..000000000 --- a/apm-collector/apm-collector-discovery/src/main/resources/META-INF/defines/module.define +++ /dev/null @@ -1 +0,0 @@ -org.skywalking.apm.collector.discovery.DiscoveryJettyModuleDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/QueueModuleContext.java b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/QueueModuleContext.java new file mode 100644 index 000000000..cf9bf00cd --- /dev/null +++ b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/QueueModuleContext.java @@ -0,0 +1,23 @@ +package org.skywalking.apm.collector.queue; + +import org.skywalking.apm.collector.core.framework.Context; +import org.skywalking.apm.collector.core.queue.QueueCreator; + +/** + * @author pengys5 + */ +public class QueueModuleContext extends Context { + private QueueCreator queueCreator; + + public QueueModuleContext(String groupName) { + super(groupName); + } + + public QueueCreator getQueueCreator() { + return queueCreator; + } + + public void setQueueCreator(QueueCreator queueCreator) { + this.queueCreator = queueCreator; + } +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/queue/QueueModuleDefine.java b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/QueueModuleDefine.java similarity index 73% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/queue/QueueModuleDefine.java rename to apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/QueueModuleDefine.java index 9b6bd447f..88bbae0c7 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/queue/QueueModuleDefine.java +++ b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/QueueModuleDefine.java @@ -1,7 +1,7 @@ -package org.skywalking.apm.collector.core.queue; +package org.skywalking.apm.collector.queue; import org.skywalking.apm.collector.core.client.Client; -import org.skywalking.apm.collector.core.framework.DataInitializer; +import org.skywalking.apm.collector.core.client.DataMonitor; import org.skywalking.apm.collector.core.module.ModuleConfigParser; import org.skywalking.apm.collector.core.module.ModuleDefine; import org.skywalking.apm.collector.core.module.ModuleRegistration; @@ -15,11 +15,7 @@ public abstract class QueueModuleDefine extends ModuleDefine { throw new UnsupportedOperationException(""); } - @Override protected final Client createClient() { - throw new UnsupportedOperationException(""); - } - - @Override protected final DataInitializer dataInitializer() { + @Override protected Client createClient(DataMonitor dataMonitor) { throw new UnsupportedOperationException(""); } diff --git a/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/QueueModuleException.java b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/QueueModuleException.java new file mode 100644 index 000000000..b22b24089 --- /dev/null +++ b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/QueueModuleException.java @@ -0,0 +1,16 @@ +package org.skywalking.apm.collector.queue; + +import org.skywalking.apm.collector.core.module.ModuleException; + +/** + * @author pengys5 + */ +public class QueueModuleException extends ModuleException { + public QueueModuleException(String message) { + super(message); + } + + public QueueModuleException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/QueueModuleGroupDefine.java b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/QueueModuleGroupDefine.java new file mode 100644 index 000000000..9b40b7c2c --- /dev/null +++ b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/QueueModuleGroupDefine.java @@ -0,0 +1,25 @@ +package org.skywalking.apm.collector.queue; + +import org.skywalking.apm.collector.core.framework.Context; +import org.skywalking.apm.collector.core.module.ModuleGroupDefine; +import org.skywalking.apm.collector.core.module.ModuleInstaller; + +/** + * @author pengys5 + */ +public class QueueModuleGroupDefine implements ModuleGroupDefine { + + public static final String GROUP_NAME = "queue"; + + @Override public String name() { + return GROUP_NAME; + } + + @Override public Context groupContext() { + return new QueueModuleContext(GROUP_NAME); + } + + @Override public ModuleInstaller moduleInstaller() { + return new QueueModuleInstaller(); + } +} diff --git a/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/QueueModuleInstaller.java b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/QueueModuleInstaller.java new file mode 100644 index 000000000..b0ed3238a --- /dev/null +++ b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/QueueModuleInstaller.java @@ -0,0 +1,28 @@ +package org.skywalking.apm.collector.queue; + +import java.util.Map; +import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.module.ModuleDefine; +import org.skywalking.apm.collector.core.module.SingleModuleInstaller; +import org.skywalking.apm.collector.core.server.ServerHolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class QueueModuleInstaller extends SingleModuleInstaller { + + private final Logger logger = LoggerFactory.getLogger(QueueModuleInstaller.class); + + @Override public void install(Map moduleConfig, + Map moduleDefineMap, ServerHolder serverHolder) throws DefineException, ClientException { + logger.info("beginning queue module install"); + QueueModuleContext context = new QueueModuleContext(QueueModuleGroupDefine.GROUP_NAME); + CollectorContextHelper.INSTANCE.putContext(context); + + installSingle(moduleConfig, moduleDefineMap, serverHolder); + } +} diff --git a/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/datacarrier/DataCarrierCreator.java b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/datacarrier/DataCarrierCreator.java deleted file mode 100644 index 2402b83f4..000000000 --- a/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/datacarrier/DataCarrierCreator.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.skywalking.apm.collector.queue.datacarrier; - -import org.skywalking.apm.collector.core.queue.Creator; -import org.skywalking.apm.collector.core.queue.QueueEventHandler; -import org.skywalking.apm.collector.core.worker.AbstractLocalAsyncWorker; - -/** - * @author pengys5 - */ -public class DataCarrierCreator implements Creator { - - @Override public QueueEventHandler create(int queueSize, AbstractLocalAsyncWorker localAsyncWorker) { - return null; - } -} diff --git a/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/datacarrier/DataCarrierQueueCreator.java b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/datacarrier/DataCarrierQueueCreator.java new file mode 100644 index 000000000..e8593d6fc --- /dev/null +++ b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/datacarrier/DataCarrierQueueCreator.java @@ -0,0 +1,15 @@ +package org.skywalking.apm.collector.queue.datacarrier; + +import org.skywalking.apm.collector.core.queue.QueueCreator; +import org.skywalking.apm.collector.core.queue.QueueEventHandler; +import org.skywalking.apm.collector.core.queue.QueueExecutor; + +/** + * @author pengys5 + */ +public class DataCarrierQueueCreator implements QueueCreator { + + @Override public QueueEventHandler create(int queueSize, QueueExecutor executor) { + return null; + } +} diff --git a/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/datacarrier/QueueDataCarrierModuleDefine.java b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/datacarrier/QueueDataCarrierModuleDefine.java index 572557711..af5db748a 100644 --- a/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/datacarrier/QueueDataCarrierModuleDefine.java +++ b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/datacarrier/QueueDataCarrierModuleDefine.java @@ -2,18 +2,20 @@ package org.skywalking.apm.collector.queue.datacarrier; import java.util.Map; import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; import org.skywalking.apm.collector.core.framework.DefineException; -import org.skywalking.apm.collector.core.module.ModuleGroup; -import org.skywalking.apm.collector.core.queue.QueueModuleContext; -import org.skywalking.apm.collector.core.queue.QueueModuleDefine; +import org.skywalking.apm.collector.core.server.ServerHolder; +import org.skywalking.apm.collector.queue.QueueModuleContext; +import org.skywalking.apm.collector.queue.QueueModuleDefine; +import org.skywalking.apm.collector.queue.QueueModuleGroupDefine; /** * @author pengys5 */ public class QueueDataCarrierModuleDefine extends QueueModuleDefine { - @Override protected ModuleGroup group() { - return ModuleGroup.Queue; + @Override protected String group() { + return QueueModuleGroupDefine.GROUP_NAME; } @Override public String name() { @@ -21,10 +23,11 @@ public class QueueDataCarrierModuleDefine extends QueueModuleDefine { } @Override public boolean defaultModule() { - return true; + return false; } - @Override public final void initialize(Map config) throws DefineException, ClientException { - QueueModuleContext.CREATOR = new DataCarrierCreator(); + @Override + public final void initialize(Map config, ServerHolder serverHolder) throws DefineException, ClientException { + ((QueueModuleContext)CollectorContextHelper.INSTANCE.getContext(group())).setQueueCreator(new DataCarrierQueueCreator()); } } diff --git a/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/disruptor/DisruptorEventHandler.java b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/disruptor/DisruptorEventHandler.java index 34090fa08..f25396531 100644 --- a/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/disruptor/DisruptorEventHandler.java +++ b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/disruptor/DisruptorEventHandler.java @@ -5,8 +5,7 @@ import com.lmax.disruptor.RingBuffer; import org.skywalking.apm.collector.core.queue.EndOfBatchCommand; import org.skywalking.apm.collector.core.queue.MessageHolder; import org.skywalking.apm.collector.core.queue.QueueEventHandler; -import org.skywalking.apm.collector.core.worker.AbstractLocalAsyncWorker; -import org.skywalking.apm.collector.core.worker.WorkerException; +import org.skywalking.apm.collector.core.queue.QueueExecutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,11 +17,11 @@ public class DisruptorEventHandler implements EventHandler, Queue private final Logger logger = LoggerFactory.getLogger(DisruptorEventHandler.class); private RingBuffer ringBuffer; - private AbstractLocalAsyncWorker asyncWorker; + private QueueExecutor executor; - DisruptorEventHandler(RingBuffer ringBuffer, AbstractLocalAsyncWorker asyncWorker) { + DisruptorEventHandler(RingBuffer ringBuffer, QueueExecutor executor) { this.ringBuffer = ringBuffer; - this.asyncWorker = asyncWorker; + this.executor = executor; } /** @@ -34,16 +33,12 @@ public class DisruptorEventHandler implements EventHandler, Queue * @param endOfBatch flag to indicate if this is the last event in a batch from the {@link RingBuffer} */ public void onEvent(MessageHolder event, long sequence, boolean endOfBatch) { - try { - Object message = event.getMessage(); - event.reset(); + Object message = event.getMessage(); + event.reset(); - asyncWorker.allocateJob(message); - if (endOfBatch) { - asyncWorker.allocateJob(new EndOfBatchCommand()); - } - } catch (WorkerException e) { - logger.error(e.getMessage(), e); + executor.execute(message); + if (endOfBatch) { + executor.execute(new EndOfBatchCommand()); } } diff --git a/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/disruptor/DisruptorCreator.java b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/disruptor/DisruptorQueueCreator.java similarity index 77% rename from apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/disruptor/DisruptorCreator.java rename to apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/disruptor/DisruptorQueueCreator.java index 8be095b26..df027d0e6 100644 --- a/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/disruptor/DisruptorCreator.java +++ b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/disruptor/DisruptorQueueCreator.java @@ -2,18 +2,18 @@ package org.skywalking.apm.collector.queue.disruptor; import com.lmax.disruptor.RingBuffer; import com.lmax.disruptor.dsl.Disruptor; -import org.skywalking.apm.collector.core.queue.Creator; import org.skywalking.apm.collector.core.queue.DaemonThreadFactory; import org.skywalking.apm.collector.core.queue.MessageHolder; +import org.skywalking.apm.collector.core.queue.QueueCreator; import org.skywalking.apm.collector.core.queue.QueueEventHandler; -import org.skywalking.apm.collector.core.worker.AbstractLocalAsyncWorker; +import org.skywalking.apm.collector.core.queue.QueueExecutor; /** * @author pengys5 */ -public class DisruptorCreator implements Creator { +public class DisruptorQueueCreator implements QueueCreator { - public QueueEventHandler create(int queueSize, AbstractLocalAsyncWorker localAsyncWorker) { + @Override public QueueEventHandler create(int queueSize, QueueExecutor executor) { // Specify the size of the ring buffer, must be power of 2. if (!((((queueSize - 1) & queueSize) == 0) && queueSize != 0)) { throw new IllegalArgumentException("queue size must be power of 2"); @@ -23,7 +23,7 @@ public class DisruptorCreator implements Creator { Disruptor disruptor = new Disruptor(MessageHolderFactory.INSTANCE, queueSize, DaemonThreadFactory.INSTANCE); RingBuffer ringBuffer = disruptor.getRingBuffer(); - DisruptorEventHandler eventHandler = new DisruptorEventHandler(ringBuffer, localAsyncWorker); + DisruptorEventHandler eventHandler = new DisruptorEventHandler(ringBuffer, executor); // Connect the handler disruptor.handleEventsWith(eventHandler); diff --git a/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/disruptor/QueueDisruptorModuleDefine.java b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/disruptor/QueueDisruptorModuleDefine.java index cae0f1325..d74cb2dee 100644 --- a/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/disruptor/QueueDisruptorModuleDefine.java +++ b/apm-collector/apm-collector-queue/src/main/java/org/skywalking/apm/collector/queue/disruptor/QueueDisruptorModuleDefine.java @@ -2,18 +2,20 @@ package org.skywalking.apm.collector.queue.disruptor; import java.util.Map; import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; import org.skywalking.apm.collector.core.framework.DefineException; -import org.skywalking.apm.collector.core.module.ModuleGroup; -import org.skywalking.apm.collector.core.queue.QueueModuleContext; -import org.skywalking.apm.collector.core.queue.QueueModuleDefine; +import org.skywalking.apm.collector.core.server.ServerHolder; +import org.skywalking.apm.collector.queue.QueueModuleContext; +import org.skywalking.apm.collector.queue.QueueModuleDefine; +import org.skywalking.apm.collector.queue.QueueModuleGroupDefine; /** * @author pengys5 */ public class QueueDisruptorModuleDefine extends QueueModuleDefine { - @Override protected ModuleGroup group() { - return ModuleGroup.Queue; + @Override protected String group() { + return QueueModuleGroupDefine.GROUP_NAME; } @Override public String name() { @@ -24,7 +26,8 @@ public class QueueDisruptorModuleDefine extends QueueModuleDefine { return true; } - @Override public final void initialize(Map config) throws DefineException, ClientException { - QueueModuleContext.CREATOR = new DisruptorCreator(); + @Override + public final void initialize(Map config, ServerHolder serverHolder) throws DefineException, ClientException { + ((QueueModuleContext)CollectorContextHelper.INSTANCE.getContext(group())).setQueueCreator(new DisruptorQueueCreator()); } } diff --git a/apm-collector/apm-collector-queue/src/main/resources/META-INF/defines/group.define b/apm-collector/apm-collector-queue/src/main/resources/META-INF/defines/group.define new file mode 100644 index 000000000..761c2b7e0 --- /dev/null +++ b/apm-collector/apm-collector-queue/src/main/resources/META-INF/defines/group.define @@ -0,0 +1 @@ +org.skywalking.apm.collector.queue.QueueModuleGroupDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCModuleDefine.java b/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCModuleDefine.java deleted file mode 100644 index 30b74e893..000000000 --- a/apm-collector/apm-collector-remote/src/main/java/org/skywalking/apm/collector/remote/grpc/RemoteGRPCModuleDefine.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.skywalking.apm.collector.remote.grpc; - -import java.util.Map; -import org.skywalking.apm.collector.core.client.Client; -import org.skywalking.apm.collector.core.client.ClientException; -import org.skywalking.apm.collector.core.framework.DataInitializer; -import org.skywalking.apm.collector.core.framework.DefineException; -import org.skywalking.apm.collector.core.module.ModuleConfigParser; -import org.skywalking.apm.collector.core.module.ModuleGroup; -import org.skywalking.apm.collector.core.module.ModuleRegistration; -import org.skywalking.apm.collector.core.remote.RemoteModuleDefine; -import org.skywalking.apm.collector.core.server.Server; - -/** - * @author pengys5 - */ -public class RemoteGRPCModuleDefine extends RemoteModuleDefine { - @Override protected ModuleGroup group() { - return ModuleGroup.Queue; - } - - @Override public boolean defaultModule() { - return false; - } - - @Override protected ModuleConfigParser configParser() { - return null; - } - - @Override protected Client createClient() { - return null; - } - - @Override protected Server server() { - return null; - } - - @Override protected DataInitializer dataInitializer() { - return null; - } - - @Override protected ModuleRegistration registration() { - return null; - } - - @Override public void initialize(Map config) throws DefineException, ClientException { - - } - - @Override public String name() { - return null; - } -} diff --git a/apm-collector/apm-collector-remote/src/main/proto/RemoteCommonService.proto b/apm-collector/apm-collector-remote/src/main/proto/RemoteCommonService.proto deleted file mode 100644 index ae1da50f0..000000000 --- a/apm-collector/apm-collector-remote/src/main/proto/RemoteCommonService.proto +++ /dev/null @@ -1,18 +0,0 @@ -syntax = "proto3"; - -option java_multiple_files = true; -option java_package = "org.skywalking.apm.collector.remote.grpc.proto"; - -service RemoteCommonService { - rpc call (Message) returns (Empty) { - } -} - -message Message { - string workerRole = 1; - int32 objectId = 2; - bytes objectBytes = 3; // the byte array of data object -} - -message Empty { -} \ No newline at end of file diff --git a/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/grpc/GRPCHandler.java b/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/grpc/GRPCHandler.java new file mode 100644 index 000000000..7a6dd08bf --- /dev/null +++ b/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/grpc/GRPCHandler.java @@ -0,0 +1,9 @@ +package org.skywalking.apm.collector.server.grpc; + +import org.skywalking.apm.collector.core.framework.Handler; + +/** + * @author pengys5 + */ +public interface GRPCHandler extends Handler { +} diff --git a/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/grpc/GRPCServer.java b/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/grpc/GRPCServer.java index 309901ade..6f9472317 100644 --- a/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/grpc/GRPCServer.java +++ b/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/grpc/GRPCServer.java @@ -3,6 +3,7 @@ package org.skywalking.apm.collector.server.grpc; import io.grpc.netty.NettyServerBuilder; import java.io.IOException; import java.net.InetSocketAddress; +import org.skywalking.apm.collector.core.framework.Handler; import org.skywalking.apm.collector.core.server.Server; import org.skywalking.apm.collector.core.server.ServerException; import org.slf4j.Logger; @@ -17,27 +18,38 @@ public class GRPCServer implements Server { private final String host; private final int port; + private io.grpc.Server server; + private NettyServerBuilder nettyServerBuilder; public GRPCServer(String host, int port) { this.host = host; this.port = port; } + @Override public String hostPort() { + return host + ":" + port; + } + + @Override public String serverClassify() { + return "Google-RPC"; + } + @Override public void initialize() throws ServerException { InetSocketAddress address = new InetSocketAddress(host, port); - NettyServerBuilder nettyServerBuilder = NettyServerBuilder.forAddress(address); - try { - io.grpc.Server server = nettyServerBuilder.build().start(); - blockUntilShutdown(server); - } catch (InterruptedException | IOException e) { - throw new GRPCServerException(e.getMessage(), e); - } + nettyServerBuilder = NettyServerBuilder.forAddress(address); logger.info("Server started, host {} listening on {}", host, port); } - private void blockUntilShutdown(io.grpc.Server server) throws InterruptedException { - if (server != null) { - server.awaitTermination(); + @Override public void start() throws ServerException { + try { + server = nettyServerBuilder.build(); + server.start(); + } catch (IOException e) { + throw new GRPCServerException(e.getMessage(), e); } } + + @Override public void addHandler(Handler handler) { + nettyServerBuilder.addService((io.grpc.BindableService)handler); + } } diff --git a/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/jetty/ArgumentsParseException.java b/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/jetty/ArgumentsParseException.java new file mode 100644 index 000000000..e7148b58d --- /dev/null +++ b/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/jetty/ArgumentsParseException.java @@ -0,0 +1,17 @@ +package org.skywalking.apm.collector.server.jetty; + +import org.skywalking.apm.collector.core.CollectorException; + +/** + * @author pengys5 + */ +public class ArgumentsParseException extends CollectorException { + + public ArgumentsParseException(String message) { + super(message); + } + + public ArgumentsParseException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/jetty/JettyHandler.java b/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/jetty/JettyHandler.java new file mode 100644 index 000000000..7b7c39bc5 --- /dev/null +++ b/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/jetty/JettyHandler.java @@ -0,0 +1,147 @@ +package org.skywalking.apm.collector.server.jetty; + +import com.google.gson.JsonElement; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Enumeration; +import javax.servlet.ServletConfig; +import javax.servlet.ServletContext; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.skywalking.apm.collector.core.framework.Handler; + +/** + * @author pengys5 + */ +public abstract class JettyHandler extends HttpServlet implements Handler { + + public abstract String pathSpec(); + + @Override + protected final void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + try { + reply(resp, doGet(req)); + } catch (ArgumentsParseException e) { + replyError(resp, e.getMessage(), HttpServletResponse.SC_BAD_REQUEST); + } + } + + protected abstract JsonElement doGet(HttpServletRequest req) throws ArgumentsParseException; + + @Override + protected final void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + super.doPost(req, resp); + } + + @Override + protected final void doHead(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + super.doHead(req, resp); + } + + @Override protected final long getLastModified(HttpServletRequest req) { + return super.getLastModified(req); + } + + @Override + protected final void doPut(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + super.doPut(req, resp); + } + + @Override + protected final void doDelete(HttpServletRequest req, + HttpServletResponse resp) throws ServletException, IOException { + super.doDelete(req, resp); + } + + @Override + protected final void doOptions(HttpServletRequest req, + HttpServletResponse resp) throws ServletException, IOException { + super.doOptions(req, resp); + } + + @Override + protected final void doTrace(HttpServletRequest req, + HttpServletResponse resp) throws ServletException, IOException { + super.doTrace(req, resp); + } + + @Override + protected final void service(HttpServletRequest req, + HttpServletResponse resp) throws ServletException, IOException { + super.service(req, resp); + } + + @Override public final void service(ServletRequest req, ServletResponse res) throws ServletException, IOException { + super.service(req, res); + } + + @Override public final void destroy() { + super.destroy(); + } + + @Override public final String getInitParameter(String name) { + return super.getInitParameter(name); + } + + @Override public final Enumeration getInitParameterNames() { + return super.getInitParameterNames(); + } + + @Override public final ServletConfig getServletConfig() { + return super.getServletConfig(); + } + + @Override public final ServletContext getServletContext() { + return super.getServletContext(); + } + + @Override public final String getServletInfo() { + return super.getServletInfo(); + } + + @Override public final void init(ServletConfig config) throws ServletException { + super.init(config); + } + + @Override public final void init() throws ServletException { + super.init(); + } + + @Override public final void log(String msg) { + super.log(msg); + } + + @Override public final void log(String message, Throwable t) { + super.log(message, t); + } + + @Override public final String getServletName() { + return super.getServletName(); + } + + private void reply(HttpServletResponse response, JsonElement resJson) throws IOException { + response.setContentType("text/json"); + response.setCharacterEncoding("utf-8"); + response.setStatus(HttpServletResponse.SC_OK); + + PrintWriter out = response.getWriter(); + out.print(resJson); + out.flush(); + out.close(); + } + + private void replyError(HttpServletResponse response, String errorMessage, int status) throws IOException { + response.setContentType("text/plain"); + response.setCharacterEncoding("utf-8"); + response.setStatus(status); + + PrintWriter out = response.getWriter(); + out.print(errorMessage); + out.flush(); + out.close(); + } +} diff --git a/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/jetty/JettyServer.java b/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/jetty/JettyServer.java index 984e76269..441a3b16a 100644 --- a/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/jetty/JettyServer.java +++ b/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/jetty/JettyServer.java @@ -1,7 +1,10 @@ package org.skywalking.apm.collector.server.jetty; import java.net.InetSocketAddress; +import javax.servlet.http.HttpServlet; import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.skywalking.apm.collector.core.framework.Handler; import org.skywalking.apm.collector.core.server.Server; import org.skywalking.apm.collector.core.server.ServerException; import org.slf4j.Logger; @@ -17,6 +20,8 @@ public class JettyServer implements Server { private final String host; private final int port; private final String contextPath; + private org.eclipse.jetty.server.Server server; + private ServletContextHandler servletContextHandler; public JettyServer(String host, int port, String contextPath) { this.host = host; @@ -24,14 +29,31 @@ public class JettyServer implements Server { this.contextPath = contextPath; } - @Override public void initialize() throws ServerException { - org.eclipse.jetty.server.Server server = new org.eclipse.jetty.server.Server(new InetSocketAddress(host, port)); + @Override public String hostPort() { + return host + ":" + port; + } - ServletContextHandler servletContextHandler = new ServletContextHandler(ServletContextHandler.NO_SESSIONS); + @Override public String serverClassify() { + return "Jetty"; + } + + @Override public void initialize() throws ServerException { + server = new org.eclipse.jetty.server.Server(new InetSocketAddress(host, port)); + + servletContextHandler = new ServletContextHandler(ServletContextHandler.NO_SESSIONS); servletContextHandler.setContextPath(contextPath); logger.info("http server root context path: {}", contextPath); server.setHandler(servletContextHandler); + } + + @Override public void addHandler(Handler handler) { + ServletHolder servletHolder = new ServletHolder(); + servletHolder.setServlet((HttpServlet)handler); + servletContextHandler.addServlet(servletHolder, ((JettyHandler)handler).pathSpec()); + } + + @Override public void start() throws ServerException { try { server.start(); } catch (Exception e) { diff --git a/apm-collector/apm-collector-storage/elasticsearch-storage/pom.xml b/apm-collector/apm-collector-storage/elasticsearch-storage/pom.xml deleted file mode 100644 index 94b2517c0..000000000 --- a/apm-collector/apm-collector-storage/elasticsearch-storage/pom.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - apm-collector-storage - org.skywalking - 3.2-2017 - - 4.0.0 - - elasticsearch-storage - jar - \ No newline at end of file diff --git a/apm-collector/apm-collector-storage/h2-storage/pom.xml b/apm-collector/apm-collector-storage/h2-storage/pom.xml deleted file mode 100644 index 8bc63b3f0..000000000 --- a/apm-collector/apm-collector-storage/h2-storage/pom.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - apm-collector-storage - org.skywalking - 3.2-2017 - - 4.0.0 - - h2-storage - jar - \ No newline at end of file diff --git a/apm-collector/apm-collector-storage/pom.xml b/apm-collector/apm-collector-storage/pom.xml index 6ade3218b..144648e27 100644 --- a/apm-collector/apm-collector-storage/pom.xml +++ b/apm-collector/apm-collector-storage/pom.xml @@ -10,9 +10,18 @@ 4.0.0 apm-collector-storage - pom - - h2-storage - elasticsearch-storage - + jar + + + + org.skywalking + apm-collector-core + ${project.version} + + + org.skywalking + apm-collector-cluster + ${project.version} + + \ No newline at end of file diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/StorageModuleContext.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/StorageModuleContext.java new file mode 100644 index 000000000..128a2fdf8 --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/StorageModuleContext.java @@ -0,0 +1,24 @@ +package org.skywalking.apm.collector.storage; + +import org.skywalking.apm.collector.core.client.Client; +import org.skywalking.apm.collector.core.framework.Context; + +/** + * @author pengys5 + */ +public class StorageModuleContext extends Context { + + private Client client; + + public StorageModuleContext(String groupName) { + super(groupName); + } + + public Client getClient() { + return client; + } + + public void setClient(Client client) { + this.client = client; + } +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/StorageModuleDefine.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/StorageModuleDefine.java new file mode 100644 index 000000000..57ccb77e9 --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/StorageModuleDefine.java @@ -0,0 +1,63 @@ +package org.skywalking.apm.collector.storage; + +import java.util.Map; +import org.skywalking.apm.collector.core.client.Client; +import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.cluster.ClusterDataListener; +import org.skywalking.apm.collector.core.cluster.ClusterDataListenerDefine; +import org.skywalking.apm.collector.core.config.ConfigParseException; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.module.ModuleDefine; +import org.skywalking.apm.collector.core.module.ModuleRegistration; +import org.skywalking.apm.collector.core.server.Server; +import org.skywalking.apm.collector.core.server.ServerHolder; +import org.skywalking.apm.collector.core.storage.StorageException; +import org.skywalking.apm.collector.core.storage.StorageInstaller; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public abstract class StorageModuleDefine extends ModuleDefine implements ClusterDataListenerDefine { + + private final Logger logger = LoggerFactory.getLogger(StorageModuleDefine.class); + + @Override + public final void initialize(Map config, ServerHolder serverHolder) throws DefineException, ClientException { + try { + configParser().parse(config); + + StorageModuleContext context = (StorageModuleContext)CollectorContextHelper.INSTANCE.getContext(StorageModuleGroupDefine.GROUP_NAME); + Client client = createClient(null); + client.initialize(); + context.setClient(client); + injectClientIntoDAO(client); + + storageInstaller().install(client); + } catch (ConfigParseException | StorageException e) { + throw new StorageModuleException(e.getMessage(), e); + } + } + + @Override protected final Server server() { + throw new UnsupportedOperationException(""); + } + + @Override protected final ModuleRegistration registration() { + throw new UnsupportedOperationException(""); + } + + @Override public final ClusterDataListener listener() { + throw new UnsupportedOperationException(""); + } + + @Override public final boolean defaultModule() { + return true; + } + + public abstract StorageInstaller storageInstaller(); + + public abstract void injectClientIntoDAO(Client client) throws DefineException; +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/StorageModuleException.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/StorageModuleException.java new file mode 100644 index 000000000..4136c9fa4 --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/StorageModuleException.java @@ -0,0 +1,16 @@ +package org.skywalking.apm.collector.storage; + +import org.skywalking.apm.collector.core.module.ModuleException; + +/** + * @author pengys5 + */ +public class StorageModuleException extends ModuleException { + public StorageModuleException(String message) { + super(message); + } + + public StorageModuleException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/StorageModuleGroupDefine.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/StorageModuleGroupDefine.java new file mode 100644 index 000000000..c373f3a83 --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/StorageModuleGroupDefine.java @@ -0,0 +1,25 @@ +package org.skywalking.apm.collector.storage; + +import org.skywalking.apm.collector.core.framework.Context; +import org.skywalking.apm.collector.core.module.ModuleGroupDefine; +import org.skywalking.apm.collector.core.module.ModuleInstaller; + +/** + * @author pengys5 + */ +public class StorageModuleGroupDefine implements ModuleGroupDefine { + + public static final String GROUP_NAME = "storage"; + + @Override public String name() { + return GROUP_NAME; + } + + @Override public Context groupContext() { + return new StorageModuleContext(GROUP_NAME); + } + + @Override public ModuleInstaller moduleInstaller() { + return new StorageModuleInstaller(); + } +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/StorageModuleInstaller.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/StorageModuleInstaller.java new file mode 100644 index 000000000..f15b9861c --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/StorageModuleInstaller.java @@ -0,0 +1,29 @@ +package org.skywalking.apm.collector.storage; + +import java.util.Map; +import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.module.ModuleDefine; +import org.skywalking.apm.collector.core.module.SingleModuleInstaller; +import org.skywalking.apm.collector.core.server.ServerHolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class StorageModuleInstaller extends SingleModuleInstaller { + + private final Logger logger = LoggerFactory.getLogger(StorageModuleInstaller.class); + + @Override public void install(Map moduleConfig, + Map moduleDefineMap, ServerHolder serverHolder) throws DefineException, ClientException { + logger.info("beginning storage module install"); + + StorageModuleContext context = new StorageModuleContext(StorageModuleGroupDefine.GROUP_NAME); + CollectorContextHelper.INSTANCE.putContext(context); + + installSingle(moduleConfig, moduleDefineMap, serverHolder); + } +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/DAO.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/DAO.java new file mode 100644 index 000000000..1ddeb9445 --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/DAO.java @@ -0,0 +1,18 @@ +package org.skywalking.apm.collector.storage.dao; + +import org.skywalking.apm.collector.core.client.Client; + +/** + * @author pengys5 + */ +public abstract class DAO { + private C client; + + public final C getClient() { + return client; + } + + public final void setClient(C client) { + this.client = client; + } +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/DAOContainer.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/DAOContainer.java new file mode 100644 index 000000000..a4f884239 --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/DAOContainer.java @@ -0,0 +1,21 @@ +package org.skywalking.apm.collector.storage.dao; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author pengys5 + */ +public enum DAOContainer { + INSTANCE; + + private Map daos = new HashMap<>(); + + public void put(String interfaceName, DAO dao) { + daos.put(interfaceName, dao); + } + + public DAO get(String interfaceName) { + return daos.get(interfaceName); + } +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/IBatchDAO.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/IBatchDAO.java new file mode 100644 index 000000000..afcac1bf1 --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/dao/IBatchDAO.java @@ -0,0 +1,10 @@ +package org.skywalking.apm.collector.storage.dao; + +import java.util.List; + +/** + * @author pengys5 + */ +public interface IBatchDAO { + void batchPersistence(List batchCollection); +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/ElasticSearchStorageException.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/ElasticSearchStorageException.java new file mode 100644 index 000000000..be95df896 --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/ElasticSearchStorageException.java @@ -0,0 +1,16 @@ +package org.skywalking.apm.collector.storage.elasticsearch; + +import org.skywalking.apm.collector.core.storage.StorageException; + +/** + * @author pengys5 + */ +public class ElasticSearchStorageException extends StorageException { + public ElasticSearchStorageException(String message) { + super(message); + } + + public ElasticSearchStorageException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/StorageElasticSearchConfig.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/StorageElasticSearchConfig.java new file mode 100644 index 000000000..3526fc98b --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/StorageElasticSearchConfig.java @@ -0,0 +1,10 @@ +package org.skywalking.apm.collector.storage.elasticsearch; + +/** + * @author pengys5 + */ +public class StorageElasticSearchConfig { + public static String CLUSTER_NAME; + public static Boolean CLUSTER_TRANSPORT_SNIFFER; + public static String CLUSTER_NODES; +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/StorageElasticSearchConfigParser.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/StorageElasticSearchConfigParser.java new file mode 100644 index 000000000..6653fdc22 --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/StorageElasticSearchConfigParser.java @@ -0,0 +1,29 @@ +package org.skywalking.apm.collector.storage.elasticsearch; + +import java.util.Map; +import org.skywalking.apm.collector.core.config.ConfigParseException; +import org.skywalking.apm.collector.core.module.ModuleConfigParser; +import org.skywalking.apm.collector.core.util.ObjectUtils; +import org.skywalking.apm.collector.core.util.StringUtils; + +/** + * @author pengys5 + */ +public class StorageElasticSearchConfigParser implements ModuleConfigParser { + + private static final String CLUSTER_NAME = "cluster_name"; + private static final String CLUSTER_TRANSPORT_SNIFFER = "cluster_transport_sniffer"; + private static final String CLUSTER_NODES = "cluster_nodes"; + + @Override public void parse(Map config) throws ConfigParseException { + if (ObjectUtils.isNotEmpty(config) && StringUtils.isNotEmpty(config.get(CLUSTER_NAME))) { + StorageElasticSearchConfig.CLUSTER_NAME = (String)config.get(CLUSTER_NAME); + } + if (ObjectUtils.isNotEmpty(config) && StringUtils.isNotEmpty(config.get(CLUSTER_TRANSPORT_SNIFFER))) { + StorageElasticSearchConfig.CLUSTER_TRANSPORT_SNIFFER = (Boolean)config.get(CLUSTER_TRANSPORT_SNIFFER); + } + if (ObjectUtils.isNotEmpty(config) && StringUtils.isNotEmpty(config.get(CLUSTER_NODES))) { + StorageElasticSearchConfig.CLUSTER_NODES = (String)config.get(CLUSTER_NODES); + } + } +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/StorageElasticSearchModuleDefine.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/StorageElasticSearchModuleDefine.java new file mode 100644 index 000000000..826ad4cb0 --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/StorageElasticSearchModuleDefine.java @@ -0,0 +1,53 @@ +package org.skywalking.apm.collector.storage.elasticsearch; + +import java.util.List; +import org.skywalking.apm.collector.client.elasticsearch.ElasticSearchClient; +import org.skywalking.apm.collector.core.client.Client; +import org.skywalking.apm.collector.core.client.DataMonitor; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.module.ModuleConfigParser; +import org.skywalking.apm.collector.core.storage.StorageInstaller; +import org.skywalking.apm.collector.storage.StorageModuleDefine; +import org.skywalking.apm.collector.storage.StorageModuleGroupDefine; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAODefineLoader; +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchStorageInstaller; + +/** + * @author pengys5 + */ +public class StorageElasticSearchModuleDefine extends StorageModuleDefine { + + public static final String MODULE_NAME = "elasticsearch"; + + @Override protected String group() { + return StorageModuleGroupDefine.GROUP_NAME; + } + + @Override public String name() { + return MODULE_NAME; + } + + @Override protected ModuleConfigParser configParser() { + return new StorageElasticSearchConfigParser(); + } + + @Override protected Client createClient(DataMonitor dataMonitor) { + return new ElasticSearchClient(StorageElasticSearchConfig.CLUSTER_NAME, StorageElasticSearchConfig.CLUSTER_TRANSPORT_SNIFFER, StorageElasticSearchConfig.CLUSTER_NODES); + } + + @Override public StorageInstaller storageInstaller() { + return new ElasticSearchStorageInstaller(); + } + + @Override public void injectClientIntoDAO(Client client) throws DefineException { + EsDAODefineLoader loader = new EsDAODefineLoader(); + List esDAOs = loader.load(); + esDAOs.forEach(esDAO -> { + esDAO.setClient((ElasticSearchClient)client); + String interFaceName = esDAO.getClass().getInterfaces()[0].getName(); + DAOContainer.INSTANCE.put(interFaceName, esDAO); + }); + } +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/BatchEsDAO.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/BatchEsDAO.java new file mode 100644 index 000000000..74c06e634 --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/BatchEsDAO.java @@ -0,0 +1,35 @@ +package org.skywalking.apm.collector.storage.elasticsearch.dao; + +import java.util.List; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.skywalking.apm.collector.core.util.CollectionUtils; +import org.skywalking.apm.collector.storage.dao.IBatchDAO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class BatchEsDAO extends EsDAO implements IBatchDAO { + + private final Logger logger = LoggerFactory.getLogger(BatchEsDAO.class); + + @Override public void batchPersistence(List batchCollection) { + BulkRequestBuilder bulkRequest = getClient().prepareBulk(); + + logger.info("bulk data size: {}", batchCollection.size()); + if (CollectionUtils.isNotEmpty(batchCollection)) { + for (int i = 0; i < batchCollection.size(); i++) { + IndexRequestBuilder builder = (IndexRequestBuilder)batchCollection.get(i); + bulkRequest.add(builder); + } + + BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + if (bulkResponse.hasFailures()) { + logger.error(bulkResponse.buildFailureMessage()); + } + } + } +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAO.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAO.java new file mode 100644 index 000000000..f6eb18a85 --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAO.java @@ -0,0 +1,55 @@ +package org.skywalking.apm.collector.storage.elasticsearch.dao; + +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.metrics.max.Max; +import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.min.Min; +import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder; +import org.skywalking.apm.collector.client.elasticsearch.ElasticSearchClient; +import org.skywalking.apm.collector.storage.dao.DAO; + +/** + * @author pengys5 + */ +public abstract class EsDAO extends DAO { + + public final int getMaxId(String indexName, String columnName) { + ElasticSearchClient client = getClient(); + SearchRequestBuilder searchRequestBuilder = client.prepareSearch(indexName); + searchRequestBuilder.setTypes("type"); + searchRequestBuilder.setSize(0); + MaxAggregationBuilder aggregation = AggregationBuilders.max("agg").field(columnName); + searchRequestBuilder.addAggregation(aggregation); + + SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); + Max agg = searchResponse.getAggregations().get("agg"); + + int id = (int)agg.getValue(); + if (id == Integer.MAX_VALUE || id == Integer.MIN_VALUE) { + return 0; + } else { + return id; + } + } + + public final int getMinId(String indexName, String columnName) { + ElasticSearchClient client = getClient(); + SearchRequestBuilder searchRequestBuilder = client.prepareSearch(indexName); + searchRequestBuilder.setTypes("type"); + searchRequestBuilder.setSize(0); + MinAggregationBuilder aggregation = AggregationBuilders.min("agg").field(columnName); + searchRequestBuilder.addAggregation(aggregation); + + SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); + Min agg = searchResponse.getAggregations().get("agg"); + + int id = (int)agg.getValue(); + if (id == Integer.MAX_VALUE || id == Integer.MIN_VALUE) { + return 0; + } else { + return id; + } + } +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAODefineLoader.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAODefineLoader.java new file mode 100644 index 000000000..de5213bff --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAODefineLoader.java @@ -0,0 +1,30 @@ +package org.skywalking.apm.collector.storage.elasticsearch.dao; + +import java.util.ArrayList; +import java.util.List; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.framework.Loader; +import org.skywalking.apm.collector.core.util.DefinitionLoader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class EsDAODefineLoader implements Loader> { + + private final Logger logger = LoggerFactory.getLogger(EsDAODefineLoader.class); + + @Override public List load() throws DefineException { + List esDAOs = new ArrayList<>(); + + EsDAODefinitionFile definitionFile = new EsDAODefinitionFile(); + logger.info("elasticsearch dao definition file name: {}", definitionFile.fileName()); + DefinitionLoader definitionLoader = DefinitionLoader.load(EsDAO.class, definitionFile); + for (EsDAO dao : definitionLoader) { + logger.info("loaded elasticsearch dao definition class: {}", dao.getClass().getName()); + esDAOs.add(dao); + } + return esDAOs; + } +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAODefinitionFile.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAODefinitionFile.java new file mode 100644 index 000000000..08791d866 --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/EsDAODefinitionFile.java @@ -0,0 +1,13 @@ +package org.skywalking.apm.collector.storage.elasticsearch.dao; + +import org.skywalking.apm.collector.core.framework.DefinitionFile; + +/** + * @author pengys5 + */ +public class EsDAODefinitionFile extends DefinitionFile { + + @Override protected String fileName() { + return "es_dao.define"; + } +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/define/ElasticSearchColumnDefine.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/define/ElasticSearchColumnDefine.java new file mode 100644 index 000000000..883480a44 --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/define/ElasticSearchColumnDefine.java @@ -0,0 +1,16 @@ +package org.skywalking.apm.collector.storage.elasticsearch.define; + +import org.skywalking.apm.collector.core.storage.ColumnDefine; + +/** + * @author pengys5 + */ +public class ElasticSearchColumnDefine extends ColumnDefine { + public ElasticSearchColumnDefine(String name, String type) { + super(name, type); + } + + public enum Type { + Binary, Boolean, Date, Keyword, Long, Integer + } +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/define/ElasticSearchStorageInstaller.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/define/ElasticSearchStorageInstaller.java new file mode 100644 index 000000000..25dd33d61 --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/define/ElasticSearchStorageInstaller.java @@ -0,0 +1,98 @@ +package org.skywalking.apm.collector.storage.elasticsearch.define; + +import java.io.IOException; +import java.util.List; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.IndexNotFoundException; +import org.skywalking.apm.collector.client.elasticsearch.ElasticSearchClient; +import org.skywalking.apm.collector.core.client.Client; +import org.skywalking.apm.collector.core.storage.ColumnDefine; +import org.skywalking.apm.collector.core.storage.StorageInstaller; +import org.skywalking.apm.collector.core.storage.TableDefine; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class ElasticSearchStorageInstaller extends StorageInstaller { + + private final Logger logger = LoggerFactory.getLogger(ElasticSearchStorageInstaller.class); + + @Override protected void defineFilter(List tableDefines) { + int size = tableDefines.size(); + for (int i = size - 1; i >= 0; i--) { + if (!(tableDefines.get(i) instanceof ElasticSearchTableDefine)) { + tableDefines.remove(i); + } + } + } + + @Override protected boolean createTable(Client client, TableDefine tableDefine) { + ElasticSearchClient esClient = (ElasticSearchClient)client; + ElasticSearchTableDefine esTableDefine = (ElasticSearchTableDefine)tableDefine; + // settings + String settingSource = ""; + // mapping + XContentBuilder mappingBuilder = null; + try { + XContentBuilder settingsBuilder = createSettingBuilder(esTableDefine); + settingSource = settingsBuilder.string(); + mappingBuilder = createMappingBuilder(esTableDefine); + logger.info("mapping builder str: {}", mappingBuilder.string()); + } catch (Exception e) { + logger.error("create {} index mapping builder error", esTableDefine.getName()); + } + Settings settings = Settings.builder().loadFromSource(settingSource).build(); + + boolean isAcknowledged = esClient.createIndex(esTableDefine.getName(), esTableDefine.type(), settings, mappingBuilder); + logger.info("create {} index with type of {} finished, isAcknowledged: {}", esTableDefine.getName(), esTableDefine.type(), isAcknowledged); + return isAcknowledged; + } + + private XContentBuilder createSettingBuilder(ElasticSearchTableDefine tableDefine) throws IOException { + return XContentFactory.jsonBuilder() + .startObject() + .field("index.number_of_shards", tableDefine.numberOfShards()) + .field("index.number_of_replicas", tableDefine.numberOfReplicas()) + .field("index.refresh_interval", String.valueOf(tableDefine.refreshInterval()) + "s") + .endObject(); + } + + private XContentBuilder createMappingBuilder(ElasticSearchTableDefine tableDefine) throws IOException { + XContentBuilder mappingBuilder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties"); + + for (ColumnDefine columnDefine : tableDefine.getColumnDefines()) { + ElasticSearchColumnDefine elasticSearchColumnDefine = (ElasticSearchColumnDefine)columnDefine; + mappingBuilder + .startObject(elasticSearchColumnDefine.getName()) + .field("type", elasticSearchColumnDefine.getType().toLowerCase()) + .endObject(); + } + + mappingBuilder + .endObject() + .endObject(); + logger.debug("create elasticsearch index: {}", mappingBuilder.string()); + return mappingBuilder; + } + + @Override protected boolean deleteIndex(Client client, TableDefine tableDefine) { + ElasticSearchClient esClient = (ElasticSearchClient)client; + try { + return esClient.deleteIndex(tableDefine.getName()); + } catch (IndexNotFoundException e) { + logger.info("{} index not found", tableDefine.getName()); + } + return false; + } + + @Override protected boolean isExists(Client client, TableDefine tableDefine) { + ElasticSearchClient esClient = (ElasticSearchClient)client; + return esClient.isExistsIndex(tableDefine.getName()); + } +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/define/ElasticSearchTableDefine.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/define/ElasticSearchTableDefine.java new file mode 100644 index 000000000..c9f7241f4 --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/define/ElasticSearchTableDefine.java @@ -0,0 +1,23 @@ +package org.skywalking.apm.collector.storage.elasticsearch.define; + +import org.skywalking.apm.collector.core.storage.TableDefine; + +/** + * @author pengys5 + */ +public abstract class ElasticSearchTableDefine extends TableDefine { + + public ElasticSearchTableDefine(String name) { + super(name); + } + + public final String type() { + return "type"; + } + + public abstract int refreshInterval(); + + public abstract int numberOfShards(); + + public abstract int numberOfReplicas(); +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/StorageH2ConfigParser.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/StorageH2ConfigParser.java new file mode 100644 index 000000000..72f7b561f --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/StorageH2ConfigParser.java @@ -0,0 +1,14 @@ +package org.skywalking.apm.collector.storage.h2; + +import java.util.Map; +import org.skywalking.apm.collector.core.config.ConfigParseException; +import org.skywalking.apm.collector.core.module.ModuleConfigParser; + +/** + * @author pengys5 + */ +public class StorageH2ConfigParser implements ModuleConfigParser { + + @Override public void parse(Map config) throws ConfigParseException { + } +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/StorageH2ModuleDefine.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/StorageH2ModuleDefine.java new file mode 100644 index 000000000..84f9b31a7 --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/StorageH2ModuleDefine.java @@ -0,0 +1,53 @@ +package org.skywalking.apm.collector.storage.h2; + +import java.util.List; +import org.skywalking.apm.collector.client.h2.H2Client; +import org.skywalking.apm.collector.core.client.Client; +import org.skywalking.apm.collector.core.client.DataMonitor; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.module.ModuleConfigParser; +import org.skywalking.apm.collector.core.storage.StorageInstaller; +import org.skywalking.apm.collector.storage.StorageModuleDefine; +import org.skywalking.apm.collector.storage.StorageModuleGroupDefine; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.storage.h2.dao.H2DAO; +import org.skywalking.apm.collector.storage.h2.dao.H2DAODefineLoader; +import org.skywalking.apm.collector.storage.h2.define.H2StorageInstaller; + +/** + * @author pengys5 + */ +public class StorageH2ModuleDefine extends StorageModuleDefine { + + public static final String MODULE_NAME = "h2"; + + @Override protected String group() { + return StorageModuleGroupDefine.GROUP_NAME; + } + + @Override public String name() { + return MODULE_NAME; + } + + @Override protected ModuleConfigParser configParser() { + return new StorageH2ConfigParser(); + } + + @Override protected Client createClient(DataMonitor dataMonitor) { + return new H2Client(); + } + + @Override public StorageInstaller storageInstaller() { + return new H2StorageInstaller(); + } + + @Override public void injectClientIntoDAO(Client client) throws DefineException { + H2DAODefineLoader loader = new H2DAODefineLoader(); + List h2DAOs = loader.load(); + h2DAOs.forEach(h2DAO -> { + h2DAO.setClient((H2Client)client); + String interFaceName = h2DAO.getClass().getInterfaces()[0].getName(); + DAOContainer.INSTANCE.put(interFaceName, h2DAO); + }); + } +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/BatchH2DAO.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/BatchH2DAO.java new file mode 100644 index 000000000..27b69f6d4 --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/BatchH2DAO.java @@ -0,0 +1,14 @@ +package org.skywalking.apm.collector.storage.h2.dao; + +import java.util.List; +import org.skywalking.apm.collector.storage.dao.IBatchDAO; + +/** + * @author pengys5 + */ +public class BatchH2DAO extends H2DAO implements IBatchDAO { + + @Override public void batchPersistence(List batchCollection) { + + } +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/H2DAO.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/H2DAO.java new file mode 100644 index 000000000..a9c9ddb5a --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/H2DAO.java @@ -0,0 +1,10 @@ +package org.skywalking.apm.collector.storage.h2.dao; + +import org.skywalking.apm.collector.client.h2.H2Client; +import org.skywalking.apm.collector.storage.dao.DAO; + +/** + * @author pengys5 + */ +public abstract class H2DAO extends DAO { +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/H2DAODefineLoader.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/H2DAODefineLoader.java new file mode 100644 index 000000000..733eb3bdd --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/H2DAODefineLoader.java @@ -0,0 +1,30 @@ +package org.skywalking.apm.collector.storage.h2.dao; + +import java.util.ArrayList; +import java.util.List; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.framework.Loader; +import org.skywalking.apm.collector.core.util.DefinitionLoader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class H2DAODefineLoader implements Loader> { + + private final Logger logger = LoggerFactory.getLogger(H2DAODefineLoader.class); + + @Override public List load() throws DefineException { + List h2DAOs = new ArrayList<>(); + + H2DAODefinitionFile definitionFile = new H2DAODefinitionFile(); + logger.info("h2 dao definition file name: {}", definitionFile.fileName()); + DefinitionLoader definitionLoader = DefinitionLoader.load(H2DAO.class, definitionFile); + for (H2DAO dao : definitionLoader) { + logger.info("loaded h2 dao definition class: {}", dao.getClass().getName()); + h2DAOs.add(dao); + } + return h2DAOs; + } +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/H2DAODefinitionFile.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/H2DAODefinitionFile.java new file mode 100644 index 000000000..72d37883f --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/dao/H2DAODefinitionFile.java @@ -0,0 +1,13 @@ +package org.skywalking.apm.collector.storage.h2.dao; + +import org.skywalking.apm.collector.core.framework.DefinitionFile; + +/** + * @author pengys5 + */ +public class H2DAODefinitionFile extends DefinitionFile { + + @Override protected String fileName() { + return "h2_dao.define"; + } +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/define/H2ColumnDefine.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/define/H2ColumnDefine.java new file mode 100644 index 000000000..ca7cf49f1 --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/define/H2ColumnDefine.java @@ -0,0 +1,17 @@ +package org.skywalking.apm.collector.storage.h2.define; + +import org.skywalking.apm.collector.core.storage.ColumnDefine; + +/** + * @author pengys5 + */ +public class H2ColumnDefine extends ColumnDefine { + + public H2ColumnDefine(String name, String type) { + super(name, type); + } + + public enum Type { + Boolean, Varchar, Int, Bigint, BINARY + } +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/define/H2StorageInstaller.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/define/H2StorageInstaller.java new file mode 100644 index 000000000..d27a8ca74 --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/define/H2StorageInstaller.java @@ -0,0 +1,58 @@ +package org.skywalking.apm.collector.storage.h2.define; + +import java.util.List; +import org.skywalking.apm.collector.client.h2.H2Client; +import org.skywalking.apm.collector.client.h2.H2ClientException; +import org.skywalking.apm.collector.core.client.Client; +import org.skywalking.apm.collector.core.storage.StorageException; +import org.skywalking.apm.collector.core.storage.StorageInstallException; +import org.skywalking.apm.collector.core.storage.StorageInstaller; +import org.skywalking.apm.collector.core.storage.TableDefine; + +/** + * @author pengys5 + */ +public class H2StorageInstaller extends StorageInstaller { + + @Override protected void defineFilter(List tableDefines) { + int size = tableDefines.size(); + for (int i = size - 1; i >= 0; i--) { + if (!(tableDefines.get(i) instanceof H2TableDefine)) { + tableDefines.remove(i); + } + } + } + + @Override protected boolean isExists(Client client, TableDefine tableDefine) throws StorageException { + return false; + } + + @Override protected boolean deleteIndex(Client client, TableDefine tableDefine) throws StorageException { + return false; + } + + @Override protected boolean createTable(Client client, TableDefine tableDefine) throws StorageException { + H2Client h2Client = (H2Client)client; + H2TableDefine h2TableDefine = (H2TableDefine)tableDefine; + + StringBuilder sqlBuilder = new StringBuilder(); + sqlBuilder.append("CREATE TABLE ").append(h2TableDefine.getName()).append(" ("); + + h2TableDefine.getColumnDefines().forEach(columnDefine -> { + H2ColumnDefine h2ColumnDefine = (H2ColumnDefine)columnDefine; + if (h2ColumnDefine.getType().equals(H2ColumnDefine.Type.Varchar.name())) { + sqlBuilder.append(h2ColumnDefine.getName()).append(" ").append(h2ColumnDefine.getType()).append("(255)"); + } else { + sqlBuilder.append(h2ColumnDefine.getName()).append(" ").append(h2ColumnDefine.getType()); + } + }); + + sqlBuilder.append(")"); + try { + h2Client.execute(sqlBuilder.toString()); + } catch (H2ClientException e) { + throw new StorageInstallException(e.getMessage(), e); + } + return true; + } +} diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/define/H2TableDefine.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/define/H2TableDefine.java new file mode 100644 index 000000000..1854980dd --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/define/H2TableDefine.java @@ -0,0 +1,13 @@ +package org.skywalking.apm.collector.storage.h2.define; + +import org.skywalking.apm.collector.core.storage.TableDefine; + +/** + * @author pengys5 + */ +public abstract class H2TableDefine extends TableDefine { + + public H2TableDefine(String name) { + super(name); + } +} diff --git a/apm-collector/apm-collector-storage/src/main/resources/META-INF/defines/es_dao.define b/apm-collector/apm-collector-storage/src/main/resources/META-INF/defines/es_dao.define new file mode 100644 index 000000000..1fcbc9a03 --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/resources/META-INF/defines/es_dao.define @@ -0,0 +1 @@ +org.skywalking.apm.collector.storage.elasticsearch.dao.BatchEsDAO \ No newline at end of file diff --git a/apm-collector/apm-collector-storage/src/main/resources/META-INF/defines/group.define b/apm-collector/apm-collector-storage/src/main/resources/META-INF/defines/group.define new file mode 100644 index 000000000..8627ade6b --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/resources/META-INF/defines/group.define @@ -0,0 +1 @@ +org.skywalking.apm.collector.storage.StorageModuleGroupDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-storage/src/main/resources/META-INF/defines/h2_dao.define b/apm-collector/apm-collector-storage/src/main/resources/META-INF/defines/h2_dao.define new file mode 100644 index 000000000..de143481d --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/resources/META-INF/defines/h2_dao.define @@ -0,0 +1 @@ +org.skywalking.apm.collector.storage.h2.dao.BatchH2DAO \ No newline at end of file diff --git a/apm-collector/apm-collector-storage/src/main/resources/META-INF/defines/module.define b/apm-collector/apm-collector-storage/src/main/resources/META-INF/defines/module.define new file mode 100644 index 000000000..a5844ab39 --- /dev/null +++ b/apm-collector/apm-collector-storage/src/main/resources/META-INF/defines/module.define @@ -0,0 +1,2 @@ +org.skywalking.apm.collector.storage.elasticsearch.StorageElasticSearchModuleDefine +org.skywalking.apm.collector.storage.h2.StorageH2ModuleDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-remote/pom.xml b/apm-collector/apm-collector-stream/pom.xml similarity index 65% rename from apm-collector/apm-collector-remote/pom.xml rename to apm-collector/apm-collector-stream/pom.xml index 81c877d81..cc73ad3ca 100644 --- a/apm-collector/apm-collector-remote/pom.xml +++ b/apm-collector/apm-collector-stream/pom.xml @@ -9,15 +9,9 @@ 4.0.0 - apm-collector-remote + apm-collector-stream jar - - UTF-8 - 1.4.0 - 4.1.12.Final - - org.skywalking @@ -26,43 +20,18 @@ org.skywalking - apm-collector-server + apm-collector-cluster ${project.version} - io.grpc - grpc-netty - ${grpc.version} - - - io.netty - netty-codec-http2 - - - io.netty - netty-handler-proxy - - + org.skywalking + apm-collector-queue + ${project.version} - io.grpc - grpc-protobuf - ${grpc.version} - - - io.grpc - grpc-stub - ${grpc.version} - - - io.netty - netty-codec-http2 - ${netty.version} - - - io.netty - netty-handler-proxy - ${netty.version} + org.skywalking + apm-collector-server + ${project.version} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleContext.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleContext.java new file mode 100644 index 000000000..a68f89341 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleContext.java @@ -0,0 +1,37 @@ +package org.skywalking.apm.collector.stream; + +import java.util.HashMap; +import java.util.Map; +import org.skywalking.apm.collector.core.framework.Context; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; + +/** + * @author pengys5 + */ +public class StreamModuleContext extends Context { + + private Map dataDefineMap; + private ClusterWorkerContext clusterWorkerContext; + + public StreamModuleContext(String groupName) { + super(groupName); + dataDefineMap = new HashMap<>(); + } + + public void putAllDataDefine(Map dataDefineMap) { + this.dataDefineMap.putAll(dataDefineMap); + } + + public DataDefine getDataDefine(int dataDefineId) { + return this.dataDefineMap.get(dataDefineId); + } + + public ClusterWorkerContext getClusterWorkerContext() { + return clusterWorkerContext; + } + + public void setClusterWorkerContext(ClusterWorkerContext clusterWorkerContext) { + this.clusterWorkerContext = clusterWorkerContext; + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleDefine.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleDefine.java new file mode 100644 index 000000000..89dad56db --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleDefine.java @@ -0,0 +1,45 @@ +package org.skywalking.apm.collector.stream; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.cluster.ClusterModuleGroupDefine; +import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.cluster.ClusterDataListenerDefine; +import org.skywalking.apm.collector.core.cluster.ClusterModuleContext; +import org.skywalking.apm.collector.core.config.ConfigParseException; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.framework.Handler; +import org.skywalking.apm.collector.core.module.ModuleDefine; +import org.skywalking.apm.collector.core.server.Server; +import org.skywalking.apm.collector.core.server.ServerException; +import org.skywalking.apm.collector.core.server.ServerHolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public abstract class StreamModuleDefine extends ModuleDefine implements ClusterDataListenerDefine { + + private final Logger logger = LoggerFactory.getLogger(StreamModuleDefine.class); + + @Override + public final void initialize(Map config, ServerHolder serverHolder) throws DefineException, ClientException { + try { + configParser().parse(config); + Server server = server(); + serverHolder.holdServer(server, handlerList()); + + ((ClusterModuleContext)CollectorContextHelper.INSTANCE.getContext(ClusterModuleGroupDefine.GROUP_NAME)).getDataMonitor().addListener(listener(), registration()); + } catch (ConfigParseException | ServerException e) { + throw new StreamModuleException(e.getMessage(), e); + } + } + + @Override public final boolean defaultModule() { + return true; + } + + public abstract List handlerList() throws DefineException; +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleException.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleException.java new file mode 100644 index 000000000..81249ad74 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleException.java @@ -0,0 +1,17 @@ +package org.skywalking.apm.collector.stream; + +import org.skywalking.apm.collector.core.module.ModuleException; + +/** + * @author pengys5 + */ +public class StreamModuleException extends ModuleException { + + public StreamModuleException(String message) { + super(message); + } + + public StreamModuleException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleGroupDefine.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleGroupDefine.java new file mode 100644 index 000000000..e320a817d --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleGroupDefine.java @@ -0,0 +1,25 @@ +package org.skywalking.apm.collector.stream; + +import org.skywalking.apm.collector.core.framework.Context; +import org.skywalking.apm.collector.core.module.ModuleGroupDefine; +import org.skywalking.apm.collector.core.module.ModuleInstaller; + +/** + * @author pengys5 + */ +public class StreamModuleGroupDefine implements ModuleGroupDefine { + + public static final String GROUP_NAME = "stream"; + + @Override public String name() { + return GROUP_NAME; + } + + @Override public Context groupContext() { + return new StreamModuleContext(GROUP_NAME); + } + + @Override public ModuleInstaller moduleInstaller() { + return new StreamModuleInstaller(); + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java new file mode 100644 index 000000000..908eecbb0 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java @@ -0,0 +1,76 @@ +package org.skywalking.apm.collector.stream; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.module.ModuleDefine; +import org.skywalking.apm.collector.core.module.ModuleInstaller; +import org.skywalking.apm.collector.core.server.ServerHolder; +import org.skywalking.apm.collector.core.util.ObjectUtils; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; +import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.LocalAsyncWorkerProviderDefineLoader; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.RemoteWorkerProviderDefineLoader; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefineLoader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class StreamModuleInstaller implements ModuleInstaller { + + private final Logger logger = LoggerFactory.getLogger(StreamModuleInstaller.class); + + @Override public void install(Map moduleConfig, Map moduleDefineMap, + ServerHolder serverHolder) throws DefineException, ClientException { + logger.info("beginning stream module install"); + StreamModuleContext context = new StreamModuleContext(StreamModuleGroupDefine.GROUP_NAME); + CollectorContextHelper.INSTANCE.putContext(context); + + DataDefineLoader dataDefineLoader = new DataDefineLoader(); + Map dataDefineMap = dataDefineLoader.load(); + context.putAllDataDefine(dataDefineMap); + + initializeWorker(context); + + logger.info("could not configure cluster module, use the default"); + Iterator> moduleDefineEntry = moduleDefineMap.entrySet().iterator(); + while (moduleDefineEntry.hasNext()) { + ModuleDefine moduleDefine = moduleDefineEntry.next().getValue(); + logger.info("module {} initialize", moduleDefine.getClass().getName()); + moduleDefine.initialize((ObjectUtils.isNotEmpty(moduleConfig) && moduleConfig.containsKey(moduleDefine.name())) ? moduleConfig.get(moduleDefine.name()) : null, serverHolder); + } + } + + private void initializeWorker(StreamModuleContext context) throws DefineException { + ClusterWorkerContext clusterWorkerContext = new ClusterWorkerContext(); + context.setClusterWorkerContext(clusterWorkerContext); + + LocalAsyncWorkerProviderDefineLoader localAsyncProviderLoader = new LocalAsyncWorkerProviderDefineLoader(); + RemoteWorkerProviderDefineLoader remoteProviderLoader = new RemoteWorkerProviderDefineLoader(); + try { + List localAsyncProviders = localAsyncProviderLoader.load(); + for (AbstractLocalAsyncWorkerProvider provider : localAsyncProviders) { + provider.setClusterContext(clusterWorkerContext); + provider.create(); + clusterWorkerContext.putRole(provider.role()); + } + + List remoteProviders = remoteProviderLoader.load(); + for (AbstractRemoteWorkerProvider provider : remoteProviders) { + provider.setClusterContext(clusterWorkerContext); + clusterWorkerContext.putRole(provider.role()); + clusterWorkerContext.putProvider(provider); + } + } catch (ProviderNotFoundException e) { + logger.error(e.getMessage(), e); + } + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCConfig.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCConfig.java new file mode 100644 index 000000000..5d05cbd57 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCConfig.java @@ -0,0 +1,9 @@ +package org.skywalking.apm.collector.stream.grpc; + +/** + * @author pengys5 + */ +public class StreamGRPCConfig { + public static String HOST; + public static int PORT; +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCConfigParser.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCConfigParser.java new file mode 100644 index 000000000..621cdbf72 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCConfigParser.java @@ -0,0 +1,30 @@ +package org.skywalking.apm.collector.stream.grpc; + +import java.util.Map; +import org.skywalking.apm.collector.core.config.ConfigParseException; +import org.skywalking.apm.collector.core.module.ModuleConfigParser; +import org.skywalking.apm.collector.core.util.ObjectUtils; +import org.skywalking.apm.collector.core.util.StringUtils; + +/** + * @author pengys5 + */ +public class StreamGRPCConfigParser implements ModuleConfigParser { + + private static final String HOST = "host"; + private static final String PORT = "port"; + + @Override public void parse(Map config) throws ConfigParseException { + if (ObjectUtils.isEmpty(config) || StringUtils.isEmpty(config.get(HOST))) { + StreamGRPCConfig.HOST = "localhost"; + } else { + StreamGRPCConfig.HOST = (String)config.get(HOST); + } + + if (ObjectUtils.isEmpty(config) || StringUtils.isEmpty(config.get(PORT))) { + StreamGRPCConfig.PORT = 11800; + } else { + StreamGRPCConfig.PORT = (Integer)config.get(PORT); + } + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCDataListener.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCDataListener.java new file mode 100644 index 000000000..89f7e64a3 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCDataListener.java @@ -0,0 +1,74 @@ +package org.skywalking.apm.collector.stream.grpc; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.client.grpc.GRPCClient; +import org.skywalking.apm.collector.cluster.ClusterModuleDefine; +import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.cluster.ClusterDataListener; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.stream.StreamModuleContext; +import org.skywalking.apm.collector.stream.StreamModuleGroupDefine; +import org.skywalking.apm.collector.stream.worker.RemoteWorkerRef; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class StreamGRPCDataListener extends ClusterDataListener { + + private final Logger logger = LoggerFactory.getLogger(StreamGRPCDataListener.class); + + public static final String PATH = ClusterModuleDefine.BASE_CATALOG + "." + StreamModuleGroupDefine.GROUP_NAME + "." + StreamGRPCModuleDefine.MODULE_NAME; + + @Override public String path() { + return PATH; + } + + private Map clients = new HashMap<>(); + private Map workerRefs = new HashMap<>(); + + @Override public void addressChangedNotify() { + String selfAddress = StreamGRPCConfig.HOST + ":" + StreamGRPCConfig.PORT; + + StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); + + List addresses = getAddresses(); + clients.keySet().forEach(address -> { + if (!addresses.contains(address)) { + context.getClusterWorkerContext().remove(workerRefs.get(address)); + workerRefs.remove(address); + } + }); + + for (String address : addresses) { + if (!clients.containsKey(address)) { + logger.debug("new address: {}, create this address remote worker reference", address); + String[] hostPort = address.split(":"); + GRPCClient client = new GRPCClient(hostPort[0], Integer.valueOf(hostPort[1])); + try { + client.initialize(); + } catch (ClientException e) { + e.printStackTrace(); + } + clients.put(address, client); + + if (selfAddress.equals(address)) { + context.getClusterWorkerContext().getProviders().forEach(provider -> { + logger.debug("create remote worker self reference, role: {}", provider.role().roleName()); + provider.create(); + }); + } else { + context.getClusterWorkerContext().getProviders().forEach(provider -> { + logger.debug("create remote worker reference, role: {}", provider.role().roleName()); + RemoteWorkerRef workerRef = provider.create(client); + }); + } + } else { + logger.debug("address: {} had remote worker reference, ignore", address); + } + } + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCModuleDefine.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCModuleDefine.java new file mode 100644 index 000000000..3a8166476 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCModuleDefine.java @@ -0,0 +1,58 @@ +package org.skywalking.apm.collector.stream.grpc; + +import java.util.ArrayList; +import java.util.List; +import org.skywalking.apm.collector.core.client.Client; +import org.skywalking.apm.collector.core.client.DataMonitor; +import org.skywalking.apm.collector.core.cluster.ClusterDataListener; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.framework.Handler; +import org.skywalking.apm.collector.core.module.ModuleConfigParser; +import org.skywalking.apm.collector.core.module.ModuleRegistration; +import org.skywalking.apm.collector.core.server.Server; +import org.skywalking.apm.collector.server.grpc.GRPCServer; +import org.skywalking.apm.collector.stream.StreamModuleDefine; +import org.skywalking.apm.collector.stream.StreamModuleGroupDefine; +import org.skywalking.apm.collector.stream.grpc.handler.RemoteCommonServiceHandler; + +/** + * @author pengys5 + */ +public class StreamGRPCModuleDefine extends StreamModuleDefine { + + public static final String MODULE_NAME = "stream"; + + @Override public String name() { + return MODULE_NAME; + } + + @Override protected String group() { + return StreamModuleGroupDefine.GROUP_NAME; + } + + @Override protected ModuleConfigParser configParser() { + return new StreamGRPCConfigParser(); + } + + @Override protected Client createClient(DataMonitor dataMonitor) { + return null; + } + + @Override protected Server server() { + return new GRPCServer(StreamGRPCConfig.HOST, StreamGRPCConfig.PORT); + } + + @Override protected ModuleRegistration registration() { + return new StreamGRPCModuleRegistration(); + } + + @Override public ClusterDataListener listener() { + return new StreamGRPCDataListener(); + } + + @Override public List handlerList() throws DefineException { + List handlers = new ArrayList<>(); + handlers.add(new RemoteCommonServiceHandler()); + return handlers; + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCModuleRegistration.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCModuleRegistration.java new file mode 100644 index 000000000..1c74d45c3 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/StreamGRPCModuleRegistration.java @@ -0,0 +1,13 @@ +package org.skywalking.apm.collector.stream.grpc; + +import org.skywalking.apm.collector.core.module.ModuleRegistration; + +/** + * @author pengys5 + */ +public class StreamGRPCModuleRegistration extends ModuleRegistration { + + @Override public Value buildValue() { + return new Value(StreamGRPCConfig.HOST, StreamGRPCConfig.PORT, null); + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/handler/RemoteCommonServiceHandler.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/handler/RemoteCommonServiceHandler.java new file mode 100644 index 000000000..86b5dc2b0 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/grpc/handler/RemoteCommonServiceHandler.java @@ -0,0 +1,38 @@ +package org.skywalking.apm.collector.stream.grpc.handler; + +import io.grpc.stub.StreamObserver; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.remote.grpc.proto.Empty; +import org.skywalking.apm.collector.remote.grpc.proto.RemoteCommonServiceGrpc; +import org.skywalking.apm.collector.remote.grpc.proto.RemoteData; +import org.skywalking.apm.collector.remote.grpc.proto.RemoteMessage; +import org.skywalking.apm.collector.server.grpc.GRPCHandler; +import org.skywalking.apm.collector.stream.StreamModuleContext; +import org.skywalking.apm.collector.stream.StreamModuleGroupDefine; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerInvokeException; +import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class RemoteCommonServiceHandler extends RemoteCommonServiceGrpc.RemoteCommonServiceImplBase implements GRPCHandler { + + private final Logger logger = LoggerFactory.getLogger(RemoteCommonServiceHandler.class); + + @Override public void call(RemoteMessage request, StreamObserver responseObserver) { + String roleName = request.getWorkerRole(); + RemoteData remoteData = request.getRemoteData(); + + StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); + Role role = context.getClusterWorkerContext().getRole(roleName); + Object object = role.dataDefine().deserialize(remoteData); + try { + context.getClusterWorkerContext().lookupInSide(roleName).tell(object); + } catch (WorkerNotFoundException | WorkerInvokeException e) { + logger.error(e.getMessage(), e); + } + } +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractLocalAsyncWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorker.java similarity index 73% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractLocalAsyncWorker.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorker.java index bba13892b..e43561bee 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractLocalAsyncWorker.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorker.java @@ -1,4 +1,6 @@ -package org.skywalking.apm.collector.core.worker; +package org.skywalking.apm.collector.stream.worker; + +import org.skywalking.apm.collector.core.queue.QueueExecutor; /** * The AbstractLocalAsyncWorker implementations represent workers, @@ -7,7 +9,7 @@ package org.skywalking.apm.collector.core.worker; * @author pengys5 * @since v3.0-2017 */ -public abstract class AbstractLocalAsyncWorker extends AbstractLocalWorker { +public abstract class AbstractLocalAsyncWorker extends AbstractLocalWorker implements QueueExecutor { /** * Construct an AbstractLocalAsyncWorker with the worker role and context. @@ -15,10 +17,9 @@ public abstract class AbstractLocalAsyncWorker extends AbstractLocalWorker { * @param role The responsibility of worker in cluster, more than one workers can have same responsibility which use * to provide load balancing ability. * @param clusterContext See {@link ClusterWorkerContext} - * @param selfContext See {@link LocalWorkerContext} */ - public AbstractLocalAsyncWorker(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) { - super(role, clusterContext, selfContext); + public AbstractLocalAsyncWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); } /** @@ -40,12 +41,4 @@ public abstract class AbstractLocalAsyncWorker extends AbstractLocalWorker { final public void allocateJob(Object message) throws WorkerException { onWork(message); } - - /** - * The data process logic in this method. - * - * @param message Cast the message object to a expect subclass. - * @throws WorkerException Don't handle the exception, throw it. - */ - protected abstract void onWork(Object message) throws WorkerException; } diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorkerProvider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorkerProvider.java new file mode 100644 index 000000000..094b960cb --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorkerProvider.java @@ -0,0 +1,34 @@ +package org.skywalking.apm.collector.stream.worker; + +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.core.queue.QueueCreator; +import org.skywalking.apm.collector.core.queue.QueueEventHandler; +import org.skywalking.apm.collector.core.queue.QueueExecutor; +import org.skywalking.apm.collector.queue.QueueModuleContext; +import org.skywalking.apm.collector.queue.QueueModuleGroupDefine; +import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; +import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorkerContainer; + +/** + * @author pengys5 + */ +public abstract class AbstractLocalAsyncWorkerProvider extends AbstractLocalWorkerProvider { + + public abstract int queueSize(); + + @Override final public WorkerRef create() throws ProviderNotFoundException { + T localAsyncWorker = workerInstance(getClusterContext()); + localAsyncWorker.preStart(); + + if (localAsyncWorker instanceof PersistenceWorker) { + PersistenceWorkerContainer.INSTANCE.addWorker((PersistenceWorker)localAsyncWorker); + } + + QueueCreator queueCreator = ((QueueModuleContext)CollectorContextHelper.INSTANCE.getContext(QueueModuleGroupDefine.GROUP_NAME)).getQueueCreator(); + QueueEventHandler queueEventHandler = queueCreator.create(queueSize(), localAsyncWorker); + + LocalAsyncWorkerRef workerRef = new LocalAsyncWorkerRef(role(), queueEventHandler); + getClusterContext().put(workerRef); + return workerRef; + } +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractLocalSyncWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalSyncWorker.java similarity index 90% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractLocalSyncWorker.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalSyncWorker.java index 0638c4556..af38f5f50 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractLocalSyncWorker.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalSyncWorker.java @@ -1,4 +1,4 @@ -package org.skywalking.apm.collector.core.worker; +package org.skywalking.apm.collector.stream.worker; /** * The AbstractLocalSyncWorker defines workers who receive data from jvm inside call and response in real @@ -8,8 +8,8 @@ package org.skywalking.apm.collector.core.worker; * @since v3.0-2017 */ public abstract class AbstractLocalSyncWorker extends AbstractLocalWorker { - public AbstractLocalSyncWorker(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) { - super(role, clusterContext, selfContext); + public AbstractLocalSyncWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); } /** diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalSyncWorkerProvider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalSyncWorkerProvider.java new file mode 100644 index 000000000..d3a542a61 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalSyncWorkerProvider.java @@ -0,0 +1,15 @@ +package org.skywalking.apm.collector.stream.worker; + +/** + * @author pengys5 + */ +public abstract class AbstractLocalSyncWorkerProvider extends AbstractLocalWorkerProvider { + + @Override final public WorkerRef create() throws ProviderNotFoundException { + T localSyncWorker = workerInstance(getClusterContext()); + localSyncWorker.preStart(); + + LocalSyncWorkerRef workerRef = new LocalSyncWorkerRef(role(), localSyncWorker); + return workerRef; + } +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractLocalWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorker.java similarity index 52% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractLocalWorker.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorker.java index f370cdac2..c92a23fc7 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractLocalWorker.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorker.java @@ -1,10 +1,10 @@ -package org.skywalking.apm.collector.core.worker; +package org.skywalking.apm.collector.stream.worker; /** * @author pengys5 */ public abstract class AbstractLocalWorker extends AbstractWorker { - public AbstractLocalWorker(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) { - super(role, clusterContext, selfContext); + public AbstractLocalWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); } } diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractLocalWorkerProvider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorkerProvider.java similarity index 73% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractLocalWorkerProvider.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorkerProvider.java index 1f7116be4..8cc96b71f 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractLocalWorkerProvider.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorkerProvider.java @@ -1,4 +1,4 @@ -package org.skywalking.apm.collector.core.worker; +package org.skywalking.apm.collector.stream.worker; /** * @author pengys5 diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractClusterWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorker.java similarity index 52% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractClusterWorker.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorker.java index 56cfc50f4..8dab25277 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/AbstractClusterWorker.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorker.java @@ -1,7 +1,7 @@ -package org.skywalking.apm.collector.core.worker; +package org.skywalking.apm.collector.stream.worker; /** - * The AbstractClusterWorker implementations represent workers, + * The AbstractRemoteWorker implementations represent workers, * which receive remote messages. *

* Usually, the implementations are doing persistent, or aggregate works. @@ -9,18 +9,17 @@ package org.skywalking.apm.collector.core.worker; * @author pengys5 * @since v3.0-2017 */ -public abstract class AbstractClusterWorker extends AbstractWorker { +public abstract class AbstractRemoteWorker extends AbstractWorker { /** - * Construct an AbstractClusterWorker with the worker role and context. + * Construct an AbstractRemoteWorker with the worker role and context. * * @param role If multi-workers are for load balance, they should be more likely called worker instance. Meaning, * each worker have multi instances. * @param clusterContext See {@link ClusterWorkerContext} - * @param selfContext See {@link LocalWorkerContext} */ - protected AbstractClusterWorker(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) { - super(role, clusterContext, selfContext); + protected AbstractRemoteWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); } /** @@ -36,12 +35,4 @@ public abstract class AbstractClusterWorker extends AbstractWorker { throw new WorkerInvokeException(e.getMessage(), e.getCause()); } } - - /** - * This method use for message receiver to analyse message. - * - * @param message Cast the message object to a expect subclass. - * @throws Exception Don't handle the exception, throw it. - */ - protected abstract void onWork(Object message) throws WorkerException; } diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorkerProvider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorkerProvider.java new file mode 100644 index 000000000..33b97df63 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorkerProvider.java @@ -0,0 +1,34 @@ +package org.skywalking.apm.collector.stream.worker; + +import org.skywalking.apm.collector.client.grpc.GRPCClient; + +/** + * The AbstractRemoteWorkerProvider implementations represent providers, + * which create instance of cluster workers whose implemented {@link AbstractRemoteWorker}. + *

+ * + * @author pengys5 + * @since v3.0-2017 + */ +public abstract class AbstractRemoteWorkerProvider extends AbstractWorkerProvider { + + /** + * Create the worker instance into akka system, the akka system will control the cluster worker life cycle. + * + * @return The created worker reference. See {@link RemoteWorkerRef} + * @throws ProviderNotFoundException This worker instance attempted to find a provider which use to create another + * worker instance, when the worker provider not find then Throw this Exception. + */ + @Override final public WorkerRef create() { + T clusterWorker = workerInstance(getClusterContext()); + RemoteWorkerRef workerRef = new RemoteWorkerRef(role(), clusterWorker); + getClusterContext().put(workerRef); + return workerRef; + } + + public final RemoteWorkerRef create(GRPCClient client) { + RemoteWorkerRef workerRef = new RemoteWorkerRef(role(), client); + getClusterContext().put(workerRef); + return workerRef; + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractWorker.java new file mode 100644 index 000000000..42d69415a --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractWorker.java @@ -0,0 +1,48 @@ +package org.skywalking.apm.collector.stream.worker; + +import org.skywalking.apm.collector.core.framework.Executor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public abstract class AbstractWorker implements Executor { + + private final Logger logger = LoggerFactory.getLogger(AbstractWorker.class); + + private final Role role; + + private final ClusterWorkerContext clusterContext; + + public AbstractWorker(Role role, ClusterWorkerContext clusterContext) { + this.role = role; + this.clusterContext = clusterContext; + } + + @Override public final void execute(Object message) { + try { + onWork(message); + } catch (WorkerException e) { + logger.error(e.getMessage(), e); + } + } + + /** + * The data process logic in this method. + * + * @param message Cast the message object to a expect subclass. + * @throws WorkerException Don't handle the exception, throw it. + */ + protected abstract void onWork(Object message) throws WorkerException; + + public abstract void preStart() throws ProviderNotFoundException; + + final public ClusterWorkerContext getClusterContext() { + return clusterContext; + } + + final public Role getRole() { + return role; + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractWorkerProvider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractWorkerProvider.java new file mode 100644 index 000000000..142a371e5 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractWorkerProvider.java @@ -0,0 +1,21 @@ +package org.skywalking.apm.collector.stream.worker; + +/** + * @author pengys5 + */ +public abstract class AbstractWorkerProvider implements Provider { + + private ClusterWorkerContext clusterContext; + + public abstract Role role(); + + public abstract T workerInstance(ClusterWorkerContext clusterContext); + + final public void setClusterContext(ClusterWorkerContext clusterContext) { + this.clusterContext = clusterContext; + } + + final protected ClusterWorkerContext getClusterContext() { + return clusterContext; + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/ClusterWorkerContext.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/ClusterWorkerContext.java new file mode 100644 index 000000000..f6783dce1 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/ClusterWorkerContext.java @@ -0,0 +1,21 @@ +package org.skywalking.apm.collector.stream.worker; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author pengys5 + */ +public class ClusterWorkerContext extends WorkerContext { + + private List providers = new ArrayList<>(); + + public List getProviders() { + return providers; + } + + @Override + public void putProvider(AbstractRemoteWorkerProvider provider) { + providers.add(provider); + } +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/ClusterWorkerRefCounter.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/ClusterWorkerRefCounter.java similarity index 91% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/ClusterWorkerRefCounter.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/ClusterWorkerRefCounter.java index c82d3953d..3456dbb53 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/ClusterWorkerRefCounter.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/ClusterWorkerRefCounter.java @@ -1,4 +1,4 @@ -package org.skywalking.apm.collector.core.worker; +package org.skywalking.apm.collector.stream.worker; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/Context.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Context.java similarity index 52% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/Context.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Context.java index 7fdbab6a0..8b8d4e418 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/Context.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Context.java @@ -1,14 +1,16 @@ -package org.skywalking.apm.collector.core.worker; +package org.skywalking.apm.collector.stream.worker; /** * @author pengys5 */ public interface Context extends LookUp { - void putProvider(AbstractWorkerProvider provider) throws UsedRoleNameException; + void putProvider(AbstractRemoteWorkerProvider provider); WorkerRefs lookup(Role role) throws WorkerNotFoundException; + RemoteWorkerRef lookupInSide(String roleName) throws WorkerNotFoundException; + void put(WorkerRef workerRef); void remove(WorkerRef workerRef); diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefineLoader.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefineLoader.java new file mode 100644 index 000000000..27fd2f1df --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefineLoader.java @@ -0,0 +1,31 @@ +package org.skywalking.apm.collector.stream.worker; + +import java.util.ArrayList; +import java.util.List; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.framework.Loader; +import org.skywalking.apm.collector.core.util.DefinitionLoader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class LocalAsyncWorkerProviderDefineLoader implements Loader> { + + private final Logger logger = LoggerFactory.getLogger(LocalAsyncWorkerProviderDefineLoader.class); + + @Override public List load() throws DefineException { + List providers = new ArrayList<>(); + LocalAsyncWorkerProviderDefinitionFile definitionFile = new LocalAsyncWorkerProviderDefinitionFile(); + logger.info("local async worker provider definition file name: {}", definitionFile.fileName()); + + DefinitionLoader definitionLoader = DefinitionLoader.load(AbstractLocalAsyncWorkerProvider.class, definitionFile); + + for (AbstractLocalAsyncWorkerProvider provider : definitionLoader) { + logger.info("loaded local async worker provider definition class: {}", provider.getClass().getName()); + providers.add(provider); + } + return providers; + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefinitionFile.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefinitionFile.java new file mode 100644 index 000000000..356cf9ffb --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefinitionFile.java @@ -0,0 +1,12 @@ +package org.skywalking.apm.collector.stream.worker; + +import org.skywalking.apm.collector.core.framework.DefinitionFile; + +/** + * @author pengys5 + */ +public class LocalAsyncWorkerProviderDefinitionFile extends DefinitionFile { + @Override protected String fileName() { + return "local_async_worker_provider.define"; + } +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/LocalAsyncWorkerRef.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerRef.java similarity index 90% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/LocalAsyncWorkerRef.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerRef.java index 84a3000fb..06ee2f0b6 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/LocalAsyncWorkerRef.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerRef.java @@ -1,4 +1,4 @@ -package org.skywalking.apm.collector.core.worker; +package org.skywalking.apm.collector.stream.worker; import org.skywalking.apm.collector.core.queue.QueueEventHandler; diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/LocalSyncWorkerRef.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalSyncWorkerRef.java similarity index 91% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/LocalSyncWorkerRef.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalSyncWorkerRef.java index 9eb502dc8..d24d07779 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/LocalSyncWorkerRef.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalSyncWorkerRef.java @@ -1,4 +1,4 @@ -package org.skywalking.apm.collector.core.worker; +package org.skywalking.apm.collector.stream.worker; /** * @author pengys5 diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/LookUp.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LookUp.java similarity index 50% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/LookUp.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LookUp.java index 4a94fc2f8..a462632c5 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/LookUp.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LookUp.java @@ -1,4 +1,4 @@ -package org.skywalking.apm.collector.core.worker; +package org.skywalking.apm.collector.stream.worker; /** * @author pengys5 @@ -6,6 +6,4 @@ package org.skywalking.apm.collector.core.worker; public interface LookUp { WorkerRefs lookup(Role role) throws WorkerNotFoundException; - - Provider findProvider(Role role) throws ProviderNotFoundException; } diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Provider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Provider.java new file mode 100644 index 000000000..b0f4bdc05 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Provider.java @@ -0,0 +1,9 @@ +package org.skywalking.apm.collector.stream.worker; + +/** + * @author pengys5 + */ +public interface Provider { + + WorkerRef create() throws ProviderNotFoundException; +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/ProviderNotFoundException.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/ProviderNotFoundException.java similarity index 73% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/ProviderNotFoundException.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/ProviderNotFoundException.java index 0f44a778f..9b5327e72 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/ProviderNotFoundException.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/ProviderNotFoundException.java @@ -1,4 +1,4 @@ -package org.skywalking.apm.collector.core.worker; +package org.skywalking.apm.collector.stream.worker; public class ProviderNotFoundException extends Exception { public ProviderNotFoundException(String message) { diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerProviderDefineLoader.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerProviderDefineLoader.java new file mode 100644 index 000000000..001370c4d --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerProviderDefineLoader.java @@ -0,0 +1,31 @@ +package org.skywalking.apm.collector.stream.worker; + +import java.util.ArrayList; +import java.util.List; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.framework.Loader; +import org.skywalking.apm.collector.core.util.DefinitionLoader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class RemoteWorkerProviderDefineLoader implements Loader> { + + private final Logger logger = LoggerFactory.getLogger(RemoteWorkerProviderDefineLoader.class); + + @Override public List load() throws DefineException { + List providers = new ArrayList<>(); + RemoteWorkerProviderDefinitionFile definitionFile = new RemoteWorkerProviderDefinitionFile(); + logger.info("remote worker provider definition file name: {}", definitionFile.fileName()); + + DefinitionLoader definitionLoader = DefinitionLoader.load(AbstractRemoteWorkerProvider.class, definitionFile); + + for (AbstractRemoteWorkerProvider provider : definitionLoader) { + logger.info("loaded remote worker provider definition class: {}", provider.getClass().getName()); + providers.add(provider); + } + return providers; + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerProviderDefinitionFile.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerProviderDefinitionFile.java new file mode 100644 index 000000000..ed301cc92 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerProviderDefinitionFile.java @@ -0,0 +1,12 @@ +package org.skywalking.apm.collector.stream.worker; + +import org.skywalking.apm.collector.core.framework.DefinitionFile; + +/** + * @author pengys5 + */ +public class RemoteWorkerProviderDefinitionFile extends DefinitionFile { + @Override protected String fileName() { + return "remote_worker_provider.define"; + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerRef.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerRef.java new file mode 100644 index 000000000..5e99ab101 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/RemoteWorkerRef.java @@ -0,0 +1,48 @@ +package org.skywalking.apm.collector.stream.worker; + +import org.skywalking.apm.collector.client.grpc.GRPCClient; +import org.skywalking.apm.collector.remote.grpc.proto.RemoteCommonServiceGrpc; +import org.skywalking.apm.collector.remote.grpc.proto.RemoteData; +import org.skywalking.apm.collector.remote.grpc.proto.RemoteMessage; + +/** + * @author pengys5 + */ +public class RemoteWorkerRef extends WorkerRef { + + private final Boolean acrossJVM; + private final RemoteCommonServiceGrpc.RemoteCommonServiceBlockingStub stub; + private final AbstractRemoteWorker remoteWorker; + + public RemoteWorkerRef(Role role, AbstractRemoteWorker remoteWorker) { + super(role); + this.remoteWorker = remoteWorker; + this.acrossJVM = false; + this.stub = null; + } + + public RemoteWorkerRef(Role role, GRPCClient client) { + super(role); + this.remoteWorker = null; + this.acrossJVM = true; + this.stub = RemoteCommonServiceGrpc.newBlockingStub(client.getChannel()); + } + + @Override + public void tell(Object message) throws WorkerInvokeException { + if (acrossJVM) { + RemoteData remoteData = getRole().dataDefine().serialize(message); + + RemoteMessage.Builder builder = RemoteMessage.newBuilder(); + builder.setWorkerRole(getRole().roleName()); + builder.setRemoteData(remoteData); + stub.call(builder.build()); + } else { + remoteWorker.allocateJob(message); + } + } + + public Boolean isAcrossJVM() { + return acrossJVM; + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Role.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Role.java new file mode 100644 index 000000000..6d9ce1497 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/Role.java @@ -0,0 +1,16 @@ +package org.skywalking.apm.collector.stream.worker; + +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public interface Role { + + String roleName(); + + WorkerSelector workerSelector(); + + DataDefine dataDefine(); +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/UsedRoleNameException.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/UsedRoleNameException.java similarity index 72% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/UsedRoleNameException.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/UsedRoleNameException.java index df2629a23..15df1fc82 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/UsedRoleNameException.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/UsedRoleNameException.java @@ -1,4 +1,4 @@ -package org.skywalking.apm.collector.core.worker; +package org.skywalking.apm.collector.stream.worker; public class UsedRoleNameException extends Exception { public UsedRoleNameException(String message) { diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerContext.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerContext.java new file mode 100644 index 000000000..a1fd4bcb7 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerContext.java @@ -0,0 +1,73 @@ +package org.skywalking.apm.collector.stream.worker; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public abstract class WorkerContext implements Context { + + private final Logger logger = LoggerFactory.getLogger(WorkerContext.class); + + private Map remoteWorkerRefs; + private Map> roleWorkers; + private Map roles; + + WorkerContext() { + this.roleWorkers = new HashMap<>(); + this.roles = new HashMap<>(); + this.remoteWorkerRefs = new HashMap<>(); + } + + private Map> getRoleWorkers() { + return this.roleWorkers; + } + + @Override final public WorkerRefs lookup(Role role) throws WorkerNotFoundException { + if (getRoleWorkers().containsKey(role.roleName())) { + return new WorkerRefs(getRoleWorkers().get(role.roleName()), role.workerSelector()); + } else { + throw new WorkerNotFoundException("role=" + role.roleName() + ", no available worker."); + } + } + + @Override final public RemoteWorkerRef lookupInSide(String roleName) throws WorkerNotFoundException { + if (remoteWorkerRefs.containsKey(roleName)) { + return remoteWorkerRefs.get(roleName); + } else { + throw new WorkerNotFoundException("role=" + roleName + ", no available worker."); + } + } + + public final void putRole(Role role) { + roles.put(role.roleName(), role); + } + + public final Role getRole(String roleName) { + return roles.get(roleName); + } + + @Override final public void put(WorkerRef workerRef) { + logger.debug("put worker reference into context, role name: {}", workerRef.getRole().roleName()); + if (!getRoleWorkers().containsKey(workerRef.getRole().roleName())) { + getRoleWorkers().putIfAbsent(workerRef.getRole().roleName(), new ArrayList<>()); + } + getRoleWorkers().get(workerRef.getRole().roleName()).add(workerRef); + + if (workerRef instanceof RemoteWorkerRef) { + RemoteWorkerRef remoteWorkerRef = (RemoteWorkerRef)workerRef; + if (!remoteWorkerRef.isAcrossJVM()) { + remoteWorkerRefs.put(workerRef.getRole().roleName(), remoteWorkerRef); + } + } + } + + @Override final public void remove(WorkerRef workerRef) { + getRoleWorkers().remove(workerRef.getRole().roleName()); + } +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/WorkerException.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerException.java similarity index 87% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/WorkerException.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerException.java index b0bd12310..ada554175 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/WorkerException.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerException.java @@ -1,4 +1,4 @@ -package org.skywalking.apm.collector.core.worker; +package org.skywalking.apm.collector.stream.worker; /** * Defines a general exception a worker can throw when it diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/WorkerInvokeException.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerInvokeException.java similarity index 88% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/WorkerInvokeException.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerInvokeException.java index 03f5de06f..0fe9b6de4 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/WorkerInvokeException.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerInvokeException.java @@ -1,4 +1,4 @@ -package org.skywalking.apm.collector.core.worker; +package org.skywalking.apm.collector.stream.worker; /** * This exception is raised when worker fails to process job during "call" or "ask" diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/WorkerNotFoundException.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerNotFoundException.java similarity index 74% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/WorkerNotFoundException.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerNotFoundException.java index 6c6fc9b2c..ce231e1a1 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/WorkerNotFoundException.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerNotFoundException.java @@ -1,4 +1,4 @@ -package org.skywalking.apm.collector.core.worker; +package org.skywalking.apm.collector.stream.worker; public class WorkerNotFoundException extends WorkerException { public WorkerNotFoundException(String message) { diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/WorkerRef.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerRef.java similarity index 84% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/WorkerRef.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerRef.java index 28268d0b7..899e3978e 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/WorkerRef.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerRef.java @@ -1,4 +1,4 @@ -package org.skywalking.apm.collector.core.worker; +package org.skywalking.apm.collector.stream.worker; /** * @author pengys5 diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/WorkerRefs.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerRefs.java similarity index 53% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/WorkerRefs.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerRefs.java index 8708bec21..1fb3dfeda 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/WorkerRefs.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/WorkerRefs.java @@ -1,7 +1,7 @@ -package org.skywalking.apm.collector.core.worker; +package org.skywalking.apm.collector.stream.worker; import java.util.List; -import org.skywalking.apm.collector.core.worker.selector.WorkerSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -14,23 +14,21 @@ public class WorkerRefs { private List workerRefs; private WorkerSelector workerSelector; + private Role role; protected WorkerRefs(List workerRefs, WorkerSelector workerSelector) { this.workerRefs = workerRefs; this.workerSelector = workerSelector; } - public void tell(Object message) throws WorkerInvokeException { - logger.debug("WorkerSelector instance of %s", workerSelector.getClass()); - workerSelector.select(workerRefs, message).tell(message); + protected WorkerRefs(List workerRefs, WorkerSelector workerSelector, Role role) { + this.workerRefs = workerRefs; + this.workerSelector = workerSelector; + this.role = role; } - public void ask(Object request, Object response) throws WorkerInvokeException { - WorkerRef workerRef = workerSelector.select(workerRefs, request); - if (workerRef instanceof LocalSyncWorkerRef) { - ((LocalSyncWorkerRef)workerRef).ask(request, response); - } else { - throw new IllegalAccessError("only local sync worker can ask"); - } + public void tell(Object message) throws WorkerInvokeException { + logger.debug("WorkerSelector instance of {}", workerSelector.getClass()); + workerSelector.select(workerRefs, message).tell(message); } } diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/AggregationWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/AggregationWorker.java new file mode 100644 index 000000000..321b892b0 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/AggregationWorker.java @@ -0,0 +1,80 @@ +package org.skywalking.apm.collector.stream.worker.impl; + +import org.skywalking.apm.collector.core.queue.EndOfBatchCommand; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorker; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerException; +import org.skywalking.apm.collector.stream.worker.WorkerInvokeException; +import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; +import org.skywalking.apm.collector.stream.worker.WorkerRefs; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataCache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public abstract class AggregationWorker extends AbstractLocalAsyncWorker { + + private final Logger logger = LoggerFactory.getLogger(AggregationWorker.class); + + private DataCache dataCache; + private int messageNum; + + public AggregationWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + dataCache = new DataCache(); + } + + @Override public void preStart() throws ProviderNotFoundException { + super.preStart(); + } + + @Override protected final void onWork(Object message) throws WorkerException { + if (message instanceof EndOfBatchCommand) { + sendToNext(); + } else { + messageNum++; + aggregate(message); + + if (messageNum >= 100) { + sendToNext(); + messageNum = 0; + } + } + } + + protected abstract WorkerRefs nextWorkRef(String id) throws WorkerNotFoundException; + + private void sendToNext() throws WorkerException { + dataCache.switchPointer(); + while (dataCache.getLast().isHolding()) { + try { + Thread.sleep(10); + } catch (InterruptedException e) { + throw new WorkerException(e.getMessage(), e); + } + } + dataCache.getLast().asMap().forEach((id, data) -> { + try { + nextWorkRef(id).tell(data); + } catch (WorkerNotFoundException | WorkerInvokeException e) { + logger.error(e.getMessage(), e); + } + }); + } + + protected final void aggregate(Object message) { + Data data = (Data)message; + dataCache.hold(); + if (dataCache.containsKey(data.id())) { + getRole().dataDefine().mergeData(data, dataCache.get(data.id())); + } else { + dataCache.put(data.id(), data); + } + dataCache.release(); + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/FlushAndSwitch.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/FlushAndSwitch.java new file mode 100644 index 000000000..15b8ee037 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/FlushAndSwitch.java @@ -0,0 +1,7 @@ +package org.skywalking.apm.collector.stream.worker.impl; + +/** + * @author pengys5 + */ +public class FlushAndSwitch { +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/GRPCRemoteWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/GRPCRemoteWorker.java new file mode 100644 index 000000000..4c76735f8 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/GRPCRemoteWorker.java @@ -0,0 +1,25 @@ +package org.skywalking.apm.collector.stream.worker.impl; + +import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorker; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerException; + +/** + * @author pengys5 + */ +public class GRPCRemoteWorker extends AbstractRemoteWorker { + + protected GRPCRemoteWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + + } + + @Override protected final void onWork(Object message) throws WorkerException { + + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/PersistenceWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/PersistenceWorker.java new file mode 100644 index 000000000..f0f3bae3b --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/PersistenceWorker.java @@ -0,0 +1,82 @@ +package org.skywalking.apm.collector.stream.worker.impl; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.core.queue.EndOfBatchCommand; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorker; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerException; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataCache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public abstract class PersistenceWorker extends AbstractLocalAsyncWorker { + + private final Logger logger = LoggerFactory.getLogger(PersistenceWorker.class); + + private DataCache dataCache; + + public PersistenceWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + dataCache = new DataCache(); + } + + @Override public void preStart() throws ProviderNotFoundException { + super.preStart(); + } + + @Override protected final void onWork(Object message) throws WorkerException { + if (message instanceof EndOfBatchCommand || message instanceof FlushAndSwitch) { + if (dataCache.trySwitchPointer()) { + dataCache.switchPointer(); + } + } else { + if (dataCache.currentCollectionSize() >= 1000) { + if (dataCache.trySwitchPointer()) { + dataCache.switchPointer(); + } + } + aggregate(message); + } + } + + public List buildBatchCollection() throws WorkerException { + List batchCollection; + try { + while (dataCache.getLast().isHolding()) { + try { + Thread.sleep(10); + } catch (InterruptedException e) { + logger.warn("thread wake up"); + } + } + batchCollection = prepareBatch(dataCache.getLast().asMap()); + } finally { + dataCache.releaseLast(); + } + return batchCollection; + } + + protected abstract List prepareBatch(Map dataMap); + + private void aggregate(Object message) { + dataCache.hold(); + Data data = (Data)message; + + if (dataCache.containsKey(data.id())) { + getRole().dataDefine().mergeData(data, dataCache.get(data.id())); + } else { + if (dataCache.currentCollectionSize() < 1000) { + dataCache.put(data.id(), data); + } + } + + dataCache.release(); + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/PersistenceWorkerContainer.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/PersistenceWorkerContainer.java new file mode 100644 index 000000000..1a71b1153 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/PersistenceWorkerContainer.java @@ -0,0 +1,21 @@ +package org.skywalking.apm.collector.stream.worker.impl; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author pengys5 + */ +public enum PersistenceWorkerContainer { + INSTANCE; + + private List persistenceWorkers = new ArrayList<>(); + + public void addWorker(PersistenceWorker worker) { + persistenceWorkers.add(worker); + } + + public List getPersistenceWorkers() { + return persistenceWorkers; + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Attribute.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Attribute.java new file mode 100644 index 000000000..bba24de2c --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Attribute.java @@ -0,0 +1,28 @@ +package org.skywalking.apm.collector.stream.worker.impl.data; + +/** + * @author pengys5 + */ +public class Attribute { + private final String name; + private final AttributeType type; + private final Operation operation; + + public Attribute(String name, AttributeType type, Operation operation) { + this.name = name; + this.type = type; + this.operation = operation; + } + + public String getName() { + return name; + } + + public AttributeType getType() { + return type; + } + + public Operation getOperation() { + return operation; + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/AttributeType.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/AttributeType.java new file mode 100644 index 000000000..fa53ad9da --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/AttributeType.java @@ -0,0 +1,8 @@ +package org.skywalking.apm.collector.stream.worker.impl.data; + +/** + * @author pengys5 + */ +public enum AttributeType { + STRING, LONG, FLOAT, INTEGER, BYTE, BOOLEAN +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Data.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Data.java new file mode 100644 index 000000000..5d5e25af8 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Data.java @@ -0,0 +1,128 @@ +package org.skywalking.apm.collector.stream.worker.impl.data; + +import com.google.protobuf.ByteString; +import org.skywalking.apm.collector.remote.grpc.proto.RemoteData; +import org.skywalking.apm.collector.stream.worker.selector.AbstractHashMessage; + +/** + * @author pengys5 + */ +public class Data extends AbstractHashMessage { + private int defineId; + private final int stringCapacity; + private final int longCapacity; + private final int floatCapacity; + private final int integerCapacity; + private final int booleanCapacity; + private final int byteCapacity; + private String[] dataStrings; + private Long[] dataLongs; + private Float[] dataFloats; + private Integer[] dataIntegers; + private Boolean[] dataBooleans; + private byte[][] dataBytes; + + public Data(String id, int defineId, int stringCapacity, int longCapacity, int floatCapacity, int integerCapacity, + int booleanCapacity, int byteCapacity) { + super(id); + this.defineId = defineId; + this.dataStrings = new String[stringCapacity]; + this.dataLongs = new Long[longCapacity]; + this.dataFloats = new Float[floatCapacity]; + this.dataIntegers = new Integer[integerCapacity]; + this.dataBooleans = new Boolean[booleanCapacity]; + this.dataBytes = new byte[byteCapacity][]; + this.stringCapacity = stringCapacity; + this.longCapacity = longCapacity; + this.floatCapacity = floatCapacity; + this.integerCapacity = integerCapacity; + this.booleanCapacity = booleanCapacity; + this.byteCapacity = byteCapacity; + } + + public void setDataString(int position, String value) { + dataStrings[position] = value; + } + + public void setDataLong(int position, Long value) { + dataLongs[position] = value; + } + + public void setDataFloat(int position, Float value) { + dataFloats[position] = value; + } + + public void setDataInteger(int position, Integer value) { + dataIntegers[position] = value; + } + + public void setDataBoolean(int position, Boolean value) { + dataBooleans[position] = value; + } + + public void setDataBytes(int position, byte[] dataBytes) { + this.dataBytes[position] = dataBytes; + } + + public String getDataString(int position) { + return dataStrings[position]; + } + + public Long getDataLong(int position) { + return dataLongs[position]; + } + + public Float getDataFloat(int position) { + return dataFloats[position]; + } + + public Integer getDataInteger(int position) { + return dataIntegers[position]; + } + + public Boolean getDataBoolean(int position) { + return dataBooleans[position]; + } + + public byte[] getDataBytes(int position) { + return dataBytes[position]; + } + + public String id() { + return dataStrings[0]; + } + + public int getDefineId() { + return defineId; + } + + public RemoteData serialize() { + RemoteData.Builder builder = RemoteData.newBuilder(); + builder.setIntegerCapacity(integerCapacity); + builder.setFloatCapacity(floatCapacity); + builder.setStringCapacity(stringCapacity); + builder.setLongCapacity(longCapacity); + builder.setByteCapacity(byteCapacity); + builder.setBooleanCapacity(booleanCapacity); + + for (int i = 0; i < dataStrings.length; i++) { + builder.setDataStrings(i, dataStrings[i]); + } + for (int i = 0; i < dataIntegers.length; i++) { + builder.setDataIntegers(i, dataIntegers[i]); + } + for (int i = 0; i < dataFloats.length; i++) { + builder.setDataFloats(i, dataFloats[i]); + } + for (int i = 0; i < dataLongs.length; i++) { + builder.setDataLongs(i, dataLongs[i]); + } + for (int i = 0; i < dataBooleans.length; i++) { + builder.setDataBooleans(i, dataBooleans[i]); + } + for (int i = 0; i < dataBytes.length; i++) { + builder.setDataBytes(i, ByteString.copyFrom(dataBytes[i])); + } + return builder.build(); + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataCache.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataCache.java new file mode 100644 index 000000000..c22189fb5 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataCache.java @@ -0,0 +1,34 @@ +package org.skywalking.apm.collector.stream.worker.impl.data; + +/** + * @author pengys5 + */ +public class DataCache extends Window { + + private DataCollection lockedDataCollection; + + public boolean containsKey(String id) { + return lockedDataCollection.containsKey(id); + } + + public Data get(String id) { + return lockedDataCollection.get(id); + } + + public void put(String id, Data data) { + lockedDataCollection.put(id, data); + } + + public void hold() { + lockedDataCollection = getCurrentAndHold(); + } + + public int currentCollectionSize() { + return getCurrentAndHold().size(); + } + + public void release() { + lockedDataCollection.release(); + lockedDataCollection = null; + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataCollection.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataCollection.java new file mode 100644 index 000000000..12cc2f282 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataCollection.java @@ -0,0 +1,53 @@ +package org.skywalking.apm.collector.stream.worker.impl.data; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author pengys5 + */ +public class DataCollection { + private Map data; + private volatile boolean isHold; + + public DataCollection() { + this.data = new HashMap<>(); + this.isHold = false; + } + + public void release() { + isHold = false; + } + + public void hold() { + isHold = true; + } + + public boolean isHolding() { + return isHold; + } + + public boolean containsKey(String key) { + return data.containsKey(key); + } + + public void put(String key, Data value) { + data.put(key, value); + } + + public Data get(String key) { + return data.get(key); + } + + public int size() { + return data.size(); + } + + public void clear() { + data.clear(); + } + + public Map asMap() { + return data; + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefine.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefine.java new file mode 100644 index 000000000..254e742d6 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefine.java @@ -0,0 +1,89 @@ +package org.skywalking.apm.collector.stream.worker.impl.data; + +import org.skywalking.apm.collector.remote.grpc.proto.RemoteData; + +/** + * @author pengys5 + */ +public abstract class DataDefine { + private Attribute[] attributes; + private int stringCapacity; + private int longCapacity; + private int floatCapacity; + private int integerCapacity; + private int booleanCapacity; + private int byteCapacity; + + public DataDefine() { + initial(); + } + + private void initial() { + attributes = new Attribute[initialCapacity()]; + attributeDefine(); + for (Attribute attribute : attributes) { + if (AttributeType.STRING.equals(attribute.getType())) { + stringCapacity++; + } else if (AttributeType.LONG.equals(attribute.getType())) { + longCapacity++; + } else if (AttributeType.FLOAT.equals(attribute.getType())) { + floatCapacity++; + } else if (AttributeType.INTEGER.equals(attribute.getType())) { + integerCapacity++; + } else if (AttributeType.BOOLEAN.equals(attribute.getType())) { + booleanCapacity++; + } else if (AttributeType.BYTE.equals(attribute.getType())) { + byteCapacity++; + } + } + } + + public final void addAttribute(int position, Attribute attribute) { + attributes[position] = attribute; + } + + public abstract int defineId(); + + protected abstract int initialCapacity(); + + protected abstract void attributeDefine(); + + public final Data build(String id) { + return new Data(id, defineId(), stringCapacity, longCapacity, floatCapacity, integerCapacity, booleanCapacity, byteCapacity); + } + + public void mergeData(Data newData, Data oldData) { + int stringPosition = 0; + int longPosition = 0; + int floatPosition = 0; + int integerPosition = 0; + int booleanPosition = 0; + int bytePosition = 0; + for (int i = 0; i < initialCapacity(); i++) { + Attribute attribute = attributes[i]; + if (AttributeType.STRING.equals(attribute.getType())) { + attribute.getOperation().operate(newData.getDataString(stringPosition), oldData.getDataString(stringPosition)); + stringPosition++; + } else if (AttributeType.LONG.equals(attribute.getType())) { + attribute.getOperation().operate(newData.getDataLong(longPosition), oldData.getDataLong(longPosition)); + longPosition++; + } else if (AttributeType.FLOAT.equals(attribute.getType())) { + attribute.getOperation().operate(newData.getDataFloat(floatPosition), oldData.getDataFloat(floatPosition)); + floatPosition++; + } else if (AttributeType.INTEGER.equals(attribute.getType())) { + attribute.getOperation().operate(newData.getDataInteger(integerPosition), oldData.getDataInteger(integerPosition)); + integerPosition++; + } else if (AttributeType.BOOLEAN.equals(attribute.getType())) { + attribute.getOperation().operate(newData.getDataBoolean(booleanPosition), oldData.getDataBoolean(booleanPosition)); + integerPosition++; + } else if (AttributeType.BYTE.equals(attribute.getType())) { + attribute.getOperation().operate(newData.getDataBytes(bytePosition), oldData.getDataBytes(integerPosition)); + bytePosition++; + } + } + } + + public abstract Object deserialize(RemoteData remoteData); + + public abstract RemoteData serialize(Object object); +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefineLoader.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefineLoader.java new file mode 100644 index 000000000..425fcb427 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefineLoader.java @@ -0,0 +1,29 @@ +package org.skywalking.apm.collector.stream.worker.impl.data; + +import java.util.HashMap; +import java.util.Map; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.framework.Loader; +import org.skywalking.apm.collector.core.util.DefinitionLoader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class DataDefineLoader implements Loader> { + + private final Logger logger = LoggerFactory.getLogger(DataDefineLoader.class); + + @Override public Map load() throws DefineException { + Map dataDefineMap = new HashMap<>(); + + DataDefinitionFile definitionFile = new DataDefinitionFile(); + DefinitionLoader definitionLoader = DefinitionLoader.load(DataDefine.class, definitionFile); + for (DataDefine dataDefine : definitionLoader) { + logger.info("loaded data definition class: {}", dataDefine.getClass().getName()); + dataDefineMap.put(dataDefine.defineId(), dataDefine); + } + return dataDefineMap; + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefinitionFile.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefinitionFile.java new file mode 100644 index 000000000..e7875d8b8 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefinitionFile.java @@ -0,0 +1,13 @@ +package org.skywalking.apm.collector.stream.worker.impl.data; + +import org.skywalking.apm.collector.core.framework.DefinitionFile; + +/** + * @author pengys5 + */ +public class DataDefinitionFile extends DefinitionFile { + + @Override protected String fileName() { + return "data.define"; + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Operation.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Operation.java new file mode 100644 index 000000000..667051ab0 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Operation.java @@ -0,0 +1,18 @@ +package org.skywalking.apm.collector.stream.worker.impl.data; + +/** + * @author pengys5 + */ +public interface Operation { + String operate(String newValue, String oldValue); + + Long operate(Long newValue, Long oldValue); + + Float operate(Float newValue, Float oldValue); + + Integer operate(Integer newValue, Integer oldValue); + + Boolean operate(Boolean newValue, Boolean oldValue); + + byte[] operate(byte[] newValue, byte[] oldValue); +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/TransformToData.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/TransformToData.java new file mode 100644 index 000000000..1ce257f78 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/TransformToData.java @@ -0,0 +1,8 @@ +package org.skywalking.apm.collector.stream.worker.impl.data; + +/** + * @author pengys5 + */ +public interface TransformToData { + Data transform(); +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Window.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Window.java new file mode 100644 index 000000000..887f24809 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Window.java @@ -0,0 +1,62 @@ +package org.skywalking.apm.collector.stream.worker.impl.data; + +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author pengys5 + */ +public abstract class Window { + + private AtomicInteger windowSwitch = new AtomicInteger(0); + + private DataCollection pointer; + + private DataCollection windowDataA; + private DataCollection windowDataB; + + public Window() { + windowDataA = new DataCollection(); + windowDataB = new DataCollection(); + pointer = windowDataA; + } + + public boolean trySwitchPointer() { + if (windowSwitch.incrementAndGet() == 1) { + return true; + } else { + windowSwitch.addAndGet(-1); + return false; + } + } + + public void switchPointer() { + if (pointer == windowDataA) { + pointer = windowDataB; + } else { + pointer = windowDataA; + } + } + + protected DataCollection getCurrentAndHold() { + if (pointer == windowDataA) { + windowDataA.hold(); + return windowDataA; + } else { + windowDataB.hold(); + return windowDataB; + } + } + + public DataCollection getLast() { + if (pointer == windowDataA) { + return windowDataB; + } else { + return windowDataA; + } + } + + public void releaseLast() { + getLast().clear(); + windowSwitch.addAndGet(-1); + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/AddOperation.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/AddOperation.java new file mode 100644 index 000000000..0772806e6 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/AddOperation.java @@ -0,0 +1,33 @@ +package org.skywalking.apm.collector.stream.worker.impl.data.operate; + +import org.skywalking.apm.collector.stream.worker.impl.data.Operation; + +/** + * @author pengys5 + */ +public class AddOperation implements Operation { + + @Override public String operate(String newValue, String oldValue) { + throw new UnsupportedOperationException("not support string addition operation"); + } + + @Override public Long operate(Long newValue, Long oldValue) { + return newValue + oldValue; + } + + @Override public Float operate(Float newValue, Float oldValue) { + return newValue + oldValue; + } + + @Override public Integer operate(Integer newValue, Integer oldValue) { + return newValue + oldValue; + } + + @Override public Boolean operate(Boolean newValue, Boolean oldValue) { + throw new UnsupportedOperationException("not support boolean addition operation"); + } + + @Override public byte[] operate(byte[] newValue, byte[] oldValue) { + throw new UnsupportedOperationException("not support byte addition operation"); + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/CoverOperation.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/CoverOperation.java new file mode 100644 index 000000000..06b93fdb0 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/CoverOperation.java @@ -0,0 +1,32 @@ +package org.skywalking.apm.collector.stream.worker.impl.data.operate; + +import org.skywalking.apm.collector.stream.worker.impl.data.Operation; + +/** + * @author pengys5 + */ +public class CoverOperation implements Operation { + @Override public String operate(String newValue, String oldValue) { + return newValue; + } + + @Override public Long operate(Long newValue, Long oldValue) { + return newValue; + } + + @Override public Float operate(Float newValue, Float oldValue) { + return newValue; + } + + @Override public Integer operate(Integer newValue, Integer oldValue) { + return newValue; + } + + @Override public Boolean operate(Boolean newValue, Boolean oldValue) { + return newValue; + } + + @Override public byte[] operate(byte[] newValue, byte[] oldValue) { + return newValue; + } +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/NonOperation.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/NonOperation.java new file mode 100644 index 000000000..62ebb2f95 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/operate/NonOperation.java @@ -0,0 +1,32 @@ +package org.skywalking.apm.collector.stream.worker.impl.data.operate; + +import org.skywalking.apm.collector.stream.worker.impl.data.Operation; + +/** + * @author pengys5 + */ +public class NonOperation implements Operation { + @Override public String operate(String newValue, String oldValue) { + return oldValue; + } + + @Override public Long operate(Long newValue, Long oldValue) { + return oldValue; + } + + @Override public Float operate(Float newValue, Float oldValue) { + return oldValue; + } + + @Override public Integer operate(Integer newValue, Integer oldValue) { + return oldValue; + } + + @Override public Boolean operate(Boolean newValue, Boolean oldValue) { + return oldValue; + } + + @Override public byte[] operate(byte[] newValue, byte[] oldValue) { + return oldValue; + } +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/selector/AbstractHashMessage.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/AbstractHashMessage.java similarity index 88% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/selector/AbstractHashMessage.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/AbstractHashMessage.java index 7d13191d9..e6181510c 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/selector/AbstractHashMessage.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/AbstractHashMessage.java @@ -1,4 +1,4 @@ -package org.skywalking.apm.collector.core.worker.selector; +package org.skywalking.apm.collector.stream.worker.selector; /** * The AbstractHashMessage implementations represent aggregate message, diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/ForeverFirstSelector.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/ForeverFirstSelector.java new file mode 100644 index 000000000..f4af9237f --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/ForeverFirstSelector.java @@ -0,0 +1,19 @@ +package org.skywalking.apm.collector.stream.worker.selector; + +import java.util.List; +import org.skywalking.apm.collector.stream.worker.WorkerRef; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class ForeverFirstSelector implements WorkerSelector { + + private final Logger logger = LoggerFactory.getLogger(ForeverFirstSelector.class); + + @Override public WorkerRef select(List members, Object message) { + logger.debug("member size: {}", members.size()); + return members.get(0); + } +} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/selector/HashCodeSelector.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/HashCodeSelector.java similarity index 81% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/selector/HashCodeSelector.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/HashCodeSelector.java index fea6323be..cb3434110 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/selector/HashCodeSelector.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/HashCodeSelector.java @@ -1,7 +1,8 @@ -package org.skywalking.apm.collector.core.worker.selector; +package org.skywalking.apm.collector.stream.worker.selector; import java.util.List; -import org.skywalking.apm.collector.core.worker.WorkerRef; +import org.skywalking.apm.collector.stream.worker.WorkerRef; +import org.skywalking.apm.collector.stream.worker.AbstractWorker; /** * The HashCodeSelector is a simple implementation of {@link WorkerSelector}. It choose {@link WorkerRef} @@ -17,13 +18,13 @@ public class HashCodeSelector implements WorkerSelector { * Use message hashcode to select {@link WorkerRef}. * * @param members given {@link WorkerRef} list, which size is greater than 0; - * @param message the {@link org.skywalking.apm.collector.core.worker.AbstractWorker} is going to send. + * @param message the {@link AbstractWorker} is going to send. * @return the selected {@link WorkerRef} */ @Override public WorkerRef select(List members, Object message) { if (message instanceof AbstractHashMessage) { - AbstractHashMessage hashMessage = (AbstractHashMessage) message; + AbstractHashMessage hashMessage = (AbstractHashMessage)message; int size = members.size(); int selectIndex = Math.abs(hashMessage.getHashCode()) % size; return members.get(selectIndex); diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/selector/RollingSelector.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/RollingSelector.java similarity index 74% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/selector/RollingSelector.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/RollingSelector.java index 5bb133477..f81e64bae 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/selector/RollingSelector.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/RollingSelector.java @@ -1,7 +1,8 @@ -package org.skywalking.apm.collector.core.worker.selector; +package org.skywalking.apm.collector.stream.worker.selector; import java.util.List; -import org.skywalking.apm.collector.core.worker.WorkerRef; +import org.skywalking.apm.collector.stream.worker.WorkerRef; +import org.skywalking.apm.collector.stream.worker.AbstractWorker; /** * The RollingSelector is a simple implementation of {@link WorkerSelector}. @@ -18,7 +19,7 @@ public class RollingSelector implements WorkerSelector { * Use round-robin to select {@link WorkerRef}. * * @param members given {@link WorkerRef} list, which size is greater than 0; - * @param message message the {@link org.skywalking.apm.collector.core.worker.AbstractWorker} is going to send. + * @param message message the {@link AbstractWorker} is going to send. * @return the selected {@link WorkerRef} */ @Override diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/selector/WorkerSelector.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/WorkerSelector.java similarity index 72% rename from apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/selector/WorkerSelector.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/WorkerSelector.java index 6013ac22e..1409ef746 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/worker/selector/WorkerSelector.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/selector/WorkerSelector.java @@ -1,7 +1,8 @@ -package org.skywalking.apm.collector.core.worker.selector; +package org.skywalking.apm.collector.stream.worker.selector; import java.util.List; -import org.skywalking.apm.collector.core.worker.WorkerRef; +import org.skywalking.apm.collector.stream.worker.WorkerRef; +import org.skywalking.apm.collector.stream.worker.AbstractWorker; /** * The WorkerSelector should be implemented by any class whose instances @@ -18,7 +19,7 @@ public interface WorkerSelector { * select a {@link WorkerRef} from a {@link WorkerRef} list. * * @param members given {@link WorkerRef} list, which size is greater than 0; - * @param message the {@link org.skywalking.apm.collector.core.worker.AbstractWorker} is going to send. + * @param message the {@link AbstractWorker} is going to send. * @return the selected {@link WorkerRef} */ T select(List members, Object message); diff --git a/apm-collector/apm-collector-stream/src/main/proto/RemoteCommonService.proto b/apm-collector/apm-collector-stream/src/main/proto/RemoteCommonService.proto new file mode 100644 index 000000000..5f933c024 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/proto/RemoteCommonService.proto @@ -0,0 +1,32 @@ +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "org.skywalking.apm.collector.remote.grpc.proto"; + +service RemoteCommonService { + rpc call (RemoteMessage) returns (Empty) { + } +} + +message RemoteMessage { + string workerRole = 1; + RemoteData remoteData = 2; +} + +message RemoteData { + int32 stringCapacity = 1; + int32 longCapacity = 2; + int32 floatCapacity = 3; + int32 integerCapacity = 4; + int32 byteCapacity = 5; + int32 booleanCapacity = 6; + repeated string dataStrings = 7; + repeated int64 dataLongs = 8; + repeated float dataFloats = 9; + repeated int32 dataIntegers = 10; + repeated bytes dataBytes = 11; + repeated bool dataBooleans = 12; +} + +message Empty { +} \ No newline at end of file diff --git a/apm-collector/apm-collector-stream/src/main/resources/META-INF/defines/group.define b/apm-collector/apm-collector-stream/src/main/resources/META-INF/defines/group.define new file mode 100644 index 000000000..dfcb24058 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/resources/META-INF/defines/group.define @@ -0,0 +1 @@ +org.skywalking.apm.collector.stream.StreamModuleGroupDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-stream/src/main/resources/META-INF/defines/module.define b/apm-collector/apm-collector-stream/src/main/resources/META-INF/defines/module.define new file mode 100644 index 000000000..b58f3f133 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/resources/META-INF/defines/module.define @@ -0,0 +1 @@ +org.skywalking.apm.collector.stream.grpc.StreamGRPCModuleDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-ui/pom.xml b/apm-collector/apm-collector-ui/pom.xml index 6ece91dd8..6a434730b 100644 --- a/apm-collector/apm-collector-ui/pom.xml +++ b/apm-collector/apm-collector-ui/pom.xml @@ -11,4 +11,32 @@ apm-collector-ui jar + + + + org.skywalking + apm-collector-core + ${project.version} + + + org.skywalking + apm-collector-cluster + ${project.version} + + + org.skywalking + apm-collector-server + ${project.version} + + + org.skywalking + apm-collector-storage + ${project.version} + + + org.skywalking + apm-collector-agentstream + ${project.version} + + \ No newline at end of file diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/UIModuleContext.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/UIModuleContext.java new file mode 100644 index 000000000..5333eba1b --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/UIModuleContext.java @@ -0,0 +1,13 @@ +package org.skywalking.apm.collector.ui; + +import org.skywalking.apm.collector.core.framework.Context; + +/** + * @author pengys5 + */ +public class UIModuleContext extends Context { + + public UIModuleContext(String groupName) { + super(groupName); + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/UIModuleDefine.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/UIModuleDefine.java new file mode 100644 index 000000000..2f43dfeb4 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/UIModuleDefine.java @@ -0,0 +1,51 @@ +package org.skywalking.apm.collector.ui; + +import java.util.List; +import java.util.Map; +import org.skywalking.apm.collector.cluster.ClusterModuleGroupDefine; +import org.skywalking.apm.collector.core.client.Client; +import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.client.DataMonitor; +import org.skywalking.apm.collector.core.cluster.ClusterDataListenerDefine; +import org.skywalking.apm.collector.core.cluster.ClusterModuleContext; +import org.skywalking.apm.collector.core.config.ConfigParseException; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.framework.Handler; +import org.skywalking.apm.collector.core.module.ModuleDefine; +import org.skywalking.apm.collector.core.server.Server; +import org.skywalking.apm.collector.core.server.ServerException; +import org.skywalking.apm.collector.core.server.ServerHolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public abstract class UIModuleDefine extends ModuleDefine implements ClusterDataListenerDefine { + + private final Logger logger = LoggerFactory.getLogger(UIModuleDefine.class); + + @Override + public final void initialize(Map config, ServerHolder serverHolder) throws DefineException, ClientException { + try { + configParser().parse(config); + Server server = server(); + serverHolder.holdServer(server, handlerList()); + + ((ClusterModuleContext)CollectorContextHelper.INSTANCE.getContext(ClusterModuleGroupDefine.GROUP_NAME)).getDataMonitor().addListener(listener(), registration()); + } catch (ConfigParseException | ServerException e) { + throw new UIModuleException(e.getMessage(), e); + } + } + + @Override protected final Client createClient(DataMonitor dataMonitor) { + throw new UnsupportedOperationException(""); + } + + @Override public final boolean defaultModule() { + return true; + } + + public abstract List handlerList(); +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/UIModuleException.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/UIModuleException.java new file mode 100644 index 000000000..34662eb14 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/UIModuleException.java @@ -0,0 +1,16 @@ +package org.skywalking.apm.collector.ui; + +import org.skywalking.apm.collector.core.module.ModuleException; + +/** + * @author pengys5 + */ +public class UIModuleException extends ModuleException { + public UIModuleException(String message) { + super(message); + } + + public UIModuleException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/UIModuleGroupDefine.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/UIModuleGroupDefine.java new file mode 100644 index 000000000..575cd4175 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/UIModuleGroupDefine.java @@ -0,0 +1,25 @@ +package org.skywalking.apm.collector.ui; + +import org.skywalking.apm.collector.core.framework.Context; +import org.skywalking.apm.collector.core.module.ModuleGroupDefine; +import org.skywalking.apm.collector.core.module.ModuleInstaller; + +/** + * @author pengys5 + */ +public class UIModuleGroupDefine implements ModuleGroupDefine { + + public static final String GROUP_NAME = "ui"; + + @Override public String name() { + return GROUP_NAME; + } + + @Override public Context groupContext() { + return new UIModuleContext(GROUP_NAME); + } + + @Override public ModuleInstaller moduleInstaller() { + return new UIModuleInstaller(); + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/UIModuleInstaller.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/UIModuleInstaller.java new file mode 100644 index 000000000..4bda90127 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/UIModuleInstaller.java @@ -0,0 +1,36 @@ +package org.skywalking.apm.collector.ui; + +import java.util.Iterator; +import java.util.Map; +import org.skywalking.apm.collector.core.client.ClientException; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.module.ModuleDefine; +import org.skywalking.apm.collector.core.module.ModuleInstaller; +import org.skywalking.apm.collector.core.server.ServerHolder; +import org.skywalking.apm.collector.core.util.ObjectUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class UIModuleInstaller implements ModuleInstaller { + + private final Logger logger = LoggerFactory.getLogger(UIModuleInstaller.class); + + @Override public void install(Map moduleConfig, + Map moduleDefineMap, ServerHolder serverHolder) throws DefineException, ClientException { + logger.info("beginning ui module install"); + + UIModuleContext context = new UIModuleContext(UIModuleGroupDefine.GROUP_NAME); + CollectorContextHelper.INSTANCE.putContext(context); + + Iterator> moduleDefineEntry = moduleDefineMap.entrySet().iterator(); + while (moduleDefineEntry.hasNext()) { + ModuleDefine moduleDefine = moduleDefineEntry.next().getValue(); + logger.info("module {} initialize", moduleDefine.getClass().getName()); + moduleDefine.initialize((ObjectUtils.isNotEmpty(moduleConfig) && moduleConfig.containsKey(moduleDefine.name())) ? moduleConfig.get(moduleDefine.name()) : null, serverHolder); + } + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/GlobalTraceEsDAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/GlobalTraceEsDAO.java new file mode 100644 index 000000000..7148d393e --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/GlobalTraceEsDAO.java @@ -0,0 +1,59 @@ +package org.skywalking.apm.collector.ui.dao; + +import java.util.ArrayList; +import java.util.List; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.skywalking.apm.collector.agentstream.worker.global.define.GlobalTraceTable; +import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class GlobalTraceEsDAO extends EsDAO implements IGlobalTraceDAO { + + private final Logger logger = LoggerFactory.getLogger(GlobalTraceEsDAO.class); + + @Override public List getGlobalTraceId(String segmentId) { + SearchRequestBuilder searchRequestBuilder = getClient().prepareSearch(GlobalTraceTable.TABLE); + searchRequestBuilder.setTypes(GlobalTraceTable.TABLE_TYPE); + searchRequestBuilder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); + searchRequestBuilder.setQuery(QueryBuilders.termQuery(GlobalTraceTable.COLUMN_SEGMENT_ID, segmentId)); + searchRequestBuilder.setSize(10); + + SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); + + List globalTraceIds = new ArrayList<>(); + SearchHit[] searchHits = searchResponse.getHits().getHits(); + for (SearchHit searchHit : searchHits) { + String globalTraceId = (String)searchHit.getSource().get(GlobalTraceTable.COLUMN_GLOBAL_TRACE_ID); + logger.debug("segmentId: {}, global trace id: {}", segmentId, globalTraceId); + globalTraceIds.add(globalTraceId); + } + return globalTraceIds; + } + + @Override public List getSegmentIds(String globalTraceId) { + SearchRequestBuilder searchRequestBuilder = getClient().prepareSearch(GlobalTraceTable.TABLE); + searchRequestBuilder.setTypes(GlobalTraceTable.TABLE_TYPE); + searchRequestBuilder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); + searchRequestBuilder.setQuery(QueryBuilders.termQuery(GlobalTraceTable.COLUMN_GLOBAL_TRACE_ID, globalTraceId)); + searchRequestBuilder.setSize(10); + + SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); + + List segmentIds = new ArrayList<>(); + SearchHit[] searchHits = searchResponse.getHits().getHits(); + for (SearchHit searchHit : searchHits) { + String segmentId = (String)searchHit.getSource().get(GlobalTraceTable.COLUMN_SEGMENT_ID); + logger.debug("segmentId: {}, global trace id: {}", segmentId, globalTraceId); + segmentIds.add(segmentId); + } + return segmentIds; + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/GlobalTraceH2DAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/GlobalTraceH2DAO.java new file mode 100644 index 000000000..5e4082386 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/GlobalTraceH2DAO.java @@ -0,0 +1,17 @@ +package org.skywalking.apm.collector.ui.dao; + +import java.util.List; +import org.skywalking.apm.collector.storage.h2.dao.H2DAO; + +/** + * @author pengys5 + */ +public class GlobalTraceH2DAO extends H2DAO implements IGlobalTraceDAO { + @Override public List getGlobalTraceId(String segmentId) { + return null; + } + + @Override public List getSegmentIds(String globalTraceId) { + return null; + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/IGlobalTraceDAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/IGlobalTraceDAO.java new file mode 100644 index 000000000..fb468068d --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/IGlobalTraceDAO.java @@ -0,0 +1,12 @@ +package org.skywalking.apm.collector.ui.dao; + +import java.util.List; + +/** + * @author pengys5 + */ +public interface IGlobalTraceDAO { + List getGlobalTraceId(String segmentId); + + List getSegmentIds(String globalTraceId); +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/INodeComponentDAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/INodeComponentDAO.java new file mode 100644 index 000000000..36dad6d1c --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/INodeComponentDAO.java @@ -0,0 +1,10 @@ +package org.skywalking.apm.collector.ui.dao; + +import com.google.gson.JsonArray; + +/** + * @author pengys5 + */ +public interface INodeComponentDAO { + JsonArray load(long startTime, long endTime); +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/INodeMappingDAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/INodeMappingDAO.java new file mode 100644 index 000000000..17c6b266e --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/INodeMappingDAO.java @@ -0,0 +1,10 @@ +package org.skywalking.apm.collector.ui.dao; + +import com.google.gson.JsonArray; + +/** + * @author pengys5 + */ +public interface INodeMappingDAO { + JsonArray load(long startTime, long endTime); +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/INodeRefSumDAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/INodeRefSumDAO.java new file mode 100644 index 000000000..26a6e6239 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/INodeRefSumDAO.java @@ -0,0 +1,10 @@ +package org.skywalking.apm.collector.ui.dao; + +import com.google.gson.JsonArray; + +/** + * @author pengys5 + */ +public interface INodeRefSumDAO { + JsonArray load(long startTime, long endTime); +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/INodeReferenceDAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/INodeReferenceDAO.java new file mode 100644 index 000000000..947efec5e --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/INodeReferenceDAO.java @@ -0,0 +1,10 @@ +package org.skywalking.apm.collector.ui.dao; + +import com.google.gson.JsonArray; + +/** + * @author pengys5 + */ +public interface INodeReferenceDAO { + JsonArray load(long startTime, long endTime); +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/ISegmentCostDAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/ISegmentCostDAO.java new file mode 100644 index 000000000..43466b991 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/ISegmentCostDAO.java @@ -0,0 +1,11 @@ +package org.skywalking.apm.collector.ui.dao; + +import com.google.gson.JsonObject; + +/** + * @author pengys5 + */ +public interface ISegmentCostDAO { + JsonObject loadTop(long startTime, long endTime, long minCost, long maxCost, String operationName, + String globalTraceId, int limit, int from); +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/ISegmentDAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/ISegmentDAO.java new file mode 100644 index 000000000..a990201fd --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/ISegmentDAO.java @@ -0,0 +1,10 @@ +package org.skywalking.apm.collector.ui.dao; + +import org.skywalking.apm.network.proto.TraceSegmentObject; + +/** + * @author pengys5 + */ +public interface ISegmentDAO { + TraceSegmentObject load(String segmentId); +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeComponentEsDAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeComponentEsDAO.java new file mode 100644 index 000000000..3b3a9ecf4 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeComponentEsDAO.java @@ -0,0 +1,51 @@ +package org.skywalking.apm.collector.ui.dao; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.skywalking.apm.collector.agentstream.worker.Const; +import org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentTable; +import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class NodeComponentEsDAO extends EsDAO implements INodeComponentDAO { + + private final Logger logger = LoggerFactory.getLogger(NodeComponentEsDAO.class); + + @Override public JsonArray load(long startTime, long endTime) { + SearchRequestBuilder searchRequestBuilder = getClient().prepareSearch(NodeComponentTable.TABLE); + searchRequestBuilder.setTypes(NodeComponentTable.TABLE_TYPE); + searchRequestBuilder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); + searchRequestBuilder.setQuery(QueryBuilders.rangeQuery(NodeComponentTable.COLUMN_TIME_BUCKET).gte(startTime).lte(endTime)); + searchRequestBuilder.setSize(0); + + searchRequestBuilder.addAggregation(AggregationBuilders.terms(NodeComponentTable.COLUMN_AGG).field(NodeComponentTable.COLUMN_AGG).size(100)); + SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); + + Terms genders = searchResponse.getAggregations().get(NodeComponentTable.COLUMN_AGG); + + JsonArray nodeComponentArray = new JsonArray(); + for (Terms.Bucket entry : genders.getBuckets()) { + String aggId = entry.getKeyAsString(); + String[] aggIds = aggId.split(Const.IDS_SPLIT); + String code = aggIds[0]; + String peers = aggIds[1]; + + JsonObject nodeComponentObj = new JsonObject(); + nodeComponentObj.addProperty("name", code); + nodeComponentObj.addProperty("peers", peers); + nodeComponentArray.add(nodeComponentObj); + } + logger.debug("node component data: {}", nodeComponentArray.toString()); + return nodeComponentArray; + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeComponentH2DAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeComponentH2DAO.java new file mode 100644 index 000000000..5a73d42ce --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeComponentH2DAO.java @@ -0,0 +1,14 @@ +package org.skywalking.apm.collector.ui.dao; + +import com.google.gson.JsonArray; +import org.skywalking.apm.collector.storage.h2.dao.H2DAO; + +/** + * @author pengys5 + */ +public class NodeComponentH2DAO extends H2DAO implements INodeComponentDAO { + + @Override public JsonArray load(long startTime, long endTime) { + return null; + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeMappingEsDAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeMappingEsDAO.java new file mode 100644 index 000000000..f25dd83ad --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeMappingEsDAO.java @@ -0,0 +1,51 @@ +package org.skywalking.apm.collector.ui.dao; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.skywalking.apm.collector.agentstream.worker.Const; +import org.skywalking.apm.collector.agentstream.worker.node.mapping.define.NodeMappingTable; +import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class NodeMappingEsDAO extends EsDAO implements INodeMappingDAO { + + private final Logger logger = LoggerFactory.getLogger(NodeMappingEsDAO.class); + + @Override public JsonArray load(long startTime, long endTime) { + SearchRequestBuilder searchRequestBuilder = getClient().prepareSearch(NodeMappingTable.TABLE); + searchRequestBuilder.setTypes(NodeMappingTable.TABLE_TYPE); + searchRequestBuilder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); + searchRequestBuilder.setQuery(QueryBuilders.rangeQuery(NodeMappingTable.COLUMN_TIME_BUCKET).gte(startTime).lte(endTime)); + searchRequestBuilder.setSize(0); + + searchRequestBuilder.addAggregation(AggregationBuilders.terms(NodeMappingTable.COLUMN_AGG).field(NodeMappingTable.COLUMN_AGG).size(100)); + SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); + + Terms genders = searchResponse.getAggregations().get(NodeMappingTable.COLUMN_AGG); + + JsonArray nodeMappingArray = new JsonArray(); + for (Terms.Bucket entry : genders.getBuckets()) { + String aggId = entry.getKeyAsString(); + String[] aggIds = aggId.split(Const.IDS_SPLIT); + String code = aggIds[0]; + String peers = aggIds[1]; + + JsonObject nodeMappingObj = new JsonObject(); + nodeMappingObj.addProperty("code", code); + nodeMappingObj.addProperty("peers", peers); + nodeMappingArray.add(nodeMappingObj); + } + logger.debug("node mapping data: {}", nodeMappingArray.toString()); + return nodeMappingArray; + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeMappingH2DAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeMappingH2DAO.java new file mode 100644 index 000000000..afda33070 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeMappingH2DAO.java @@ -0,0 +1,14 @@ +package org.skywalking.apm.collector.ui.dao; + +import com.google.gson.JsonArray; +import org.skywalking.apm.collector.storage.h2.dao.H2DAO; + +/** + * @author pengys5 + */ +public class NodeMappingH2DAO extends H2DAO implements INodeMappingDAO { + + @Override public JsonArray load(long startTime, long endTime) { + return null; + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeRefSumEsDAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeRefSumEsDAO.java new file mode 100644 index 000000000..8d11dc07d --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeRefSumEsDAO.java @@ -0,0 +1,77 @@ +package org.skywalking.apm.collector.ui.dao; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.sum.Sum; +import org.skywalking.apm.collector.agentstream.worker.Const; +import org.skywalking.apm.collector.agentstream.worker.noderef.summary.define.NodeRefSumTable; +import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class NodeRefSumEsDAO extends EsDAO implements INodeRefSumDAO { + + private final Logger logger = LoggerFactory.getLogger(NodeRefSumEsDAO.class); + + @Override public JsonArray load(long startTime, long endTime) { + SearchRequestBuilder searchRequestBuilder = getClient().prepareSearch(NodeRefSumTable.TABLE); + searchRequestBuilder.setTypes(NodeRefSumTable.TABLE_TYPE); + searchRequestBuilder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); + searchRequestBuilder.setQuery(QueryBuilders.rangeQuery(NodeRefSumTable.COLUMN_TIME_BUCKET).gte(startTime).lte(endTime)); + searchRequestBuilder.setSize(0); + + TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms(NodeRefSumTable.COLUMN_AGG).field(NodeRefSumTable.COLUMN_AGG); + aggregationBuilder.subAggregation(AggregationBuilders.sum(NodeRefSumTable.COLUMN_ONE_SECOND_LESS).field(NodeRefSumTable.COLUMN_ONE_SECOND_LESS)); + aggregationBuilder.subAggregation(AggregationBuilders.sum(NodeRefSumTable.COLUMN_THREE_SECOND_LESS).field(NodeRefSumTable.COLUMN_THREE_SECOND_LESS)); + aggregationBuilder.subAggregation(AggregationBuilders.sum(NodeRefSumTable.COLUMN_FIVE_SECOND_LESS).field(NodeRefSumTable.COLUMN_FIVE_SECOND_LESS)); + aggregationBuilder.subAggregation(AggregationBuilders.sum(NodeRefSumTable.COLUMN_FIVE_SECOND_GREATER).field(NodeRefSumTable.COLUMN_FIVE_SECOND_GREATER)); + aggregationBuilder.subAggregation(AggregationBuilders.sum(NodeRefSumTable.COLUMN_ERROR).field(NodeRefSumTable.COLUMN_ERROR)); + aggregationBuilder.subAggregation(AggregationBuilders.sum(NodeRefSumTable.COLUMN_SUMMARY).field(NodeRefSumTable.COLUMN_SUMMARY)); + + searchRequestBuilder.addAggregation(aggregationBuilder); + + SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); + + JsonArray nodeRefResSumArray = new JsonArray(); + Terms aggTerms = searchResponse.getAggregations().get(NodeRefSumTable.COLUMN_AGG); + for (Terms.Bucket bucket : aggTerms.getBuckets()) { + String aggId = String.valueOf(bucket.getKey()); + Sum oneSecondLess = bucket.getAggregations().get(NodeRefSumTable.COLUMN_ONE_SECOND_LESS); + Sum threeSecondLess = bucket.getAggregations().get(NodeRefSumTable.COLUMN_THREE_SECOND_LESS); + Sum fiveSecondLess = bucket.getAggregations().get(NodeRefSumTable.COLUMN_FIVE_SECOND_LESS); + Sum fiveSecondGreater = bucket.getAggregations().get(NodeRefSumTable.COLUMN_FIVE_SECOND_GREATER); + Sum error = bucket.getAggregations().get(NodeRefSumTable.COLUMN_ERROR); + Sum summary = bucket.getAggregations().get(NodeRefSumTable.COLUMN_SUMMARY); + logger.debug("aggId: {}, oneSecondLess: {}, threeSecondLess: {}, fiveSecondLess: {}, fiveSecondGreater: {}, error: {}, summary: {}", aggId, + oneSecondLess.getValue(), threeSecondLess.getValue(), fiveSecondLess.getValue(), fiveSecondGreater.getValue(), error.getValue(), summary.getValue()); + + JsonObject nodeRefResSumObj = new JsonObject(); + String[] ids = aggId.split(Const.IDS_SPLIT); + String front = ids[0]; + String behind = ids[1]; + + nodeRefResSumObj.addProperty("front", front); + nodeRefResSumObj.addProperty("behind", behind); + + nodeRefResSumObj.addProperty(NodeRefSumTable.COLUMN_ONE_SECOND_LESS, oneSecondLess.getValue()); + nodeRefResSumObj.addProperty(NodeRefSumTable.COLUMN_THREE_SECOND_LESS, threeSecondLess.getValue()); + nodeRefResSumObj.addProperty(NodeRefSumTable.COLUMN_FIVE_SECOND_LESS, fiveSecondLess.getValue()); + nodeRefResSumObj.addProperty(NodeRefSumTable.COLUMN_FIVE_SECOND_GREATER, fiveSecondGreater.getValue()); + nodeRefResSumObj.addProperty(NodeRefSumTable.COLUMN_ERROR, error.getValue()); + nodeRefResSumObj.addProperty(NodeRefSumTable.COLUMN_SUMMARY, summary.getValue()); + nodeRefResSumArray.add(nodeRefResSumObj); + } + + return nodeRefResSumArray; + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeRefSumH2DAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeRefSumH2DAO.java new file mode 100644 index 000000000..8e4c0fab0 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeRefSumH2DAO.java @@ -0,0 +1,13 @@ +package org.skywalking.apm.collector.ui.dao; + +import com.google.gson.JsonArray; +import org.skywalking.apm.collector.storage.h2.dao.H2DAO; + +/** + * @author pengys5 + */ +public class NodeRefSumH2DAO extends H2DAO implements INodeRefSumDAO { + @Override public JsonArray load(long startTime, long endTime) { + return null; + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeReferenceEsDAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeReferenceEsDAO.java new file mode 100644 index 000000000..936ab1497 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeReferenceEsDAO.java @@ -0,0 +1,52 @@ +package org.skywalking.apm.collector.ui.dao; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.skywalking.apm.collector.agentstream.worker.Const; +import org.skywalking.apm.collector.agentstream.worker.noderef.reference.define.NodeRefTable; +import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class NodeReferenceEsDAO extends EsDAO implements INodeReferenceDAO { + + private final Logger logger = LoggerFactory.getLogger(NodeReferenceEsDAO.class); + + @Override public JsonArray load(long startTime, long endTime) { + SearchRequestBuilder searchRequestBuilder = getClient().prepareSearch(NodeRefTable.TABLE); + searchRequestBuilder.setTypes(NodeRefTable.TABLE_TYPE); + searchRequestBuilder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); + searchRequestBuilder.setQuery(QueryBuilders.rangeQuery(NodeRefTable.COLUMN_TIME_BUCKET).gte(startTime).lte(endTime)); + searchRequestBuilder.setSize(0); + + searchRequestBuilder.addAggregation(AggregationBuilders.terms(NodeRefTable.COLUMN_AGG).field(NodeRefTable.COLUMN_AGG).size(100)); + + SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); + + Terms genders = searchResponse.getAggregations().get(NodeRefTable.COLUMN_AGG); + + JsonArray nodeRefArray = new JsonArray(); + for (Terms.Bucket entry : genders.getBuckets()) { + String aggId = entry.getKeyAsString(); + String[] aggIds = aggId.split(Const.IDS_SPLIT); + String front = aggIds[0]; + String behind = aggIds[1]; + + JsonObject nodeRefObj = new JsonObject(); + nodeRefObj.addProperty("front", front); + nodeRefObj.addProperty("behind", behind); + nodeRefArray.add(nodeRefObj); + } + logger.debug("node ref data: {}", nodeRefArray.toString()); + return nodeRefArray; + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeReferenceH2DAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeReferenceH2DAO.java new file mode 100644 index 000000000..72c6cd65e --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/NodeReferenceH2DAO.java @@ -0,0 +1,13 @@ +package org.skywalking.apm.collector.ui.dao; + +import com.google.gson.JsonArray; +import org.skywalking.apm.collector.storage.h2.dao.H2DAO; + +/** + * @author pengys5 + */ +public class NodeReferenceH2DAO extends H2DAO implements INodeReferenceDAO { + @Override public JsonArray load(long startTime, long endTime) { + return null; + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/SegmentCostEsDAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/SegmentCostEsDAO.java new file mode 100644 index 000000000..c06df8079 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/SegmentCostEsDAO.java @@ -0,0 +1,90 @@ +package org.skywalking.apm.collector.ui.dao; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import java.util.List; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.sort.SortOrder; +import org.skywalking.apm.collector.agentstream.worker.global.define.GlobalTraceTable; +import org.skywalking.apm.collector.agentstream.worker.segment.cost.define.SegmentCostTable; +import org.skywalking.apm.collector.core.util.CollectionUtils; +import org.skywalking.apm.collector.core.util.StringUtils; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; + +/** + * @author pengys5 + */ +public class SegmentCostEsDAO extends EsDAO implements ISegmentCostDAO { + + @Override public JsonObject loadTop(long startTime, long endTime, long minCost, long maxCost, String operationName, + String globalTraceId, int limit, int from) { + SearchRequestBuilder searchRequestBuilder = getClient().prepareSearch(SegmentCostTable.TABLE); + searchRequestBuilder.setTypes(SegmentCostTable.TABLE_TYPE); + searchRequestBuilder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + searchRequestBuilder.setQuery(boolQueryBuilder); + List mustQueryList = boolQueryBuilder.must(); + + mustQueryList.add(QueryBuilders.rangeQuery(SegmentCostTable.COLUMN_TIME_BUCKET).gte(startTime).lte(endTime)); + if (minCost != -1 || maxCost != -1) { + RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(SegmentCostTable.COLUMN_COST); + if (minCost != -1) { + rangeQueryBuilder.gte(minCost); + } + if (maxCost != -1) { + rangeQueryBuilder.lte(maxCost); + } + boolQueryBuilder.must().add(rangeQueryBuilder); + } + if (!StringUtils.isEmpty(operationName)) { + mustQueryList.add(QueryBuilders.matchQuery(SegmentCostTable.COLUMN_OPERATION_NAME, operationName)); + } + + searchRequestBuilder.addSort(SegmentCostTable.COLUMN_COST, SortOrder.DESC); + searchRequestBuilder.setSize(limit); + searchRequestBuilder.setFrom(from); + + SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); + + JsonObject topSegPaging = new JsonObject(); + topSegPaging.addProperty("recordsTotal", searchResponse.getHits().totalHits); + + JsonArray topSegArray = new JsonArray(); + topSegPaging.add("data", topSegArray); + + int num = from; + for (SearchHit searchHit : searchResponse.getHits().getHits()) { + JsonObject topSegmentJson = new JsonObject(); + topSegmentJson.addProperty("num", num); + String segmentId = (String)searchHit.getSource().get(SegmentCostTable.COLUMN_SEGMENT_ID); + topSegmentJson.addProperty(SegmentCostTable.COLUMN_SEGMENT_ID, segmentId); + topSegmentJson.addProperty(SegmentCostTable.COLUMN_START_TIME, (Number)searchHit.getSource().get(SegmentCostTable.COLUMN_START_TIME)); + if (searchHit.getSource().containsKey(SegmentCostTable.COLUMN_END_TIME)) { + topSegmentJson.addProperty(SegmentCostTable.COLUMN_END_TIME, (Number)searchHit.getSource().get(SegmentCostTable.COLUMN_END_TIME)); + } + + IGlobalTraceDAO globalTraceDAO = (IGlobalTraceDAO)DAOContainer.INSTANCE.get(IGlobalTraceDAO.class.getName()); + List globalTraces = globalTraceDAO.getGlobalTraceId(segmentId); + if (CollectionUtils.isNotEmpty(globalTraces)) { + topSegmentJson.addProperty(GlobalTraceTable.COLUMN_GLOBAL_TRACE_ID, globalTraces.get(0)); + } + + topSegmentJson.addProperty(SegmentCostTable.COLUMN_OPERATION_NAME, (String)searchHit.getSource().get(SegmentCostTable.COLUMN_OPERATION_NAME)); + topSegmentJson.addProperty(SegmentCostTable.COLUMN_COST, (Number)searchHit.getSource().get(SegmentCostTable.COLUMN_COST)); + topSegmentJson.addProperty(SegmentCostTable.COLUMN_IS_ERROR, (Boolean)searchHit.getSource().get(SegmentCostTable.COLUMN_IS_ERROR)); + + num++; + topSegArray.add(topSegmentJson); + } + + return topSegPaging; + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/SegmentCostH2DAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/SegmentCostH2DAO.java new file mode 100644 index 000000000..fcafd0f60 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/SegmentCostH2DAO.java @@ -0,0 +1,14 @@ +package org.skywalking.apm.collector.ui.dao; + +import com.google.gson.JsonObject; +import org.skywalking.apm.collector.storage.h2.dao.H2DAO; + +/** + * @author pengys5 + */ +public class SegmentCostH2DAO extends H2DAO implements ISegmentCostDAO { + @Override public JsonObject loadTop(long startTime, long endTime, long minCost, long maxCost, String operationName, + String globalTraceId, int limit, int from) { + return null; + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/SegmentEsDAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/SegmentEsDAO.java new file mode 100644 index 000000000..fd1dc4362 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/SegmentEsDAO.java @@ -0,0 +1,35 @@ +package org.skywalking.apm.collector.ui.dao; + +import com.google.protobuf.InvalidProtocolBufferException; +import java.util.Base64; +import java.util.Map; +import org.elasticsearch.action.get.GetResponse; +import org.skywalking.apm.collector.agentstream.worker.segment.origin.define.SegmentTable; +import org.skywalking.apm.collector.core.util.StringUtils; +import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.network.proto.TraceSegmentObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class SegmentEsDAO extends EsDAO implements ISegmentDAO { + + private final Logger logger = LoggerFactory.getLogger(SegmentEsDAO.class); + + @Override public TraceSegmentObject load(String segmentId) { + GetResponse response = getClient().prepareGet(SegmentTable.TABLE, segmentId).get(); + Map source = response.getSource(); + String dataBinaryBase64 = (String)source.get(SegmentTable.COLUMN_DATA_BINARY); + if (StringUtils.isNotEmpty(dataBinaryBase64)) { + byte[] dataBinary = Base64.getDecoder().decode(dataBinaryBase64); + try { + return TraceSegmentObject.parseFrom(dataBinary); + } catch (InvalidProtocolBufferException e) { + logger.error(e.getMessage(), e); + } + } + return null; + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/SegmentH2DAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/SegmentH2DAO.java new file mode 100644 index 000000000..06b55d904 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/SegmentH2DAO.java @@ -0,0 +1,13 @@ +package org.skywalking.apm.collector.ui.dao; + +import org.skywalking.apm.collector.storage.h2.dao.H2DAO; +import org.skywalking.apm.network.proto.TraceSegmentObject; + +/** + * @author pengys5 + */ +public class SegmentH2DAO extends H2DAO implements ISegmentDAO { + @Override public TraceSegmentObject load(String segmentId) { + return null; + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/UIJettyConfig.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/UIJettyConfig.java new file mode 100644 index 000000000..0f761058a --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/UIJettyConfig.java @@ -0,0 +1,10 @@ +package org.skywalking.apm.collector.ui.jetty; + +/** + * @author pengys5 + */ +public class UIJettyConfig { + public static String HOST; + public static int PORT; + public static String CONTEXT_PATH; +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/UIJettyConfigParser.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/UIJettyConfigParser.java new file mode 100644 index 000000000..4f1a5baf5 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/UIJettyConfigParser.java @@ -0,0 +1,36 @@ +package org.skywalking.apm.collector.ui.jetty; + +import java.util.Map; +import org.skywalking.apm.collector.core.config.ConfigParseException; +import org.skywalking.apm.collector.core.module.ModuleConfigParser; +import org.skywalking.apm.collector.core.util.ObjectUtils; +import org.skywalking.apm.collector.core.util.StringUtils; + +/** + * @author pengys5 + */ +public class UIJettyConfigParser implements ModuleConfigParser { + + private static final String HOST = "host"; + private static final String PORT = "port"; + public static final String CONTEXT_PATH = "contextPath"; + + @Override public void parse(Map config) throws ConfigParseException { + UIJettyConfig.CONTEXT_PATH = "/"; + + if (ObjectUtils.isEmpty(config) || StringUtils.isEmpty(config.get(HOST))) { + UIJettyConfig.HOST = "localhost"; + } else { + UIJettyConfig.HOST = (String)config.get(HOST); + } + + if (ObjectUtils.isEmpty(config) || StringUtils.isEmpty(config.get(PORT))) { + UIJettyConfig.PORT = 12800; + } else { + UIJettyConfig.PORT = (Integer)config.get(PORT); + } + if (ObjectUtils.isNotEmpty(config) && StringUtils.isNotEmpty(config.get(CONTEXT_PATH))) { + UIJettyConfig.CONTEXT_PATH = (String)config.get(CONTEXT_PATH); + } + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/UIJettyDataListener.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/UIJettyDataListener.java new file mode 100644 index 000000000..00a1104dd --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/UIJettyDataListener.java @@ -0,0 +1,20 @@ +package org.skywalking.apm.collector.ui.jetty; + +import org.skywalking.apm.collector.cluster.ClusterModuleDefine; +import org.skywalking.apm.collector.core.cluster.ClusterDataListener; +import org.skywalking.apm.collector.ui.UIModuleGroupDefine; + +/** + * @author pengys5 + */ +public class UIJettyDataListener extends ClusterDataListener { + + public static final String PATH = ClusterModuleDefine.BASE_CATALOG + "." + UIModuleGroupDefine.GROUP_NAME + "." + UIJettyModuleDefine.MODULE_NAME; + + @Override public String path() { + return PATH; + } + + @Override public void addressChangedNotify() { + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/UIJettyModuleDefine.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/UIJettyModuleDefine.java new file mode 100644 index 000000000..9d0e73bb3 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/UIJettyModuleDefine.java @@ -0,0 +1,59 @@ +package org.skywalking.apm.collector.ui.jetty; + +import java.util.LinkedList; +import java.util.List; +import org.skywalking.apm.collector.core.cluster.ClusterDataListener; +import org.skywalking.apm.collector.core.framework.Handler; +import org.skywalking.apm.collector.core.module.ModuleConfigParser; +import org.skywalking.apm.collector.core.module.ModuleRegistration; +import org.skywalking.apm.collector.core.server.Server; +import org.skywalking.apm.collector.server.jetty.JettyServer; +import org.skywalking.apm.collector.ui.UIModuleDefine; +import org.skywalking.apm.collector.ui.UIModuleGroupDefine; +import org.skywalking.apm.collector.ui.jetty.handler.SegmentTopGetHandler; +import org.skywalking.apm.collector.ui.jetty.handler.SpanGetHandler; +import org.skywalking.apm.collector.ui.jetty.handler.TraceDagGetHandler; +import org.skywalking.apm.collector.ui.jetty.handler.TraceStackGetHandler; +import org.skywalking.apm.collector.ui.jetty.handler.UIJettyServerHandler; + +/** + * @author pengys5 + */ +public class UIJettyModuleDefine extends UIModuleDefine { + + public static final String MODULE_NAME = "jetty"; + + @Override protected String group() { + return UIModuleGroupDefine.GROUP_NAME; + } + + @Override public String name() { + return MODULE_NAME; + } + + @Override protected ModuleConfigParser configParser() { + return new UIJettyConfigParser(); + } + + @Override protected Server server() { + return new JettyServer(UIJettyConfig.HOST, UIJettyConfig.PORT, UIJettyConfig.CONTEXT_PATH); + } + + @Override protected ModuleRegistration registration() { + return new UIJettyModuleRegistration(); + } + + @Override public ClusterDataListener listener() { + return new UIJettyDataListener(); + } + + @Override public List handlerList() { + List handlers = new LinkedList<>(); + handlers.add(new UIJettyServerHandler()); + handlers.add(new TraceDagGetHandler()); + handlers.add(new SegmentTopGetHandler()); + handlers.add(new TraceStackGetHandler()); + handlers.add(new SpanGetHandler()); + return handlers; + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/UIJettyModuleRegistration.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/UIJettyModuleRegistration.java new file mode 100644 index 000000000..c29a8cb4e --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/UIJettyModuleRegistration.java @@ -0,0 +1,13 @@ +package org.skywalking.apm.collector.ui.jetty; + +import org.skywalking.apm.collector.core.module.ModuleRegistration; + +/** + * @author pengys5 + */ +public class UIJettyModuleRegistration extends ModuleRegistration { + + @Override public Value buildValue() { + return new Value(UIJettyConfig.HOST, UIJettyConfig.PORT, UIJettyConfig.CONTEXT_PATH); + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SegmentTopGetHandler.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SegmentTopGetHandler.java new file mode 100644 index 000000000..0feae9192 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SegmentTopGetHandler.java @@ -0,0 +1,82 @@ +package org.skywalking.apm.collector.ui.jetty.handler; + +import com.google.gson.JsonElement; +import javax.servlet.http.HttpServletRequest; +import org.skywalking.apm.collector.server.jetty.ArgumentsParseException; +import org.skywalking.apm.collector.server.jetty.JettyHandler; +import org.skywalking.apm.collector.ui.service.SegmentTopService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class SegmentTopGetHandler extends JettyHandler { + + private final Logger logger = LoggerFactory.getLogger(SegmentTopGetHandler.class); + + @Override public String pathSpec() { + return "/segment/top"; + } + + private SegmentTopService service = new SegmentTopService(); + + @Override protected JsonElement doGet(HttpServletRequest req) throws ArgumentsParseException { + if (!req.getParameterMap().containsKey("startTime") || !req.getParameterMap().containsKey("endTime") || !req.getParameterMap().containsKey("from") || !req.getParameterMap().containsKey("limit")) { + throw new ArgumentsParseException("the request parameter must contains startTime, endTime, from, limit"); + } + + if (logger.isDebugEnabled()) { + logger.debug("startTime: {}, endTime: {}, from: {}", req.getParameter("startTime"), req.getParameter("endTime"), req.getParameter("from")); + } + + long startTime; + try { + startTime = Long.valueOf(req.getParameter("startTime")); + } catch (NumberFormatException e) { + throw new ArgumentsParseException("the request parameter startTime must be a long"); + } + + long endTime; + try { + endTime = Long.valueOf(req.getParameter("endTime")); + } catch (NumberFormatException e) { + throw new ArgumentsParseException("the request parameter endTime must be a long"); + } + + int from; + try { + from = Integer.valueOf(req.getParameter("from")); + } catch (NumberFormatException e) { + throw new ArgumentsParseException("the request parameter from must be an integer"); + } + + int limit; + try { + limit = Integer.valueOf(req.getParameter("limit")); + } catch (NumberFormatException e) { + throw new ArgumentsParseException("the request parameter from must be an integer"); + } + + int minCost = -1; + if (req.getParameterMap().containsKey("minCost")) { + minCost = Integer.valueOf(req.getParameter("minCost")); + } + int maxCost = -1; + if (req.getParameterMap().containsKey("maxCost")) { + maxCost = Integer.valueOf(req.getParameter("maxCost")); + } + + String globalTraceId = null; + if (req.getParameterMap().containsKey("globalTraceId")) { + globalTraceId = req.getParameter("globalTraceId"); + } + + String operationName = null; + if (req.getParameterMap().containsKey("operationName")) { + operationName = req.getParameter("operationName"); + } + + return service.loadTop(startTime, endTime, minCost, maxCost, operationName, globalTraceId, limit, from); + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SpanGetHandler.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SpanGetHandler.java new file mode 100644 index 000000000..eda5239f1 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SpanGetHandler.java @@ -0,0 +1,38 @@ +package org.skywalking.apm.collector.ui.jetty.handler; + +import com.google.gson.JsonElement; +import javax.servlet.http.HttpServletRequest; +import org.skywalking.apm.collector.server.jetty.ArgumentsParseException; +import org.skywalking.apm.collector.server.jetty.JettyHandler; +import org.skywalking.apm.collector.ui.service.SpanService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class SpanGetHandler extends JettyHandler { + + private final Logger logger = LoggerFactory.getLogger(SpanGetHandler.class); + + @Override public String pathSpec() { + return "/span/spanId"; + } + + private SpanService service = new SpanService(); + + @Override protected JsonElement doGet(HttpServletRequest req) throws ArgumentsParseException { + String segmentId = req.getParameter("segmentId"); + String spanIdStr = req.getParameter("spanId"); + logger.debug("segmentSpanId: {}, spanIdStr: {}", segmentId, spanIdStr); + + int spanId; + try { + spanId = Integer.parseInt(spanIdStr); + } catch (NumberFormatException e) { + throw new ArgumentsParseException("span id must be integer"); + } + + return service.load(segmentId, spanId); + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceDagGetHandler.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceDagGetHandler.java new file mode 100644 index 000000000..71a8a2c40 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceDagGetHandler.java @@ -0,0 +1,34 @@ +package org.skywalking.apm.collector.ui.jetty.handler; + +import com.google.gson.JsonElement; +import javax.servlet.http.HttpServletRequest; +import org.skywalking.apm.collector.server.jetty.ArgumentsParseException; +import org.skywalking.apm.collector.server.jetty.JettyHandler; +import org.skywalking.apm.collector.ui.service.TraceDagService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class TraceDagGetHandler extends JettyHandler { + + private final Logger logger = LoggerFactory.getLogger(TraceDagGetHandler.class); + + @Override public String pathSpec() { + return "/traceDag"; + } + + private TraceDagService service = new TraceDagService(); + + @Override protected JsonElement doGet(HttpServletRequest req) throws ArgumentsParseException { + String startTimeStr = req.getParameter("startTime"); + String endTimeStr = req.getParameter("endTime"); + String timeBucketType = req.getParameter("timeBucketType"); + logger.debug("startTime: {}, endTimeStr: {}, timeBucketType: {}", startTimeStr, endTimeStr, timeBucketType); + + long startTime = Long.valueOf(startTimeStr); + long endTime = Long.valueOf(endTimeStr); + return service.load(startTime, endTime, timeBucketType); + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceStackGetHandler.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceStackGetHandler.java new file mode 100644 index 000000000..1792158b0 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceStackGetHandler.java @@ -0,0 +1,30 @@ +package org.skywalking.apm.collector.ui.jetty.handler; + +import com.google.gson.JsonElement; +import javax.servlet.http.HttpServletRequest; +import org.skywalking.apm.collector.server.jetty.ArgumentsParseException; +import org.skywalking.apm.collector.server.jetty.JettyHandler; +import org.skywalking.apm.collector.ui.service.TraceStackService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class TraceStackGetHandler extends JettyHandler { + + private final Logger logger = LoggerFactory.getLogger(TraceStackGetHandler.class); + + @Override public String pathSpec() { + return "/traceStack/globalTraceId"; + } + + private TraceStackService service = new TraceStackService(); + + @Override protected JsonElement doGet(HttpServletRequest req) throws ArgumentsParseException { + String globalTraceId = req.getParameter("globalTraceId"); + logger.debug("globalTraceId: {}", globalTraceId); + + return service.load(globalTraceId); + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/UIJettyServerHandler.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/UIJettyServerHandler.java new file mode 100644 index 000000000..ca6f39340 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/UIJettyServerHandler.java @@ -0,0 +1,33 @@ +package org.skywalking.apm.collector.ui.jetty.handler; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import java.util.List; +import javax.servlet.http.HttpServletRequest; +import org.skywalking.apm.collector.cluster.ClusterModuleGroupDefine; +import org.skywalking.apm.collector.core.cluster.ClusterModuleContext; +import org.skywalking.apm.collector.core.cluster.ClusterModuleRegistrationReader; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.server.jetty.ArgumentsParseException; +import org.skywalking.apm.collector.server.jetty.JettyHandler; +import org.skywalking.apm.collector.ui.jetty.UIJettyDataListener; + +/** + * @author pengys5 + */ +public class UIJettyServerHandler extends JettyHandler { + + @Override public String pathSpec() { + return "/ui/jetty"; + } + + @Override protected JsonElement doGet(HttpServletRequest req) throws ArgumentsParseException { + ClusterModuleRegistrationReader reader = ((ClusterModuleContext)CollectorContextHelper.INSTANCE.getContext(ClusterModuleGroupDefine.GROUP_NAME)).getReader(); + List servers = reader.read(UIJettyDataListener.PATH); + JsonArray serverArray = new JsonArray(); + servers.forEach(server -> { + serverArray.add(server); + }); + return serverArray; + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/service/SegmentTopService.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/service/SegmentTopService.java new file mode 100644 index 000000000..97958b48b --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/service/SegmentTopService.java @@ -0,0 +1,22 @@ +package org.skywalking.apm.collector.ui.service; + +import com.google.gson.JsonObject; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.ui.dao.ISegmentCostDAO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class SegmentTopService { + + private final Logger logger = LoggerFactory.getLogger(SegmentTopService.class); + + public JsonObject loadTop(long startTime, long endTime, long minCost, long maxCost, String operationName, + String globalTraceId, int limit, int from) { + logger.debug("startTime: {}, endTime: {}, minCost: {}, maxCost: {}, operationName: {}, globalTraceId: {}, limit: {}, from: {}", startTime, endTime, minCost, maxCost, operationName, globalTraceId, limit, from); + ISegmentCostDAO segmentCostDAO = (ISegmentCostDAO)DAOContainer.INSTANCE.get(ISegmentCostDAO.class.getName()); + return segmentCostDAO.loadTop(startTime, endTime, minCost, maxCost, operationName, globalTraceId, limit, from); + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/service/SpanService.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/service/SpanService.java new file mode 100644 index 000000000..b0e68267b --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/service/SpanService.java @@ -0,0 +1,61 @@ +package org.skywalking.apm.collector.ui.service; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import java.util.List; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.ui.dao.ISegmentDAO; +import org.skywalking.apm.network.proto.KeyWithStringValue; +import org.skywalking.apm.network.proto.LogMessage; +import org.skywalking.apm.network.proto.SpanObject; +import org.skywalking.apm.network.proto.TraceSegmentObject; + +/** + * @author pengys5 + */ +public class SpanService { + + public JsonObject load(String segmentId, int spanId) { + ISegmentDAO segmentDAO = (ISegmentDAO)DAOContainer.INSTANCE.get(ISegmentDAO.class.getName()); + TraceSegmentObject segmentObject = segmentDAO.load(segmentId); + + JsonObject spanJson = new JsonObject(); + List spans = segmentObject.getSpansList(); + for (SpanObject spanObject : spans) { + if (spanId == spanObject.getSpanId()) { + spanJson.addProperty("operationName", spanObject.getOperationName()); + spanJson.addProperty("startTime", spanObject.getStartTime()); + spanJson.addProperty("endTime", spanObject.getEndTime()); + + JsonArray logsArray = new JsonArray(); + List logs = spanObject.getLogsList(); + for (LogMessage logMessage : logs) { + JsonObject logJson = new JsonObject(); + logJson.addProperty("time", logMessage.getTime()); + + JsonArray logInfoArray = new JsonArray(); + for (KeyWithStringValue value : logMessage.getDataList()) { + JsonObject valueJson = new JsonObject(); + valueJson.addProperty("key", value.getKey()); + valueJson.addProperty("value", value.getValue()); + logInfoArray.add(valueJson); + } + logJson.add("logInfo", logInfoArray); + logsArray.add(logJson); + } + spanJson.add("logMessage", logsArray); + + JsonArray tagsArray = new JsonArray(); + for (KeyWithStringValue tagValue : spanObject.getTagsList()) { + JsonObject tagJson = new JsonObject(); + tagJson.addProperty("key", tagValue.getKey()); + tagJson.addProperty("value", tagValue.getValue()); + tagsArray.add(tagJson); + } + spanJson.add("tags", tagsArray); + } + } + + return spanJson; + } +} \ No newline at end of file diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/service/TraceDagDataBuilder.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/service/TraceDagDataBuilder.java new file mode 100644 index 000000000..5ffa44920 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/service/TraceDagDataBuilder.java @@ -0,0 +1,121 @@ +package org.skywalking.apm.collector.ui.service; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import java.util.HashMap; +import java.util.Map; +import org.skywalking.apm.collector.agentstream.worker.Const; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class TraceDagDataBuilder { + private final Logger logger = LoggerFactory.getLogger(TraceDagDataBuilder.class); + + private Integer nodeId = new Integer(-1); + private Map mappingMap = new HashMap<>(); + private Map nodeCompMap = new HashMap<>(); + private Map resSumMap = new HashMap<>(); + private Map nodeIdMap = new HashMap<>(); + private JsonArray pointArray = new JsonArray(); + private JsonArray lineArray = new JsonArray(); + + public JsonObject build(JsonArray nodeCompArray, JsonArray nodesMappingArray, JsonArray nodeRefsArray, + JsonArray resSumArray) { + changeMapping2Map(nodesMappingArray); + changeNodeComp2Map(nodeCompArray); + resSumMerge(resSumArray); + + for (int i = 0; i < nodeRefsArray.size(); i++) { + JsonObject nodeRefJsonObj = nodeRefsArray.get(i).getAsJsonObject(); + String front = nodeRefJsonObj.get("front").getAsString(); + String behind = nodeRefJsonObj.get("behind").getAsString(); + + String behindCode = findRealCode(behind); + logger.debug("behind: %s, behindCode: {}", behind, behindCode); + + JsonObject lineJsonObj = new JsonObject(); + lineJsonObj.addProperty("from", findOrCreateNode(front)); + lineJsonObj.addProperty("to", findOrCreateNode(behindCode)); + lineJsonObj.addProperty("resSum", resSumMap.get(front + Const.ID_SPLIT + behindCode)); + + lineArray.add(lineJsonObj); + logger.debug("line: {}", lineJsonObj); + } + + JsonObject dagJsonObj = new JsonObject(); + dagJsonObj.add("nodes", pointArray); + dagJsonObj.add("nodeRefs", lineArray); + return dagJsonObj; + } + + private Integer findOrCreateNode(String peers) { + if (nodeIdMap.containsKey(peers) && !peers.equals(Const.USER_CODE)) { + return nodeIdMap.get(peers); + } else { + nodeId++; + JsonObject nodeJsonObj = new JsonObject(); + nodeJsonObj.addProperty("id", nodeId); + nodeJsonObj.addProperty("peer", peers); + if (peers.equals(Const.USER_CODE)) { + nodeJsonObj.addProperty("component", Const.USER_CODE); + } else { + nodeJsonObj.addProperty("component", nodeCompMap.get(peers)); + } + pointArray.add(nodeJsonObj); + + nodeIdMap.put(peers, nodeId); + logger.debug("node: {}", nodeJsonObj); + } + return nodeId; + } + + private void changeMapping2Map(JsonArray nodesMappingArray) { + for (int i = 0; i < nodesMappingArray.size(); i++) { + JsonObject nodesMappingJsonObj = nodesMappingArray.get(i).getAsJsonObject(); + String code = nodesMappingJsonObj.get("code").getAsString(); + String peers = nodesMappingJsonObj.get("peers").getAsString(); + mappingMap.put(peers, code); + } + } + + private void changeNodeComp2Map(JsonArray nodeCompArray) { + for (int i = 0; i < nodeCompArray.size(); i++) { + JsonObject nodesJsonObj = nodeCompArray.get(i).getAsJsonObject(); + logger.debug(nodesJsonObj.toString()); + String component = nodesJsonObj.get("name").getAsString(); + String peers = nodesJsonObj.get("peers").getAsString(); + nodeCompMap.put(peers, component); + } + } + + private String findRealCode(String peers) { + if (mappingMap.containsKey(peers)) { + return mappingMap.get(peers); + } else { + return peers; + } + } + + private void resSumMerge(JsonArray resSumArray) { + for (int i = 0; i < resSumArray.size(); i++) { + JsonObject resSumJsonObj = resSumArray.get(i).getAsJsonObject(); + String front = resSumJsonObj.get("front").getAsString(); + String behind = resSumJsonObj.get("behind").getAsString(); + Long summary = resSumJsonObj.get("summary").getAsLong(); + + if (mappingMap.containsKey(behind)) { + behind = mappingMap.get(behind); + } + + String id = front + Const.ID_SPLIT + behind; + if (resSumMap.containsKey(id)) { + resSumMap.put(id, summary + resSumMap.get(id)); + } else { + resSumMap.put(id, summary); + } + } + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/service/TraceDagService.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/service/TraceDagService.java new file mode 100644 index 000000000..1dbc43fff --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/service/TraceDagService.java @@ -0,0 +1,39 @@ +package org.skywalking.apm.collector.ui.service; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.ui.dao.INodeComponentDAO; +import org.skywalking.apm.collector.ui.dao.INodeMappingDAO; +import org.skywalking.apm.collector.ui.dao.INodeRefSumDAO; +import org.skywalking.apm.collector.ui.dao.INodeReferenceDAO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class TraceDagService { + + private final Logger logger = LoggerFactory.getLogger(TraceDagService.class); + + public JsonObject load(long startTime, long endTime, String timeBucketType) { + logger.debug("startTime: {}, endTime: {}, timeSliceType: {}", startTime, endTime, timeBucketType); + INodeComponentDAO nodeComponentDAO = (INodeComponentDAO)DAOContainer.INSTANCE.get(INodeComponentDAO.class.getName()); + JsonArray nodeComponentArray = nodeComponentDAO.load(startTime, endTime); + + INodeMappingDAO nodeMappingDAO = (INodeMappingDAO)DAOContainer.INSTANCE.get(INodeMappingDAO.class.getName()); + JsonArray nodeMappingArray = nodeMappingDAO.load(startTime, endTime); + + INodeReferenceDAO nodeReferenceDAO = (INodeReferenceDAO)DAOContainer.INSTANCE.get(INodeReferenceDAO.class.getName()); + JsonArray nodeRefArray = nodeReferenceDAO.load(startTime, endTime); + + INodeRefSumDAO nodeRefSumDAO = (INodeRefSumDAO)DAOContainer.INSTANCE.get(INodeRefSumDAO.class.getName()); + JsonArray nodeRefSumArray = nodeRefSumDAO.load(startTime, endTime); + + TraceDagDataBuilder builder = new TraceDagDataBuilder(); + JsonObject traceDag = builder.build(nodeComponentArray, nodeMappingArray, nodeRefArray, nodeRefSumArray); + + return traceDag; + } +} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/service/TraceStackService.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/service/TraceStackService.java new file mode 100644 index 000000000..355aa6a61 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/service/TraceStackService.java @@ -0,0 +1,221 @@ +package org.skywalking.apm.collector.ui.service; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import java.util.ArrayList; +import java.util.List; +import org.skywalking.apm.collector.agentstream.worker.Const; +import org.skywalking.apm.collector.core.util.CollectionUtils; +import org.skywalking.apm.collector.core.util.ObjectUtils; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.ui.dao.IGlobalTraceDAO; +import org.skywalking.apm.collector.ui.dao.ISegmentDAO; +import org.skywalking.apm.network.proto.SpanObject; +import org.skywalking.apm.network.proto.TraceSegmentObject; +import org.skywalking.apm.network.proto.TraceSegmentReference; +import org.skywalking.apm.network.proto.UniqueId; + +/** + * @author pengys5 + */ +public class TraceStackService { + + public JsonArray load(String globalTraceId) { + IGlobalTraceDAO globalTraceDAO = (IGlobalTraceDAO)DAOContainer.INSTANCE.get(IGlobalTraceDAO.class.getName()); + ISegmentDAO segmentDAO = (ISegmentDAO)DAOContainer.INSTANCE.get(ISegmentDAO.class.getName()); + + List spans = new ArrayList<>(); + List segmentIds = globalTraceDAO.getSegmentIds(globalTraceId); + if (CollectionUtils.isNotEmpty(segmentIds)) { + for (String segmentId : segmentIds) { + TraceSegmentObject segment = segmentDAO.load(segmentId); + if (ObjectUtils.isNotEmpty(segment)) { + spans.addAll(buildSpanList(segmentId, segment)); + } + } + } + + List sortedSpans = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(spans)) { + List rootSpans = findRoot(spans); + + if (CollectionUtils.isNotEmpty(rootSpans)) { + rootSpans.forEach(span -> { + List childrenSpan = new ArrayList<>(); + childrenSpan.add(span); + findChildren(spans, span, childrenSpan); + sortedSpans.addAll(childrenSpan); + }); + } + } + minStartTime(sortedSpans); + + return toJsonArray(sortedSpans); + } + + private JsonArray toJsonArray(List sortedSpans) { + JsonArray traceStackArray = new JsonArray(); + sortedSpans.forEach(span -> { + JsonObject spanJson = new JsonObject(); + spanJson.addProperty("spanId", span.getSpanId()); + spanJson.addProperty("parentSpanId", span.getParentSpanId()); + spanJson.addProperty("segmentSpanId", span.getSegmentSpanId()); + spanJson.addProperty("segmentParentSpanId", span.getSegmentParentSpanId()); + spanJson.addProperty("startTime", span.getStartTime()); + spanJson.addProperty("operationName", span.getOperationName()); + spanJson.addProperty("applicationCode", span.getApplicationCode()); + spanJson.addProperty("cost", span.getCost()); + spanJson.addProperty("isRoot", span.isRoot()); + traceStackArray.add(spanJson); + }); + return traceStackArray; + } + + private void minStartTime(List spans) { + long minStartTime = Long.MAX_VALUE; + for (Span span : spans) { + if (span.getStartTime() < minStartTime) { + minStartTime = span.getStartTime(); + } + } + + for (Span span : spans) { + span.setStartTime(span.getStartTime() - minStartTime); + } + } + + private List buildSpanList(String segmentId, TraceSegmentObject segment) { + List spans = new ArrayList<>(); + if (segment.getSpansCount() > 0) { + for (SpanObject spanObject : segment.getSpansList()) { + int spanId = spanObject.getSpanId(); + int parentSpanId = spanObject.getParentSpanId(); + String segmentSpanId = segmentId + Const.SEGMENT_SPAN_SPLIT + String.valueOf(spanId); + String segmentParentSpanId = segmentId + Const.SEGMENT_SPAN_SPLIT + String.valueOf(parentSpanId); + long startTime = spanObject.getStartTime(); + String operationName = spanObject.getOperationName(); + String applicationCode = "Code" + String.valueOf(segment.getApplicationId()); + + long cost = spanObject.getEndTime() - spanObject.getStartTime(); + if (cost == 0) { + cost = 1; + } + + if (parentSpanId == -1 && segment.getRefsCount() > 0) { + for (TraceSegmentReference reference : segment.getRefsList()) { + parentSpanId = reference.getParentSpanId(); + UniqueId uniqueId = reference.getParentTraceSegmentId(); + + StringBuilder segmentIdBuilder = new StringBuilder(); + uniqueId.getIdPartsList().forEach(part -> segmentIdBuilder.append(String.valueOf(part))); + + String parentSegmentId = segmentIdBuilder.toString(); + segmentParentSpanId = parentSegmentId + Const.SEGMENT_SPAN_SPLIT + String.valueOf(parentSpanId); + + spans.add(new Span(spanId, parentSpanId, segmentSpanId, segmentParentSpanId, startTime, operationName, applicationCode, cost)); + } + } else { + spans.add(new Span(spanId, parentSpanId, segmentSpanId, segmentParentSpanId, startTime, operationName, applicationCode, cost)); + } + } + } + return spans; + } + + private List findRoot(List spans) { + List rootSpans = new ArrayList<>(); + spans.forEach(span -> { + String segmentParentSpanId = span.getSegmentParentSpanId(); + + boolean hasParent = false; + for (Span span1 : spans) { + if (segmentParentSpanId.equals(span1.getSegmentSpanId())) { + hasParent = true; + } + } + + if (!hasParent) { + span.setRoot(true); + rootSpans.add(span); + } + }); + return rootSpans; + } + + private void findChildren(List spans, Span parentSpan, List childrenSpan) { + spans.forEach(span -> { + if (span.getSegmentParentSpanId().equals(parentSpan.getSegmentSpanId())) { + childrenSpan.add(span); + findChildren(spans, span, childrenSpan); + } + }); + } + + class Span { + private int spanId; + private int parentSpanId; + private String segmentSpanId; + private String segmentParentSpanId; + private long startTime; + private String operationName; + private String applicationCode; + private long cost; + private boolean isRoot = false; + + Span(int spanId, int parentSpanId, String segmentSpanId, String segmentParentSpanId, long startTime, + String operationName, String applicationCode, long cost) { + this.spanId = spanId; + this.parentSpanId = parentSpanId; + this.segmentSpanId = segmentSpanId; + this.segmentParentSpanId = segmentParentSpanId; + this.startTime = startTime; + this.operationName = operationName; + this.applicationCode = applicationCode; + this.cost = cost; + } + + int getSpanId() { + return spanId; + } + + int getParentSpanId() { + return parentSpanId; + } + + String getSegmentSpanId() { + return segmentSpanId; + } + + String getSegmentParentSpanId() { + return segmentParentSpanId; + } + + long getStartTime() { + return startTime; + } + + String getOperationName() { + return operationName; + } + + String getApplicationCode() { + return applicationCode; + } + + long getCost() { + return cost; + } + + public boolean isRoot() { + return isRoot; + } + + public void setRoot(boolean root) { + isRoot = root; + } + + public void setStartTime(long startTime) { + this.startTime = startTime; + } + } +} diff --git a/apm-collector/apm-collector-ui/src/main/resources/META-INF/defines/es_dao.define b/apm-collector/apm-collector-ui/src/main/resources/META-INF/defines/es_dao.define new file mode 100644 index 000000000..646bb9a3f --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/resources/META-INF/defines/es_dao.define @@ -0,0 +1,7 @@ +org.skywalking.apm.collector.ui.dao.NodeComponentEsDAO +org.skywalking.apm.collector.ui.dao.NodeMappingEsDAO +org.skywalking.apm.collector.ui.dao.NodeReferenceEsDAO +org.skywalking.apm.collector.ui.dao.NodeRefSumEsDAO +org.skywalking.apm.collector.ui.dao.SegmentCostEsDAO +org.skywalking.apm.collector.ui.dao.GlobalTraceEsDAO +org.skywalking.apm.collector.ui.dao.SegmentEsDAO \ No newline at end of file diff --git a/apm-collector/apm-collector-ui/src/main/resources/META-INF/defines/group.define b/apm-collector/apm-collector-ui/src/main/resources/META-INF/defines/group.define new file mode 100644 index 000000000..35b83f180 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/resources/META-INF/defines/group.define @@ -0,0 +1 @@ +org.skywalking.apm.collector.ui.UIModuleGroupDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-ui/src/main/resources/META-INF/defines/h2_dao.define b/apm-collector/apm-collector-ui/src/main/resources/META-INF/defines/h2_dao.define new file mode 100644 index 000000000..4b9a02fe8 --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/resources/META-INF/defines/h2_dao.define @@ -0,0 +1,7 @@ +org.skywalking.apm.collector.ui.dao.NodeComponentH2DAO +org.skywalking.apm.collector.ui.dao.NodeMappingH2DAO +org.skywalking.apm.collector.ui.dao.NodeReferenceH2DAO +org.skywalking.apm.collector.ui.dao.NodeRefSumH2DAO +org.skywalking.apm.collector.ui.dao.SegmentCostH2DAO +org.skywalking.apm.collector.ui.dao.GlobalTraceH2DAO +org.skywalking.apm.collector.ui.dao.SegmentH2DAO \ No newline at end of file diff --git a/apm-collector/apm-collector-ui/src/main/resources/META-INF/defines/module.define b/apm-collector/apm-collector-ui/src/main/resources/META-INF/defines/module.define new file mode 100644 index 000000000..285a6d99d --- /dev/null +++ b/apm-collector/apm-collector-ui/src/main/resources/META-INF/defines/module.define @@ -0,0 +1 @@ +org.skywalking.apm.collector.ui.jetty.UIJettyModuleDefine \ No newline at end of file diff --git a/apm-collector/pom.xml b/apm-collector/pom.xml index d7b882735..aa077d687 100644 --- a/apm-collector/pom.xml +++ b/apm-collector/pom.xml @@ -9,11 +9,13 @@ apm-collector-storage apm-collector-client apm-collector-server - apm-collector-discovery apm-collector-agentstream apm-collector-ui apm-collector-boot - apm-collector-remote + apm-collector-stream + apm-collector-agentserver + apm-collector-agentregister + apm-collector-agentjvm apm @@ -24,10 +26,28 @@ pom - 2.4.17 - 2.8.1 + + org.slf4j + slf4j-api + 1.7.25 + + + ch.qos.logback + logback-classic + 1.2.3 + + + org.slf4j + log4j-over-slf4j + 1.7.25 + + + org.apache.logging.log4j + log4j-core + 2.8.2 + diff --git a/pom.xml b/pom.xml index 45d1a7924..6a624830a 100644 --- a/pom.xml +++ b/pom.xml @@ -159,25 +159,17 @@ - org.codehaus.mojo - cobertura-maven-plugin - 2.7 - - true - UTF-8 - true - - xml - html - - - - org/skywalking/apm/network/**/*.class - org/skywalking/apm/collector/remote/grpc/**/*.class - org/skywalking/apm/agent/core/context/ids/base64/*.class - - - + org.jacoco + jacoco-maven-plugin + 0.7.9 + + + prepare-agent + + prepare-agent + + + com.spotify