From 82dd3c4474a3056d9e54a5997bbe4ca63ad37243 Mon Sep 17 00:00:00 2001 From: pengys5 <8082209@qq.com> Date: Sat, 5 Aug 2017 13:10:37 +0800 Subject: [PATCH 1/8] Save exchanged and not exchanged data to storage. e.g. node reference, A -> B will be 1->2 or 1->10.0.0.1 --- .../application/ApplicationIDService.java | 6 +- .../AgentStreamModuleInstaller.java | 2 - .../handler/TraceSegmentServiceHandler.java | 34 ++++++++ .../agentstream/worker/CommonTable.java | 1 - .../worker/cache/ApplicationCache.java | 25 ++++++ .../worker/cache/ComponentCache.java | 26 ------ .../worker/cache/ServiceNameCache.java | 26 ++++++ .../global/GlobalTracePersistenceWorker.java | 13 +-- .../worker/global/dao/GlobalTraceEsDAO.java | 35 ++++---- .../worker/global/dao/GlobalTraceH2DAO.java | 6 -- .../worker/global/dao/IGlobalTraceDAO.java | 5 -- .../NodeComponentExchangeWorker.java | 83 ------------------- .../NodeComponentPersistenceWorker.java | 13 +-- .../component/NodeComponentSpanListener.java | 80 +++++++++--------- .../node/component/dao/INodeComponentDAO.java | 7 -- .../component/dao/NodeComponentEsDAO.java | 67 +++++++-------- .../component/dao/NodeComponentH2DAO.java | 9 -- .../define/NodeComponentDataDefine.java | 69 ++++++--------- .../define/NodeComponentEsTableDefine.java | 5 +- .../define/NodeComponentH2TableDefine.java | 5 +- .../component/define/NodeComponentTable.java | 3 - .../mapping/NodeMappingPersistenceWorker.java | 13 +-- .../node/mapping/NodeMappingSpanListener.java | 5 +- .../node/mapping/dao/INodeMappingDAO.java | 5 -- .../node/mapping/dao/NodeMappingEsDAO.java | 44 +++++++--- .../node/mapping/dao/NodeMappingH2DAO.java | 6 -- .../mapping/define/NodeMappingDataDefine.java | 11 ++- .../reference/NodeRefPersistenceWorker.java | 13 +-- .../reference/NodeRefSpanListener.java | 8 +- .../reference/dao/INodeReferenceDAO.java | 5 -- .../reference/dao/NodeReferenceEsDAO.java | 44 +++++++--- .../reference/dao/NodeReferenceH2DAO.java | 17 +++- .../reference/define/NodeRefDataDefine.java | 11 +-- .../summary/NodeRefSumPersistenceWorker.java | 13 +-- .../summary/NodeRefSumSpanListener.java | 7 +- .../noderef/summary/dao/INodeRefSumDAO.java | 5 -- .../noderef/summary/dao/NodeRefSumEsDAO.java | 69 ++++++++++----- .../noderef/summary/dao/NodeRefSumH2DAO.java | 6 -- .../worker/segment/SegmentParse.java | 1 - .../cost/SegmentCostPersistenceWorker.java | 13 +-- .../segment/cost/SegmentCostSpanListener.java | 9 +- .../segment/cost/dao/ISegmentCostDAO.java | 5 -- .../segment/cost/dao/SegmentCostEsDAO.java | 44 +++++----- .../segment/cost/dao/SegmentCostH2DAO.java | 5 -- .../cost/define/SegmentCostDataDefine.java | 24 +++--- .../cost/define/SegmentCostEsTableDefine.java | 2 +- .../cost/define/SegmentCostH2TableDefine.java | 2 +- .../segment/cost/define/SegmentCostTable.java | 2 +- .../origin/SegmentPersistenceWorker.java | 13 +-- .../segment/origin/dao/ISegmentDAO.java | 5 -- .../segment/origin/dao/SegmentEsDAO.java | 31 +++---- .../segment/origin/dao/SegmentH2DAO.java | 6 -- .../worker/storage/IDNameExchangeTimer.java | 41 --------- .../worker/storage/PersistenceTimer.java | 2 +- .../worker/util/ExchangeMarkUtils.java | 14 ++++ .../resources/META-INF/defines/data.define | 4 - ...er.define => local_worker_provider.define} | 1 - .../TraceSegmentServiceHandlerTestCase.java | 2 +- .../elasticsearch/ElasticSearchClient.java | 6 ++ .../core/storage/StorageInstaller.java | 7 +- .../collector/server/jetty/JettyHandler.java | 21 ++++- .../define/ElasticSearchStorageInstaller.java | 2 +- .../storage/h2/define/H2StorageInstaller.java | 2 +- .../stream/StreamModuleInstaller.java | 6 -- .../worker/AbstractLocalAsyncWorker.java | 12 ++- .../AbstractLocalAsyncWorkerProvider.java | 7 +- .../worker/AbstractLocalSyncWorker.java | 46 ---------- .../AbstractLocalSyncWorkerProvider.java | 15 ---- .../stream/worker/AbstractLocalWorker.java | 10 --- .../worker/AbstractLocalWorkerProvider.java | 7 -- .../stream/worker/AbstractRemoteWorker.java | 12 ++- .../stream/worker/AbstractWorker.java | 6 +- .../LocalAsyncWorkerProviderDefineLoader.java | 2 +- .../stream/worker/LocalSyncWorkerRef.java | 23 ----- ...=> LocalWorkerProviderDefinitionFile.java} | 4 +- .../stream/worker/impl/ExchangeWorker.java | 74 ----------------- .../worker/impl/ExchangeWorkerContainer.java | 21 ----- .../stream/worker/impl/PersistenceWorker.java | 31 ++++++- .../worker/impl/dao/IPersistenceDAO.java | 15 ++++ .../worker/impl/data/DataDefineLoader.java | 29 ------- .../worker/impl/data/DataDefinitionFile.java | 13 --- .../stream/worker/impl/data/Exchange.java | 24 ------ .../collector/ui/dao/SegmentCostEsDAO.java | 4 +- 83 files changed, 592 insertions(+), 835 deletions(-) create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/TraceSegmentServiceHandler.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/ApplicationCache.java delete mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/ComponentCache.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/ServiceNameCache.java delete mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentExchangeWorker.java delete mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/IDNameExchangeTimer.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/util/ExchangeMarkUtils.java delete mode 100644 apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define rename apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/{local_async_worker_provider.define => local_worker_provider.define} (90%) delete mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalSyncWorker.java delete mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalSyncWorkerProvider.java delete mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorker.java delete mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorkerProvider.java delete mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalSyncWorkerRef.java rename apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/{LocalAsyncWorkerProviderDefinitionFile.java => LocalWorkerProviderDefinitionFile.java} (60%) delete mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/ExchangeWorker.java delete mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/ExchangeWorkerContainer.java create mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/dao/IPersistenceDAO.java delete mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefineLoader.java delete mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefinitionFile.java delete mode 100644 apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Exchange.java diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDService.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDService.java index 2cf836e90..fd169c8f7 100644 --- a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDService.java +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDService.java @@ -1,10 +1,9 @@ package org.skywalking.apm.collector.agentregister.application; +import org.skywalking.apm.collector.agentstream.worker.cache.ApplicationCache; import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine; import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterRemoteWorker; -import org.skywalking.apm.collector.agentstream.worker.register.application.dao.IApplicationDAO; import org.skywalking.apm.collector.core.framework.CollectorContextHelper; -import org.skywalking.apm.collector.storage.dao.DAOContainer; import org.skywalking.apm.collector.stream.StreamModuleContext; import org.skywalking.apm.collector.stream.StreamModuleGroupDefine; import org.skywalking.apm.collector.stream.worker.WorkerInvokeException; @@ -20,8 +19,7 @@ public class ApplicationIDService { private final Logger logger = LoggerFactory.getLogger(ApplicationIDService.class); public int getOrCreate(String applicationCode) { - IApplicationDAO dao = (IApplicationDAO)DAOContainer.INSTANCE.get(IApplicationDAO.class.getName()); - int applicationId = dao.getApplicationId(applicationCode); + int applicationId = ApplicationCache.get(applicationCode); if (applicationId == 0) { StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleInstaller.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleInstaller.java index 2b53fa68a..bf5227d48 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleInstaller.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleInstaller.java @@ -2,7 +2,6 @@ package org.skywalking.apm.collector.agentstream; import java.util.Iterator; import java.util.Map; -import org.skywalking.apm.collector.agentstream.worker.storage.IDNameExchangeTimer; import org.skywalking.apm.collector.agentstream.worker.storage.PersistenceTimer; import org.skywalking.apm.collector.core.client.ClientException; import org.skywalking.apm.collector.core.framework.CollectorContextHelper; @@ -36,6 +35,5 @@ public class AgentStreamModuleInstaller implements ModuleInstaller { } new PersistenceTimer().start(); - new IDNameExchangeTimer().start(); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/TraceSegmentServiceHandler.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/TraceSegmentServiceHandler.java new file mode 100644 index 000000000..a92102319 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/TraceSegmentServiceHandler.java @@ -0,0 +1,34 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler; + +import com.google.gson.JsonElement; +import java.io.BufferedReader; +import java.io.IOException; +import javax.servlet.http.HttpServletRequest; +import org.skywalking.apm.collector.server.jetty.ArgumentsParseException; +import org.skywalking.apm.collector.server.jetty.JettyHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class TraceSegmentServiceHandler extends JettyHandler { + + private final Logger logger = LoggerFactory.getLogger(TraceSegmentServiceHandler.class); + + @Override public String pathSpec() { + return null; + } + + @Override protected JsonElement doGet(HttpServletRequest req) throws ArgumentsParseException { + throw new UnsupportedOperationException(); + } + + @Override protected void doPost(HttpServletRequest req) throws ArgumentsParseException { + try { + BufferedReader reader = req.getReader(); + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/CommonTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/CommonTable.java index 9ddc733b8..c66f627ad 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/CommonTable.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/CommonTable.java @@ -7,6 +7,5 @@ public class CommonTable { public static final String TABLE_TYPE = "type"; public static final String COLUMN_ID = "id"; public static final String COLUMN_AGG = "agg"; - public static final String COLUMN_EXCHANGE_TIMES = "exchange_times"; public static final String COLUMN_TIME_BUCKET = "time_bucket"; } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/ApplicationCache.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/ApplicationCache.java new file mode 100644 index 000000000..ff5574029 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/ApplicationCache.java @@ -0,0 +1,25 @@ +package org.skywalking.apm.collector.agentstream.worker.cache; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.skywalking.apm.collector.agentstream.worker.register.application.dao.IApplicationDAO; +import org.skywalking.apm.collector.storage.dao.DAOContainer; + +/** + * @author pengys5 + */ +public class ApplicationCache { + + private static Cache CACHE = CacheBuilder.newBuilder().maximumSize(1000).build(); + + public static int get(String applicationCode) { + try { + return CACHE.get(applicationCode, () -> { + IApplicationDAO dao = (IApplicationDAO)DAOContainer.INSTANCE.get(IApplicationDAO.class.getName()); + return dao.getApplicationId(applicationCode); + }); + } catch (Throwable e) { + return 0; + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/ComponentCache.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/ComponentCache.java deleted file mode 100644 index fe472ce6e..000000000 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/ComponentCache.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.skywalking.apm.collector.agentstream.worker.cache; - -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import org.skywalking.apm.collector.agentstream.worker.Const; -import org.skywalking.apm.collector.agentstream.worker.node.component.dao.INodeComponentDAO; -import org.skywalking.apm.collector.storage.dao.DAOContainer; - -/** - * @author pengys5 - */ -public class ComponentCache { - - private static Cache CACHE = CacheBuilder.newBuilder().maximumSize(1000).build(); - - public static int get(int applicationId, String componentName) { - try { - return CACHE.get(applicationId + Const.ID_SPLIT + componentName, () -> { - INodeComponentDAO dao = (INodeComponentDAO)DAOContainer.INSTANCE.get(INodeComponentDAO.class.getName()); - return dao.getComponentId(applicationId, componentName); - }); - } catch (Throwable e) { - return 0; - } - } -} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/ServiceNameCache.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/ServiceNameCache.java new file mode 100644 index 000000000..f6a88b980 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/ServiceNameCache.java @@ -0,0 +1,26 @@ +package org.skywalking.apm.collector.agentstream.worker.cache; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.skywalking.apm.collector.agentstream.worker.Const; +import org.skywalking.apm.collector.agentstream.worker.register.servicename.dao.IServiceNameDAO; +import org.skywalking.apm.collector.storage.dao.DAOContainer; + +/** + * @author pengys5 + */ +public class ServiceNameCache { + + private static Cache CACHE = CacheBuilder.newBuilder().maximumSize(2000).build(); + + public static int get(int applicationId, String serviceName) { + try { + return CACHE.get(applicationId + Const.ID_SPLIT + serviceName, () -> { + IServiceNameDAO dao = (IServiceNameDAO)DAOContainer.INSTANCE.get(IServiceNameDAO.class.getName()); + return dao.getServiceId(applicationId, serviceName); + }); + } catch (Throwable e) { + return 0; + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/GlobalTracePersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/GlobalTracePersistenceWorker.java index ba58718b6..8998c71ee 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/GlobalTracePersistenceWorker.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/GlobalTracePersistenceWorker.java @@ -1,7 +1,5 @@ package org.skywalking.apm.collector.agentstream.worker.global; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.agentstream.worker.global.dao.IGlobalTraceDAO; import org.skywalking.apm.collector.agentstream.worker.global.define.GlobalTraceDataDefine; import org.skywalking.apm.collector.storage.dao.DAOContainer; @@ -10,7 +8,7 @@ import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; import org.skywalking.apm.collector.stream.worker.Role; import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; import org.skywalking.apm.collector.stream.worker.selector.RollingSelector; import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; @@ -28,9 +26,12 @@ public class GlobalTracePersistenceWorker extends PersistenceWorker { super.preStart(); } - @Override protected List prepareBatch(Map dataMap) { - IGlobalTraceDAO dao = (IGlobalTraceDAO)DAOContainer.INSTANCE.get(IGlobalTraceDAO.class.getName()); - return dao.prepareBatch(dataMap); + @Override protected boolean needMergeDBData() { + return false; + } + + @Override protected IPersistenceDAO persistenceDAO() { + return (IPersistenceDAO)DAOContainer.INSTANCE.get(IGlobalTraceDAO.class.getName()); } public static class Factory extends AbstractLocalAsyncWorkerProvider { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/GlobalTraceEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/GlobalTraceEsDAO.java index 4c1c49707..c36ca1108 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/GlobalTraceEsDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/GlobalTraceEsDAO.java @@ -1,35 +1,38 @@ package org.skywalking.apm.collector.agentstream.worker.global.dao; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.skywalking.apm.collector.agentstream.worker.global.define.GlobalTraceTable; import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author pengys5 */ -public class GlobalTraceEsDAO extends EsDAO implements IGlobalTraceDAO { +public class GlobalTraceEsDAO extends EsDAO implements IGlobalTraceDAO, IPersistenceDAO { private final Logger logger = LoggerFactory.getLogger(GlobalTraceEsDAO.class); - @Override public List prepareBatch(Map dataMap) { - List indexRequestBuilders = new ArrayList<>(); - dataMap.forEach((id, data) -> { - logger.debug("global trace prepareBatch, id: {}", id); - Map source = new HashMap(); - source.put(GlobalTraceTable.COLUMN_SEGMENT_ID, data.getDataString(1)); - source.put(GlobalTraceTable.COLUMN_GLOBAL_TRACE_ID, data.getDataString(2)); - source.put(GlobalTraceTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); - logger.debug("global trace source: {}", source.toString()); - IndexRequestBuilder builder = getClient().prepareIndex(GlobalTraceTable.TABLE, id).setSource(source); - indexRequestBuilders.add(builder); - }); - return indexRequestBuilders; + @Override public Data get(String id, DataDefine dataDefine) { + return null; + } + + @Override public UpdateRequestBuilder prepareBatchUpdate(Data data) { + return null; + } + + @Override public IndexRequestBuilder prepareBatchInsert(Data data) { + Map source = new HashMap<>(); + source.put(GlobalTraceTable.COLUMN_SEGMENT_ID, data.getDataString(1)); + source.put(GlobalTraceTable.COLUMN_GLOBAL_TRACE_ID, data.getDataString(2)); + source.put(GlobalTraceTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + logger.debug("global trace source: {}", source.toString()); + return getClient().prepareIndex(GlobalTraceTable.TABLE, data.getDataString(0)).setSource(source); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/GlobalTraceH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/GlobalTraceH2DAO.java index 1a34c616a..ea48e612f 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/GlobalTraceH2DAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/GlobalTraceH2DAO.java @@ -1,15 +1,9 @@ package org.skywalking.apm.collector.agentstream.worker.global.dao; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.storage.h2.dao.H2DAO; /** * @author pengys5 */ public class GlobalTraceH2DAO extends H2DAO implements IGlobalTraceDAO { - - @Override public List prepareBatch(Map map) { - return null; - } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/IGlobalTraceDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/IGlobalTraceDAO.java index 4138f423c..1b73837c8 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/IGlobalTraceDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/IGlobalTraceDAO.java @@ -1,12 +1,7 @@ package org.skywalking.apm.collector.agentstream.worker.global.dao; -import java.util.List; -import java.util.Map; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; - /** * @author pengys5 */ public interface IGlobalTraceDAO { - List prepareBatch(Map dataMap); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentExchangeWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentExchangeWorker.java deleted file mode 100644 index 056847430..000000000 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentExchangeWorker.java +++ /dev/null @@ -1,83 +0,0 @@ -package org.skywalking.apm.collector.agentstream.worker.node.component; - -import org.skywalking.apm.collector.agentstream.worker.cache.ComponentCache; -import org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentDataDefine; -import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; -import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; -import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; -import org.skywalking.apm.collector.stream.worker.Role; -import org.skywalking.apm.collector.stream.worker.WorkerInvokeException; -import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; -import org.skywalking.apm.collector.stream.worker.impl.ExchangeWorker; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; -import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; -import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; -import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author pengys5 - */ -public class NodeComponentExchangeWorker extends ExchangeWorker { - - private final Logger logger = LoggerFactory.getLogger(NodeComponentExchangeWorker.class); - - public NodeComponentExchangeWorker(Role role, ClusterWorkerContext clusterContext) { - super(role, clusterContext); - } - - @Override public void preStart() throws ProviderNotFoundException { - super.preStart(); - } - - @Override protected void exchange(Data data) { - NodeComponentDataDefine.NodeComponent nodeComponent = new NodeComponentDataDefine.NodeComponent(); - nodeComponent.toSelf(data); - - int componentId = ComponentCache.get(nodeComponent.getApplicationId(), nodeComponent.getComponentName()); - if (componentId == 0 && nodeComponent.getTimes() < 10) { - try { - nodeComponent.increase(); - getClusterContext().lookup(NodeComponentExchangeWorker.WorkerRole.INSTANCE).tell(nodeComponent.toData()); - } catch (WorkerNotFoundException | WorkerInvokeException e) { - logger.error(e.getMessage(), e); - } - } - } - - public static class Factory extends AbstractLocalAsyncWorkerProvider { - @Override - public Role role() { - return WorkerRole.INSTANCE; - } - - @Override - public NodeComponentExchangeWorker workerInstance(ClusterWorkerContext clusterContext) { - return new NodeComponentExchangeWorker(role(), clusterContext); - } - - @Override - public int queueSize() { - return 1024; - } - } - - public enum WorkerRole implements Role { - INSTANCE; - - @Override - public String roleName() { - return NodeComponentExchangeWorker.class.getSimpleName(); - } - - @Override - public WorkerSelector workerSelector() { - return new HashCodeSelector(); - } - - @Override public DataDefine dataDefine() { - return new NodeComponentDataDefine(); - } - } -} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentPersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentPersistenceWorker.java index 2e7ccf0b9..5ff722b86 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentPersistenceWorker.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentPersistenceWorker.java @@ -1,7 +1,5 @@ package org.skywalking.apm.collector.agentstream.worker.node.component; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.agentstream.worker.node.component.dao.INodeComponentDAO; import org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentDataDefine; import org.skywalking.apm.collector.storage.dao.DAOContainer; @@ -10,7 +8,7 @@ import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; import org.skywalking.apm.collector.stream.worker.Role; import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; @@ -28,9 +26,12 @@ public class NodeComponentPersistenceWorker extends PersistenceWorker { super.preStart(); } - @Override protected List prepareBatch(Map dataMap) { - INodeComponentDAO dao = (INodeComponentDAO)DAOContainer.INSTANCE.get(INodeComponentDAO.class.getName()); - return dao.prepareBatch(dataMap); + @Override protected boolean needMergeDBData() { + return true; + } + + @Override protected IPersistenceDAO persistenceDAO() { + return (IPersistenceDAO)DAOContainer.INSTANCE.get(INodeComponentDAO.class.getName()); } public static class Factory extends AbstractLocalAsyncWorkerProvider { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java index 3bc4be924..4f4691802 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java @@ -3,85 +3,89 @@ package org.skywalking.apm.collector.agentstream.worker.node.component; import java.util.ArrayList; import java.util.List; import org.skywalking.apm.collector.agentstream.worker.Const; -import org.skywalking.apm.collector.agentstream.worker.cache.ComponentCache; import org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentDataDefine; import org.skywalking.apm.collector.agentstream.worker.segment.EntrySpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.ExitSpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.LocalSpanListener; +import org.skywalking.apm.collector.agentstream.worker.util.ExchangeMarkUtils; +import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils; import org.skywalking.apm.collector.core.framework.CollectorContextHelper; import org.skywalking.apm.collector.stream.StreamModuleContext; import org.skywalking.apm.collector.stream.StreamModuleGroupDefine; import org.skywalking.apm.collector.stream.worker.WorkerInvokeException; import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; import org.skywalking.apm.network.proto.SpanObject; -import org.skywalking.apm.network.trace.component.ComponentsDefine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author pengys5 */ -public class NodeComponentSpanListener implements EntrySpanListener, ExitSpanListener, LocalSpanListener { +public class NodeComponentSpanListener implements EntrySpanListener, ExitSpanListener, FirstSpanListener, LocalSpanListener { private final Logger logger = LoggerFactory.getLogger(NodeComponentSpanListener.class); - private List nodeComponents = new ArrayList<>(); + private List nodeComponents = new ArrayList<>(); + private long timeBucket; @Override public void parseExit(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { - String componentName = ComponentsDefine.getInstance().getComponentName(spanObject.getComponentId()); - createNodeComponent(spanObject, applicationId, componentName); + String componentName = ExchangeMarkUtils.INSTANCE.buildMarkedID(spanObject.getComponentId()); + if (spanObject.getComponentId() == 0) { + componentName = spanObject.getComponent(); + } + String peer = ExchangeMarkUtils.INSTANCE.buildMarkedID(spanObject.getPeerId()); + if (spanObject.getPeerId() == 0) { + peer = spanObject.getPeer(); + } + + String agg = componentName + Const.ID_SPLIT + peer; + nodeComponents.add(agg); } @Override public void parseEntry(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { - String componentName = ComponentsDefine.getInstance().getComponentName(spanObject.getComponentId()); - createNodeComponent(spanObject, applicationId, componentName); + buildEntryOrLocal(spanObject, applicationId); } @Override public void parseLocal(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { - int componentId = ComponentCache.get(applicationId, spanObject.getComponent()); - - NodeComponentDataDefine.NodeComponent nodeComponent = new NodeComponentDataDefine.NodeComponent(); - nodeComponent.setApplicationId(applicationId); - nodeComponent.setComponentId(componentId); - nodeComponent.setComponentName(spanObject.getComponent()); - - if (componentId == 0) { - StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); - - logger.debug("send to node component exchange worker, id: {}", nodeComponent.getId()); - nodeComponent.setId(applicationId + Const.ID_SPLIT + spanObject.getComponent()); - try { - context.getClusterWorkerContext().lookup(NodeComponentExchangeWorker.WorkerRole.INSTANCE).tell(nodeComponent.toData()); - } catch (WorkerInvokeException | WorkerNotFoundException e) { - logger.error(e.getMessage(), e); - } - } else { - nodeComponent.setId(applicationId + Const.ID_SPLIT + componentId); - nodeComponents.add(nodeComponent); - } + buildEntryOrLocal(spanObject, applicationId); } - private void createNodeComponent(SpanObject spanObject, int applicationId, String componentName) { - NodeComponentDataDefine.NodeComponent nodeComponent = new NodeComponentDataDefine.NodeComponent(); - nodeComponent.setApplicationId(applicationId); - nodeComponent.setComponentId(spanObject.getComponentId()); - nodeComponent.setComponentName(componentName); - nodeComponent.setId(applicationId + Const.ID_SPLIT + spanObject.getComponentId()); - nodeComponents.add(nodeComponent); + private void buildEntryOrLocal(SpanObject spanObject, int applicationId) { + String componentName = ExchangeMarkUtils.INSTANCE.buildMarkedID(spanObject.getComponentId()); + + if (spanObject.getComponentId() == 0) { + componentName = spanObject.getComponent(); + } + + String peer = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId); + String agg = componentName + Const.ID_SPLIT + peer; + nodeComponents.add(agg); + } + + @Override + public void parseFirst(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + timeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(spanObject.getStartTime()); } @Override public void build() { StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); - for (NodeComponentDataDefine.NodeComponent nodeComponent : nodeComponents) { + + nodeComponents.forEach(agg -> { + NodeComponentDataDefine.NodeComponent nodeComponent = new NodeComponentDataDefine.NodeComponent(); + nodeComponent.setId(timeBucket + Const.ID_SPLIT + agg); + nodeComponent.setAgg(agg); + nodeComponent.setTimeBucket(timeBucket); + try { logger.debug("send to node component aggregation worker, id: {}", nodeComponent.getId()); context.getClusterWorkerContext().lookup(NodeComponentAggregationWorker.WorkerRole.INSTANCE).tell(nodeComponent.toData()); } catch (WorkerInvokeException | WorkerNotFoundException e) { logger.error(e.getMessage(), e); } - } + }); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/INodeComponentDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/INodeComponentDAO.java index 3ae17bd74..79f44ccb5 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/INodeComponentDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/INodeComponentDAO.java @@ -1,14 +1,7 @@ package org.skywalking.apm.collector.agentstream.worker.node.component.dao; -import java.util.List; -import java.util.Map; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; - /** * @author pengys5 */ public interface INodeComponentDAO { - List prepareBatch(Map dataMap); - - int getComponentId(int applicationId, String componentName); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentEsDAO.java index 54e2e1442..a3e114f90 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentEsDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentEsDAO.java @@ -1,58 +1,47 @@ package org.skywalking.apm.collector.agentstream.worker.node.component.dao; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; +import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentTable; -import org.skywalking.apm.collector.client.elasticsearch.ElasticSearchClient; import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; /** * @author pengys5 */ -public class NodeComponentEsDAO extends EsDAO implements INodeComponentDAO { +public class NodeComponentEsDAO extends EsDAO implements INodeComponentDAO, IPersistenceDAO { - @Override public List prepareBatch(Map dataMap) { - List indexRequestBuilders = new ArrayList<>(); - dataMap.forEach((id, data) -> { - Map source = new HashMap(); - source.put(NodeComponentTable.COLUMN_APPLICATION_ID, data.getDataInteger(0)); - source.put(NodeComponentTable.COLUMN_COMPONENT_NAME, data.getDataString(1)); - source.put(NodeComponentTable.COLUMN_COMPONENT_ID, data.getDataInteger(1)); - - IndexRequestBuilder builder = getClient().prepareIndex(NodeComponentTable.TABLE, id).setSource(source); - indexRequestBuilders.add(builder); - }); - return indexRequestBuilders; + @Override public Data get(String id, DataDefine dataDefine) { + GetResponse getResponse = getClient().prepareGet(NodeComponentTable.TABLE, id).get(); + if (getResponse.isExists()) { + Data data = dataDefine.build(id); + Map source = getResponse.getSource(); + data.setDataString(1, (String)source.get(NodeComponentTable.COLUMN_AGG)); + data.setDataLong(0, (Long)source.get(NodeComponentTable.COLUMN_TIME_BUCKET)); + return data; + } else { + return null; + } } - public int getComponentId(int applicationId, String componentName) { - ElasticSearchClient client = getClient(); + @Override public IndexRequestBuilder prepareBatchInsert(Data data) { + Map source = new HashMap<>(); + source.put(NodeComponentTable.COLUMN_AGG, data.getDataString(1)); + source.put(NodeComponentTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); - SearchRequestBuilder searchRequestBuilder = client.prepareSearch(NodeComponentTable.TABLE); - searchRequestBuilder.setTypes("type"); - searchRequestBuilder.setSearchType(SearchType.QUERY_THEN_FETCH); - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.must(QueryBuilders.termQuery(NodeComponentTable.COLUMN_APPLICATION_ID, applicationId)); - boolQueryBuilder.must(QueryBuilders.termQuery(NodeComponentTable.COLUMN_COMPONENT_NAME, componentName)); - searchRequestBuilder.setQuery(boolQueryBuilder); - searchRequestBuilder.setSize(1); + return getClient().prepareIndex(NodeComponentTable.TABLE, data.getDataString(0)).setSource(source); + } - SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); - if (searchResponse.getHits().totalHits > 0) { - SearchHit searchHit = searchResponse.getHits().iterator().next(); - int componentId = (int)searchHit.getSource().get(NodeComponentTable.COLUMN_COMPONENT_ID); - return componentId; - } - return 0; + @Override public UpdateRequestBuilder prepareBatchUpdate(Data data) { + Map source = new HashMap<>(); + source.put(NodeComponentTable.COLUMN_AGG, data.getDataString(1)); + source.put(NodeComponentTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + + return getClient().prepareUpdate(NodeComponentTable.TABLE, data.getDataString(0)).setDoc(source); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentH2DAO.java index 3277e059d..5e08b3445 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentH2DAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentH2DAO.java @@ -1,7 +1,5 @@ package org.skywalking.apm.collector.agentstream.worker.node.component.dao; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.storage.h2.dao.H2DAO; /** @@ -9,11 +7,4 @@ import org.skywalking.apm.collector.storage.h2.dao.H2DAO; */ public class NodeComponentH2DAO extends H2DAO implements INodeComponentDAO { - @Override public List prepareBatch(Map map) { - return null; - } - - @Override public int getComponentId(int applicationId, String componentName) { - return 0; - } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java index 7e2342d22..0f08484ce 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java @@ -5,7 +5,6 @@ import org.skywalking.apm.collector.stream.worker.impl.data.Attribute; import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType; import org.skywalking.apm.collector.stream.worker.impl.data.Data; import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; -import org.skywalking.apm.collector.stream.worker.impl.data.Exchange; import org.skywalking.apm.collector.stream.worker.impl.data.Transform; import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation; import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation; @@ -20,15 +19,13 @@ public class NodeComponentDataDefine extends DataDefine { } @Override protected int initialCapacity() { - return 5; + return 3; } @Override protected void attributeDefine() { addAttribute(0, new Attribute(NodeComponentTable.COLUMN_ID, AttributeType.STRING, new NonOperation())); - addAttribute(1, new Attribute(NodeComponentTable.COLUMN_APPLICATION_ID, AttributeType.INTEGER, new CoverOperation())); - addAttribute(2, new Attribute(NodeComponentTable.COLUMN_COMPONENT_NAME, AttributeType.STRING, new CoverOperation())); - addAttribute(3, new Attribute(NodeComponentTable.COLUMN_COMPONENT_ID, AttributeType.INTEGER, new CoverOperation())); - addAttribute(4, new Attribute(NodeComponentTable.COLUMN_EXCHANGE_TIMES, AttributeType.INTEGER, new NonOperation())); + addAttribute(1, new Attribute(NodeComponentTable.COLUMN_AGG, AttributeType.STRING, new CoverOperation())); + addAttribute(2, new Attribute(NodeComponentTable.COLUMN_TIME_BUCKET, AttributeType.LONG, new CoverOperation())); } @Override public Object deserialize(RemoteData remoteData) { @@ -39,41 +36,33 @@ public class NodeComponentDataDefine extends DataDefine { return null; } - public static class NodeComponent extends Exchange implements Transform { + public static class NodeComponent implements Transform { private String id; - private int applicationId; - private String componentName; - private int componentId; + private String agg; + private long timeBucket; - public NodeComponent(String id, int applicationId, String componentName, int componentId) { - super(0); + NodeComponent(String id, String agg, long timeBucket) { this.id = id; - this.applicationId = applicationId; - this.componentName = componentName; - this.componentId = componentId; + this.agg = agg; + this.timeBucket = timeBucket; } public NodeComponent() { - super(0); } @Override public Data toData() { NodeComponentDataDefine define = new NodeComponentDataDefine(); Data data = define.build(id); data.setDataString(0, this.id); - data.setDataInteger(0, this.applicationId); - data.setDataString(1, this.componentName); - data.setDataInteger(1, this.componentId); - data.setDataInteger(2, this.getTimes()); + data.setDataString(1, this.agg); + data.setDataLong(0, this.timeBucket); return data; } @Override public NodeComponent toSelf(Data data) { this.id = data.getDataString(0); - this.applicationId = data.getDataInteger(0); - this.componentName = data.getDataString(1); - this.componentId = data.getDataInteger(1); - this.setTimes(data.getDataInteger(2)); + this.agg = data.getDataString(1); + this.timeBucket = data.getDataLong(0); return this; } @@ -81,32 +70,24 @@ public class NodeComponentDataDefine extends DataDefine { return id; } + public String getAgg() { + return agg; + } + + public long getTimeBucket() { + return timeBucket; + } + public void setId(String id) { this.id = id; } - public String getComponentName() { - return componentName; + public void setAgg(String agg) { + this.agg = agg; } - public void setComponentName(String componentName) { - this.componentName = componentName; - } - - public int getComponentId() { - return componentId; - } - - public void setComponentId(int componentId) { - this.componentId = componentId; - } - - public int getApplicationId() { - return applicationId; - } - - public void setApplicationId(int applicationId) { - this.applicationId = applicationId; + public void setTimeBucket(long timeBucket) { + this.timeBucket = timeBucket; } } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentEsTableDefine.java index 9085e2b83..e4e8e2809 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentEsTableDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentEsTableDefine.java @@ -25,8 +25,7 @@ public class NodeComponentEsTableDefine extends ElasticSearchTableDefine { } @Override public void initialize() { - addColumn(new ElasticSearchColumnDefine(NodeComponentTable.COLUMN_APPLICATION_ID, ElasticSearchColumnDefine.Type.Integer.name())); - addColumn(new ElasticSearchColumnDefine(NodeComponentTable.COLUMN_COMPONENT_NAME, ElasticSearchColumnDefine.Type.Keyword.name())); - addColumn(new ElasticSearchColumnDefine(NodeComponentTable.COLUMN_COMPONENT_ID, ElasticSearchColumnDefine.Type.Integer.name())); + addColumn(new ElasticSearchColumnDefine(NodeComponentTable.COLUMN_AGG, ElasticSearchColumnDefine.Type.Keyword.name())); + addColumn(new ElasticSearchColumnDefine(NodeComponentTable.COLUMN_TIME_BUCKET, ElasticSearchColumnDefine.Type.Long.name())); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentH2TableDefine.java index b85787ee7..62b77b04e 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentH2TableDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentH2TableDefine.java @@ -14,8 +14,7 @@ public class NodeComponentH2TableDefine extends H2TableDefine { @Override public void initialize() { addColumn(new H2ColumnDefine(NodeComponentTable.COLUMN_ID, H2ColumnDefine.Type.Varchar.name())); - addColumn(new H2ColumnDefine(NodeComponentTable.COLUMN_APPLICATION_ID, H2ColumnDefine.Type.Int.name())); - addColumn(new H2ColumnDefine(NodeComponentTable.COLUMN_COMPONENT_ID, H2ColumnDefine.Type.Int.name())); - addColumn(new H2ColumnDefine(NodeComponentTable.COLUMN_COMPONENT_NAME, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(NodeComponentTable.COLUMN_AGG, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(NodeComponentTable.COLUMN_TIME_BUCKET, H2ColumnDefine.Type.Bigint.name())); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentTable.java index e95365287..1bce8df83 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentTable.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentTable.java @@ -7,7 +7,4 @@ import org.skywalking.apm.collector.agentstream.worker.CommonTable; */ public class NodeComponentTable extends CommonTable { public static final String TABLE = "node_component"; - public static final String COLUMN_APPLICATION_ID = "application_id"; - public static final String COLUMN_COMPONENT_NAME = "component_name"; - public static final String COLUMN_COMPONENT_ID = "component_id"; } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingPersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingPersistenceWorker.java index 48a882d5a..57edf24c5 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingPersistenceWorker.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingPersistenceWorker.java @@ -1,7 +1,5 @@ package org.skywalking.apm.collector.agentstream.worker.node.mapping; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.agentstream.worker.node.mapping.dao.INodeMappingDAO; import org.skywalking.apm.collector.agentstream.worker.node.mapping.define.NodeMappingDataDefine; import org.skywalking.apm.collector.storage.dao.DAOContainer; @@ -10,7 +8,7 @@ import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; import org.skywalking.apm.collector.stream.worker.Role; import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; @@ -28,9 +26,12 @@ public class NodeMappingPersistenceWorker extends PersistenceWorker { super.preStart(); } - @Override protected List prepareBatch(Map dataMap) { - INodeMappingDAO dao = (INodeMappingDAO)DAOContainer.INSTANCE.get(INodeMappingDAO.class.getName()); - return dao.prepareBatch(dataMap); + @Override protected boolean needMergeDBData() { + return true; + } + + @Override protected IPersistenceDAO persistenceDAO() { + return (IPersistenceDAO)DAOContainer.INSTANCE.get(INodeMappingDAO.class.getName()); } public static class Factory extends AbstractLocalAsyncWorkerProvider { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingSpanListener.java index 4b8e4af71..2c5ebe600 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingSpanListener.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingSpanListener.java @@ -6,6 +6,7 @@ import org.skywalking.apm.collector.agentstream.worker.Const; import org.skywalking.apm.collector.agentstream.worker.node.mapping.define.NodeMappingDataDefine; import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.RefsListener; +import org.skywalking.apm.collector.agentstream.worker.util.ExchangeMarkUtils; import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils; import org.skywalking.apm.collector.core.framework.CollectorContextHelper; import org.skywalking.apm.collector.stream.StreamModuleContext; @@ -30,9 +31,9 @@ public class NodeMappingSpanListener implements RefsListener, FirstSpanListener @Override public void parseRef(TraceSegmentReference reference, int applicationId, int applicationInstanceId, String segmentId) { logger.debug("node mapping listener parse reference"); - String peers = Const.PEERS_FRONT_SPLIT + reference.getNetworkAddressId() + Const.PEERS_BEHIND_SPLIT; + String peers = reference.getNetworkAddress(); if (reference.getNetworkAddressId() == 0) { - peers = Const.PEERS_FRONT_SPLIT + reference.getNetworkAddress() + Const.PEERS_BEHIND_SPLIT; + peers = ExchangeMarkUtils.INSTANCE.buildMarkedID(reference.getNetworkAddressId()); } String agg = applicationId + Const.ID_SPLIT + peers; diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/INodeMappingDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/INodeMappingDAO.java index f864617a2..450cc5451 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/INodeMappingDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/INodeMappingDAO.java @@ -1,12 +1,7 @@ package org.skywalking.apm.collector.agentstream.worker.node.mapping.dao; -import java.util.List; -import java.util.Map; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; - /** * @author pengys5 */ public interface INodeMappingDAO { - List prepareBatch(Map dataMap); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/NodeMappingEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/NodeMappingEsDAO.java index 4a36d0378..66a22211d 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/NodeMappingEsDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/NodeMappingEsDAO.java @@ -1,29 +1,47 @@ package org.skywalking.apm.collector.agentstream.worker.node.mapping.dao; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; +import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.skywalking.apm.collector.agentstream.worker.node.mapping.define.NodeMappingTable; import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; /** * @author pengys5 */ -public class NodeMappingEsDAO extends EsDAO implements INodeMappingDAO { +public class NodeMappingEsDAO extends EsDAO implements INodeMappingDAO, IPersistenceDAO { - @Override public List prepareBatch(Map dataMap) { - List indexRequestBuilders = new ArrayList<>(); - dataMap.forEach((id, data) -> { - Map source = new HashMap(); - source.put(NodeMappingTable.COLUMN_AGG, data.getDataString(1)); - source.put(NodeMappingTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + @Override public Data get(String id, DataDefine dataDefine) { + GetResponse getResponse = getClient().prepareGet(NodeMappingTable.TABLE, id).get(); + if (getResponse.isExists()) { + Data data = dataDefine.build(id); + Map source = getResponse.getSource(); + data.setDataString(1, (String)source.get(NodeMappingTable.COLUMN_AGG)); + data.setDataLong(0, (Long)source.get(NodeMappingTable.COLUMN_TIME_BUCKET)); + return data; + } else { + return null; + } + } - IndexRequestBuilder builder = getClient().prepareIndex(NodeMappingTable.TABLE, id).setSource(source); - indexRequestBuilders.add(builder); - }); - return indexRequestBuilders; + @Override public IndexRequestBuilder prepareBatchInsert(Data data) { + Map source = new HashMap<>(); + source.put(NodeMappingTable.COLUMN_AGG, data.getDataString(1)); + source.put(NodeMappingTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + + return getClient().prepareIndex(NodeMappingTable.TABLE, data.getDataString(0)).setSource(source); + } + + @Override public UpdateRequestBuilder prepareBatchUpdate(Data data) { + Map source = new HashMap<>(); + source.put(NodeMappingTable.COLUMN_AGG, data.getDataString(1)); + source.put(NodeMappingTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + + return getClient().prepareUpdate(NodeMappingTable.TABLE, data.getDataString(0)).setDoc(source); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/NodeMappingH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/NodeMappingH2DAO.java index 8ad24c091..045d0738e 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/NodeMappingH2DAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/NodeMappingH2DAO.java @@ -1,15 +1,9 @@ package org.skywalking.apm.collector.agentstream.worker.node.mapping.dao; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.storage.h2.dao.H2DAO; /** * @author pengys5 */ public class NodeMappingH2DAO extends H2DAO implements INodeMappingDAO { - - @Override public List prepareBatch(Map map) { - return null; - } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingDataDefine.java index 4c949355b..e227b291e 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingDataDefine.java @@ -36,12 +36,12 @@ public class NodeMappingDataDefine extends DataDefine { return null; } - public static class NodeMapping implements Transform { + public static class NodeMapping implements Transform { private String id; private String agg; private long timeBucket; - public NodeMapping(String id, String agg, long timeBucket) { + NodeMapping(String id, String agg, long timeBucket) { this.id = id; this.agg = agg; this.timeBucket = timeBucket; @@ -59,8 +59,11 @@ public class NodeMappingDataDefine extends DataDefine { return data; } - @Override public Object toSelf(Data data) { - return null; + @Override public NodeMapping toSelf(Data data) { + this.id = data.getDataString(0); + this.agg = data.getDataString(1); + this.timeBucket = data.getDataLong(0); + return this; } public String getId() { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefPersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefPersistenceWorker.java index 06e363a1a..ec48c3a8c 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefPersistenceWorker.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefPersistenceWorker.java @@ -1,7 +1,5 @@ package org.skywalking.apm.collector.agentstream.worker.noderef.reference; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.agentstream.worker.noderef.reference.dao.INodeReferenceDAO; import org.skywalking.apm.collector.agentstream.worker.noderef.reference.define.NodeRefDataDefine; import org.skywalking.apm.collector.storage.dao.DAOContainer; @@ -10,7 +8,7 @@ import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; import org.skywalking.apm.collector.stream.worker.Role; import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; @@ -28,9 +26,12 @@ public class NodeRefPersistenceWorker extends PersistenceWorker { super.preStart(); } - @Override protected List prepareBatch(Map dataMap) { - INodeReferenceDAO dao = (INodeReferenceDAO)DAOContainer.INSTANCE.get(INodeReferenceDAO.class.getName()); - return dao.prepareBatch(dataMap); + @Override protected boolean needMergeDBData() { + return true; + } + + @Override protected IPersistenceDAO persistenceDAO() { + return (IPersistenceDAO)DAOContainer.INSTANCE.get(INodeReferenceDAO.class.getName()); } public static class Factory extends AbstractLocalAsyncWorkerProvider { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefSpanListener.java index 0e5da0430..a57e678b1 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefSpanListener.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefSpanListener.java @@ -8,6 +8,7 @@ import org.skywalking.apm.collector.agentstream.worker.segment.EntrySpanListener import org.skywalking.apm.collector.agentstream.worker.segment.ExitSpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.RefsListener; +import org.skywalking.apm.collector.agentstream.worker.util.ExchangeMarkUtils; import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils; import org.skywalking.apm.collector.core.framework.CollectorContextHelper; import org.skywalking.apm.collector.stream.StreamModuleContext; @@ -34,9 +35,9 @@ public class NodeRefSpanListener implements EntrySpanListener, ExitSpanListener, @Override public void parseExit(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { String front = String.valueOf(applicationId); - String behind = String.valueOf(spanObject.getPeerId()); + String behind = spanObject.getPeer(); if (spanObject.getPeerId() == 0) { - behind = spanObject.getPeer(); + behind = ExchangeMarkUtils.INSTANCE.buildMarkedID(spanObject.getPeerId()); } String agg = front + Const.ID_SPLIT + behind; @@ -45,9 +46,8 @@ public class NodeRefSpanListener implements EntrySpanListener, ExitSpanListener, @Override public void parseEntry(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { - String behind = String.valueOf(applicationId); + String behind = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId); String front = Const.USER_CODE; - String agg = front + Const.ID_SPLIT + behind; nodeEntryReferences.add(agg); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/INodeReferenceDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/INodeReferenceDAO.java index 122829dfb..7f2406d1e 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/INodeReferenceDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/INodeReferenceDAO.java @@ -1,12 +1,7 @@ package org.skywalking.apm.collector.agentstream.worker.noderef.reference.dao; -import java.util.List; -import java.util.Map; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; - /** * @author pengys5 */ public interface INodeReferenceDAO { - List prepareBatch(Map dataMap); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/NodeReferenceEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/NodeReferenceEsDAO.java index d1334bec6..84d572a70 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/NodeReferenceEsDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/NodeReferenceEsDAO.java @@ -1,29 +1,47 @@ package org.skywalking.apm.collector.agentstream.worker.noderef.reference.dao; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; +import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.skywalking.apm.collector.agentstream.worker.noderef.reference.define.NodeRefTable; import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; /** * @author pengys5 */ -public class NodeReferenceEsDAO extends EsDAO implements INodeReferenceDAO { +public class NodeReferenceEsDAO extends EsDAO implements INodeReferenceDAO, IPersistenceDAO { - @Override public List prepareBatch(Map dataMap) { - List indexRequestBuilders = new ArrayList<>(); - dataMap.forEach((id, data) -> { - Map source = new HashMap(); - source.put(NodeRefTable.COLUMN_AGG, data.getDataString(1)); - source.put(NodeRefTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + @Override public Data get(String id, DataDefine dataDefine) { + GetResponse getResponse = getClient().prepareGet(NodeRefTable.TABLE, id).get(); + if (getResponse.isExists()) { + Data data = dataDefine.build(id); + Map source = getResponse.getSource(); + data.setDataString(1, (String)source.get(NodeRefTable.COLUMN_AGG)); + data.setDataLong(0, (Long)source.get(NodeRefTable.COLUMN_TIME_BUCKET)); + return data; + } else { + return null; + } + } - IndexRequestBuilder builder = getClient().prepareIndex(NodeRefTable.TABLE, id).setSource(source); - indexRequestBuilders.add(builder); - }); - return indexRequestBuilders; + @Override public IndexRequestBuilder prepareBatchInsert(Data data) { + Map source = new HashMap<>(); + source.put(NodeRefTable.COLUMN_AGG, data.getDataString(1)); + source.put(NodeRefTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + + return getClient().prepareIndex(NodeRefTable.TABLE, data.getDataString(0)).setSource(source); + } + + @Override public UpdateRequestBuilder prepareBatchUpdate(Data data) { + Map source = new HashMap<>(); + source.put(NodeRefTable.COLUMN_AGG, data.getDataString(1)); + source.put(NodeRefTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + + return getClient().prepareUpdate(NodeRefTable.TABLE, data.getDataString(0)).setDoc(source); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/NodeReferenceH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/NodeReferenceH2DAO.java index bc2d7a0a1..7820be366 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/NodeReferenceH2DAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/NodeReferenceH2DAO.java @@ -1,15 +1,24 @@ package org.skywalking.apm.collector.agentstream.worker.noderef.reference.dao; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.storage.h2.dao.H2DAO; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; /** * @author pengys5 */ -public class NodeReferenceH2DAO extends H2DAO implements INodeReferenceDAO { +public class NodeReferenceH2DAO extends H2DAO implements INodeReferenceDAO, IPersistenceDAO { - @Override public List prepareBatch(Map map) { + @Override public Data get(String id, DataDefine dataDefine) { + return null; + } + + @Override public String prepareBatchInsert(Data data) { + return null; + } + + @Override public String prepareBatchUpdate(Data data) { return null; } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefDataDefine.java index 10c2c969f..079062861 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefDataDefine.java @@ -14,10 +14,8 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class NodeRefDataDefine extends DataDefine { - public static final int DEFINE_ID = 201; - @Override public int defineId() { - return DEFINE_ID; + return 201; } @Override protected int initialCapacity() { @@ -51,7 +49,7 @@ public class NodeRefDataDefine extends DataDefine { private String agg; private long timeBucket; - public NodeReference(String id, String agg, long timeBucket) { + NodeReference(String id, String agg, long timeBucket) { this.id = id; this.agg = agg; this.timeBucket = timeBucket; @@ -70,7 +68,10 @@ public class NodeRefDataDefine extends DataDefine { } @Override public Object toSelf(Data data) { - return null; + this.id = data.getDataString(0); + this.agg = data.getDataString(1); + this.timeBucket = data.getDataLong(0); + return this; } public String getId() { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumPersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumPersistenceWorker.java index 96dead7e0..6fcb36eb7 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumPersistenceWorker.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumPersistenceWorker.java @@ -1,7 +1,5 @@ package org.skywalking.apm.collector.agentstream.worker.noderef.summary; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.agentstream.worker.noderef.summary.dao.INodeRefSumDAO; import org.skywalking.apm.collector.agentstream.worker.noderef.summary.define.NodeRefSumDataDefine; import org.skywalking.apm.collector.storage.dao.DAOContainer; @@ -10,7 +8,7 @@ import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; import org.skywalking.apm.collector.stream.worker.Role; import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; @@ -28,9 +26,12 @@ public class NodeRefSumPersistenceWorker extends PersistenceWorker { super.preStart(); } - @Override protected List prepareBatch(Map dataMap) { - INodeRefSumDAO dao = (INodeRefSumDAO)DAOContainer.INSTANCE.get(INodeRefSumDAO.class.getName()); - return dao.prepareBatch(dataMap); + @Override protected boolean needMergeDBData() { + return true; + } + + @Override protected IPersistenceDAO persistenceDAO() { + return (IPersistenceDAO)DAOContainer.INSTANCE.get(INodeRefSumDAO.class.getName()); } public static class Factory extends AbstractLocalAsyncWorkerProvider { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumSpanListener.java index 69bffe8ea..f3251d032 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumSpanListener.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumSpanListener.java @@ -8,6 +8,7 @@ import org.skywalking.apm.collector.agentstream.worker.segment.EntrySpanListener import org.skywalking.apm.collector.agentstream.worker.segment.ExitSpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.RefsListener; +import org.skywalking.apm.collector.agentstream.worker.util.ExchangeMarkUtils; import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils; import org.skywalking.apm.collector.core.framework.CollectorContextHelper; import org.skywalking.apm.collector.stream.StreamModuleContext; @@ -34,9 +35,9 @@ public class NodeRefSumSpanListener implements EntrySpanListener, ExitSpanListen @Override public void parseExit(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { String front = String.valueOf(applicationId); - String behind = String.valueOf(spanObject.getPeerId()); + String behind = spanObject.getPeer(); if (spanObject.getPeerId() == 0) { - behind = spanObject.getPeer(); + behind = ExchangeMarkUtils.INSTANCE.buildMarkedID(spanObject.getPeerId()); } String agg = front + Const.ID_SPLIT + behind; @@ -45,7 +46,7 @@ public class NodeRefSumSpanListener implements EntrySpanListener, ExitSpanListen @Override public void parseEntry(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { - String behind = String.valueOf(applicationId); + String behind = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId); String front = Const.USER_CODE; String agg = front + Const.ID_SPLIT + behind; diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/INodeRefSumDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/INodeRefSumDAO.java index ef4f69737..6be979957 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/INodeRefSumDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/INodeRefSumDAO.java @@ -1,12 +1,7 @@ package org.skywalking.apm.collector.agentstream.worker.noderef.summary.dao; -import java.util.List; -import java.util.Map; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; - /** * @author pengys5 */ public interface INodeRefSumDAO { - List prepareBatch(Map dataMap); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumEsDAO.java index a20099784..c117e70a9 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumEsDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumEsDAO.java @@ -1,35 +1,66 @@ package org.skywalking.apm.collector.agentstream.worker.noderef.summary.dao; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; +import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.update.UpdateRequestBuilder; +import org.skywalking.apm.collector.agentstream.worker.noderef.reference.define.NodeRefTable; import org.skywalking.apm.collector.agentstream.worker.noderef.summary.define.NodeRefSumTable; import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; /** * @author pengys5 */ -public class NodeRefSumEsDAO extends EsDAO implements INodeRefSumDAO { +public class NodeRefSumEsDAO extends EsDAO implements INodeRefSumDAO, IPersistenceDAO { - @Override public List prepareBatch(Map dataMap) { - List indexRequestBuilders = new ArrayList<>(); - dataMap.forEach((id, data) -> { - Map source = new HashMap(); - source.put(NodeRefSumTable.COLUMN_ONE_SECOND_LESS, data.getDataLong(0)); - source.put(NodeRefSumTable.COLUMN_THREE_SECOND_LESS, data.getDataLong(1)); - source.put(NodeRefSumTable.COLUMN_FIVE_SECOND_LESS, data.getDataLong(2)); - source.put(NodeRefSumTable.COLUMN_FIVE_SECOND_GREATER, data.getDataLong(3)); - source.put(NodeRefSumTable.COLUMN_ERROR, data.getDataLong(4)); - source.put(NodeRefSumTable.COLUMN_SUMMARY, data.getDataLong(5)); - source.put(NodeRefSumTable.COLUMN_AGG, data.getDataString(1)); - source.put(NodeRefSumTable.COLUMN_TIME_BUCKET, data.getDataLong(6)); + @Override public Data get(String id, DataDefine dataDefine) { + GetResponse getResponse = getClient().prepareGet(NodeRefSumTable.TABLE, id).get(); + if (getResponse.isExists()) { + Data data = dataDefine.build(id); + Map source = getResponse.getSource(); + data.setDataLong(0, (Long)source.get(NodeRefSumTable.COLUMN_ONE_SECOND_LESS)); + data.setDataLong(1, (Long)source.get(NodeRefSumTable.COLUMN_THREE_SECOND_LESS)); + data.setDataLong(2, (Long)source.get(NodeRefSumTable.COLUMN_FIVE_SECOND_LESS)); + data.setDataLong(3, (Long)source.get(NodeRefSumTable.COLUMN_FIVE_SECOND_GREATER)); + data.setDataLong(4, (Long)source.get(NodeRefSumTable.COLUMN_ERROR)); + data.setDataLong(5, (Long)source.get(NodeRefSumTable.COLUMN_SUMMARY)); + data.setDataLong(6, (Long)source.get(NodeRefSumTable.COLUMN_TIME_BUCKET)); + data.setDataString(1, (String)source.get(NodeRefSumTable.COLUMN_AGG)); + return data; + } else { + return null; + } + } - IndexRequestBuilder builder = getClient().prepareIndex(NodeRefSumTable.TABLE, id).setSource(source); - indexRequestBuilders.add(builder); - }); - return indexRequestBuilders; + @Override public IndexRequestBuilder prepareBatchInsert(Data data) { + Map source = new HashMap<>(); + source.put(NodeRefSumTable.COLUMN_ONE_SECOND_LESS, data.getDataLong(0)); + source.put(NodeRefSumTable.COLUMN_THREE_SECOND_LESS, data.getDataLong(1)); + source.put(NodeRefSumTable.COLUMN_FIVE_SECOND_LESS, data.getDataLong(2)); + source.put(NodeRefSumTable.COLUMN_FIVE_SECOND_GREATER, data.getDataLong(3)); + source.put(NodeRefSumTable.COLUMN_ERROR, data.getDataLong(4)); + source.put(NodeRefSumTable.COLUMN_SUMMARY, data.getDataLong(5)); + source.put(NodeRefSumTable.COLUMN_AGG, data.getDataString(1)); + source.put(NodeRefSumTable.COLUMN_TIME_BUCKET, data.getDataLong(6)); + + return getClient().prepareIndex(NodeRefSumTable.TABLE, data.getDataString(0)).setSource(source); + } + + @Override public UpdateRequestBuilder prepareBatchUpdate(Data data) { + Map source = new HashMap<>(); + source.put(NodeRefSumTable.COLUMN_ONE_SECOND_LESS, data.getDataLong(0)); + source.put(NodeRefSumTable.COLUMN_THREE_SECOND_LESS, data.getDataLong(1)); + source.put(NodeRefSumTable.COLUMN_FIVE_SECOND_LESS, data.getDataLong(2)); + source.put(NodeRefSumTable.COLUMN_FIVE_SECOND_GREATER, data.getDataLong(3)); + source.put(NodeRefSumTable.COLUMN_ERROR, data.getDataLong(4)); + source.put(NodeRefSumTable.COLUMN_SUMMARY, data.getDataLong(5)); + source.put(NodeRefSumTable.COLUMN_AGG, data.getDataString(1)); + source.put(NodeRefSumTable.COLUMN_TIME_BUCKET, data.getDataLong(6)); + + return getClient().prepareUpdate(NodeRefTable.TABLE, data.getDataString(0)).setDoc(source); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumH2DAO.java index 2a3b83501..7483c8a82 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumH2DAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumH2DAO.java @@ -1,15 +1,9 @@ package org.skywalking.apm.collector.agentstream.worker.noderef.summary.dao; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.storage.h2.dao.H2DAO; /** * @author pengys5 */ public class NodeRefSumH2DAO extends H2DAO implements INodeRefSumDAO { - - @Override public List prepareBatch(Map map) { - return null; - } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java index 3d3738cd2..6558a2615 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java @@ -38,7 +38,6 @@ public class SegmentParse { spanListeners.add(new NodeRefSpanListener()); spanListeners.add(new NodeComponentSpanListener()); spanListeners.add(new NodeMappingSpanListener()); - spanListeners.add(new NodeRefSpanListener()); spanListeners.add(new NodeRefSumSpanListener()); spanListeners.add(new SegmentCostSpanListener()); spanListeners.add(new GlobalTraceSpanListener()); diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/SegmentCostPersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/SegmentCostPersistenceWorker.java index d6b72f8a2..036a291fa 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/SegmentCostPersistenceWorker.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/SegmentCostPersistenceWorker.java @@ -1,7 +1,5 @@ package org.skywalking.apm.collector.agentstream.worker.segment.cost; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.agentstream.worker.segment.cost.dao.ISegmentCostDAO; import org.skywalking.apm.collector.agentstream.worker.segment.cost.define.SegmentCostDataDefine; import org.skywalking.apm.collector.storage.dao.DAOContainer; @@ -10,7 +8,7 @@ import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; import org.skywalking.apm.collector.stream.worker.Role; import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; import org.skywalking.apm.collector.stream.worker.selector.RollingSelector; import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; @@ -28,9 +26,12 @@ public class SegmentCostPersistenceWorker extends PersistenceWorker { super.preStart(); } - @Override protected List prepareBatch(Map dataMap) { - ISegmentCostDAO dao = (ISegmentCostDAO)DAOContainer.INSTANCE.get(ISegmentCostDAO.class.getName()); - return dao.prepareBatch(dataMap); + @Override protected boolean needMergeDBData() { + return false; + } + + @Override protected IPersistenceDAO persistenceDAO() { + return (IPersistenceDAO)DAOContainer.INSTANCE.get(ISegmentCostDAO.class.getName()); } public static class Factory extends AbstractLocalAsyncWorkerProvider { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/SegmentCostSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/SegmentCostSpanListener.java index e5934792f..c49e86183 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/SegmentCostSpanListener.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/SegmentCostSpanListener.java @@ -7,6 +7,7 @@ import org.skywalking.apm.collector.agentstream.worker.segment.ExitSpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.LocalSpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.cost.define.SegmentCostDataDefine; +import org.skywalking.apm.collector.agentstream.worker.util.ExchangeMarkUtils; import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils; import org.skywalking.apm.collector.core.framework.CollectorContextHelper; import org.skywalking.apm.collector.stream.StreamModuleContext; @@ -40,10 +41,14 @@ public class SegmentCostSpanListener implements EntrySpanListener, ExitSpanListe segmentCost.setCost(spanObject.getEndTime() - spanObject.getStartTime()); segmentCost.setStartTime(spanObject.getStartTime()); segmentCost.setEndTime(spanObject.getEndTime()); - segmentCost.setOperationName(spanObject.getOperationName()); segmentCost.setId(segmentId); - segmentCosts.add(segmentCost); + if (spanObject.getOperationNameId() == 0) { + segmentCost.setServiceName(spanObject.getOperationName()); + } else { + segmentCost.setServiceName(ExchangeMarkUtils.INSTANCE.buildMarkedID(spanObject.getOperationNameId())); + } + segmentCosts.add(segmentCost); isError = isError || spanObject.getIsError(); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/ISegmentCostDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/ISegmentCostDAO.java index 03787ca00..03a78dd9a 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/ISegmentCostDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/ISegmentCostDAO.java @@ -1,12 +1,7 @@ package org.skywalking.apm.collector.agentstream.worker.segment.cost.dao; -import java.util.List; -import java.util.Map; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; - /** * @author pengys5 */ public interface ISegmentCostDAO { - List prepareBatch(Map dataMap); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/SegmentCostEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/SegmentCostEsDAO.java index d1b9e19b2..794b0d32a 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/SegmentCostEsDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/SegmentCostEsDAO.java @@ -1,39 +1,43 @@ package org.skywalking.apm.collector.agentstream.worker.segment.cost.dao; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.skywalking.apm.collector.agentstream.worker.segment.cost.define.SegmentCostTable; import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author pengys5 */ -public class SegmentCostEsDAO extends EsDAO implements ISegmentCostDAO { +public class SegmentCostEsDAO extends EsDAO implements ISegmentCostDAO, IPersistenceDAO { private final Logger logger = LoggerFactory.getLogger(SegmentCostEsDAO.class); - @Override public List prepareBatch(Map dataMap) { - List indexRequestBuilders = new ArrayList<>(); - dataMap.forEach((id, data) -> { - logger.debug("segment cost prepareBatch, id: {}", id); - Map source = new HashMap(); - source.put(SegmentCostTable.COLUMN_SEGMENT_ID, data.getDataString(1)); - source.put(SegmentCostTable.COLUMN_OPERATION_NAME, data.getDataString(2)); - source.put(SegmentCostTable.COLUMN_COST, data.getDataLong(0)); - source.put(SegmentCostTable.COLUMN_START_TIME, data.getDataLong(1)); - source.put(SegmentCostTable.COLUMN_END_TIME, data.getDataLong(2)); - source.put(SegmentCostTable.COLUMN_IS_ERROR, data.getDataBoolean(0)); - source.put(SegmentCostTable.COLUMN_TIME_BUCKET, data.getDataLong(3)); - logger.debug("segment cost source: {}", source.toString()); - IndexRequestBuilder builder = getClient().prepareIndex(SegmentCostTable.TABLE, id).setSource(source); - indexRequestBuilders.add(builder); - }); - return indexRequestBuilders; + @Override public Data get(String id, DataDefine dataDefine) { + return null; + } + + @Override public UpdateRequestBuilder prepareBatchUpdate(Data data) { + return null; + } + + @Override public IndexRequestBuilder prepareBatchInsert(Data data) { + logger.debug("segment cost prepareBatchInsert, id: {}", data.getDataString(0)); + Map source = new HashMap<>(); + source.put(SegmentCostTable.COLUMN_SEGMENT_ID, data.getDataString(1)); + source.put(SegmentCostTable.COLUMN_SERVICE_NAME, data.getDataString(2)); + source.put(SegmentCostTable.COLUMN_COST, data.getDataLong(0)); + source.put(SegmentCostTable.COLUMN_START_TIME, data.getDataLong(1)); + source.put(SegmentCostTable.COLUMN_END_TIME, data.getDataLong(2)); + source.put(SegmentCostTable.COLUMN_IS_ERROR, data.getDataBoolean(0)); + source.put(SegmentCostTable.COLUMN_TIME_BUCKET, data.getDataLong(3)); + logger.debug("segment cost source: {}", source.toString()); + return getClient().prepareIndex(SegmentCostTable.TABLE, data.getDataString(0)).setSource(source); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/SegmentCostH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/SegmentCostH2DAO.java index 90fd044b9..63a6c78c5 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/SegmentCostH2DAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/SegmentCostH2DAO.java @@ -1,7 +1,5 @@ package org.skywalking.apm.collector.agentstream.worker.segment.cost.dao; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.storage.h2.dao.H2DAO; /** @@ -9,7 +7,4 @@ import org.skywalking.apm.collector.storage.h2.dao.H2DAO; */ public class SegmentCostH2DAO extends H2DAO implements ISegmentCostDAO { - @Override public List prepareBatch(Map map) { - return null; - } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostDataDefine.java index c86de4fcd..51f57fd43 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostDataDefine.java @@ -25,7 +25,7 @@ public class SegmentCostDataDefine extends DataDefine { @Override protected void attributeDefine() { addAttribute(0, new Attribute(SegmentCostTable.COLUMN_ID, AttributeType.STRING, new NonOperation())); addAttribute(1, new Attribute(SegmentCostTable.COLUMN_SEGMENT_ID, AttributeType.STRING, new CoverOperation())); - addAttribute(2, new Attribute(SegmentCostTable.COLUMN_OPERATION_NAME, AttributeType.STRING, new CoverOperation())); + addAttribute(2, new Attribute(SegmentCostTable.COLUMN_SERVICE_NAME, AttributeType.STRING, new CoverOperation())); addAttribute(3, new Attribute(SegmentCostTable.COLUMN_COST, AttributeType.LONG, new CoverOperation())); addAttribute(4, new Attribute(SegmentCostTable.COLUMN_START_TIME, AttributeType.LONG, new CoverOperation())); addAttribute(5, new Attribute(SegmentCostTable.COLUMN_END_TIME, AttributeType.LONG, new CoverOperation())); @@ -36,13 +36,13 @@ public class SegmentCostDataDefine extends DataDefine { @Override public Object deserialize(RemoteData remoteData) { String id = remoteData.getDataStrings(0); String segmentId = remoteData.getDataStrings(1); - String operationName = remoteData.getDataStrings(2); + String serviceName = remoteData.getDataStrings(2); Long cost = remoteData.getDataLongs(0); Long startTime = remoteData.getDataLongs(1); Long endTime = remoteData.getDataLongs(2); Boolean isError = remoteData.getDataBooleans(0); Long timeBucket = remoteData.getDataLongs(2); - return new SegmentCost(id, segmentId, operationName, cost, startTime, endTime, isError, timeBucket); + return new SegmentCost(id, segmentId, serviceName, cost, startTime, endTime, isError, timeBucket); } @Override public RemoteData serialize(Object object) { @@ -50,7 +50,7 @@ public class SegmentCostDataDefine extends DataDefine { RemoteData.Builder builder = RemoteData.newBuilder(); builder.addDataStrings(segmentCost.getId()); builder.addDataStrings(segmentCost.getSegmentId()); - builder.addDataStrings(segmentCost.getOperationName()); + builder.addDataStrings(segmentCost.getServiceName()); builder.addDataLongs(segmentCost.getCost()); builder.addDataLongs(segmentCost.getStartTime()); builder.addDataLongs(segmentCost.getEndTime()); @@ -62,18 +62,18 @@ public class SegmentCostDataDefine extends DataDefine { public static class SegmentCost implements Transform { private String id; private String segmentId; - private String operationName; + private String serviceName; private Long cost; private Long startTime; private Long endTime; private boolean isError; private long timeBucket; - SegmentCost(String id, String segmentId, String operationName, Long cost, + SegmentCost(String id, String segmentId, String serviceName, Long cost, Long startTime, Long endTime, boolean isError, long timeBucket) { this.id = id; this.segmentId = segmentId; - this.operationName = operationName; + this.serviceName = serviceName; this.cost = cost; this.startTime = startTime; this.endTime = endTime; @@ -89,7 +89,7 @@ public class SegmentCostDataDefine extends DataDefine { Data data = define.build(id); data.setDataString(0, this.id); data.setDataString(1, this.segmentId); - data.setDataString(2, this.operationName); + data.setDataString(2, this.serviceName); data.setDataLong(0, this.cost); data.setDataLong(1, this.startTime); data.setDataLong(2, this.endTime); @@ -118,12 +118,12 @@ public class SegmentCostDataDefine extends DataDefine { this.segmentId = segmentId; } - public String getOperationName() { - return operationName; + public String getServiceName() { + return serviceName; } - public void setOperationName(String operationName) { - this.operationName = operationName; + public void setServiceName(String serviceName) { + this.serviceName = serviceName; } public Long getCost() { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostEsTableDefine.java index 4f493da99..e44e40829 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostEsTableDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostEsTableDefine.java @@ -26,7 +26,7 @@ public class SegmentCostEsTableDefine extends ElasticSearchTableDefine { @Override public void initialize() { addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_SEGMENT_ID, ElasticSearchColumnDefine.Type.Keyword.name())); - addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_OPERATION_NAME, ElasticSearchColumnDefine.Type.Keyword.name())); + addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_SERVICE_NAME, ElasticSearchColumnDefine.Type.Keyword.name())); addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_COST, ElasticSearchColumnDefine.Type.Long.name())); addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_START_TIME, ElasticSearchColumnDefine.Type.Long.name())); addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_END_TIME, ElasticSearchColumnDefine.Type.Long.name())); diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostH2TableDefine.java index c4ca90226..f1d896919 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostH2TableDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostH2TableDefine.java @@ -15,7 +15,7 @@ public class SegmentCostH2TableDefine extends H2TableDefine { @Override public void initialize() { addColumn(new H2ColumnDefine(SegmentCostTable.COLUMN_ID, H2ColumnDefine.Type.Varchar.name())); addColumn(new H2ColumnDefine(SegmentCostTable.COLUMN_SEGMENT_ID, H2ColumnDefine.Type.Varchar.name())); - addColumn(new H2ColumnDefine(SegmentCostTable.COLUMN_OPERATION_NAME, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(SegmentCostTable.COLUMN_SERVICE_NAME, H2ColumnDefine.Type.Varchar.name())); addColumn(new H2ColumnDefine(SegmentCostTable.COLUMN_COST, H2ColumnDefine.Type.Bigint.name())); addColumn(new H2ColumnDefine(SegmentCostTable.COLUMN_START_TIME, H2ColumnDefine.Type.Bigint.name())); addColumn(new H2ColumnDefine(SegmentCostTable.COLUMN_END_TIME, H2ColumnDefine.Type.Bigint.name())); diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostTable.java index 0bc0ee947..fb9c7f83d 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostTable.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostTable.java @@ -10,7 +10,7 @@ public class SegmentCostTable extends CommonTable { public static final String COLUMN_SEGMENT_ID = "segment_id"; public static final String COLUMN_START_TIME = "start_time"; public static final String COLUMN_END_TIME = "end_time"; - public static final String COLUMN_OPERATION_NAME = "operation_name"; + public static final String COLUMN_SERVICE_NAME = "service_name"; public static final String COLUMN_COST = "cost"; public static final String COLUMN_IS_ERROR = "is_error"; } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/SegmentPersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/SegmentPersistenceWorker.java index a76fbd11d..03bbf0351 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/SegmentPersistenceWorker.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/SegmentPersistenceWorker.java @@ -1,7 +1,5 @@ package org.skywalking.apm.collector.agentstream.worker.segment.origin; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.agentstream.worker.segment.origin.dao.ISegmentDAO; import org.skywalking.apm.collector.agentstream.worker.segment.origin.define.SegmentDataDefine; import org.skywalking.apm.collector.storage.dao.DAOContainer; @@ -10,7 +8,7 @@ import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; import org.skywalking.apm.collector.stream.worker.Role; import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; import org.skywalking.apm.collector.stream.worker.selector.RollingSelector; import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; @@ -28,9 +26,12 @@ public class SegmentPersistenceWorker extends PersistenceWorker { super.preStart(); } - @Override protected List prepareBatch(Map dataMap) { - ISegmentDAO dao = (ISegmentDAO)DAOContainer.INSTANCE.get(ISegmentDAO.class.getName()); - return dao.prepareBatch(dataMap); + @Override protected boolean needMergeDBData() { + return false; + } + + @Override protected IPersistenceDAO persistenceDAO() { + return (IPersistenceDAO)DAOContainer.INSTANCE.get(ISegmentDAO.class.getName()); } public static class Factory extends AbstractLocalAsyncWorkerProvider { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/ISegmentDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/ISegmentDAO.java index 1c1d67e03..4e8bb0353 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/ISegmentDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/ISegmentDAO.java @@ -1,12 +1,7 @@ package org.skywalking.apm.collector.agentstream.worker.segment.origin.dao; -import java.util.List; -import java.util.Map; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; - /** * @author pengys5 */ public interface ISegmentDAO { - List prepareBatch(Map dataMap); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/SegmentEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/SegmentEsDAO.java index 11ced7431..edfb0cd12 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/SegmentEsDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/SegmentEsDAO.java @@ -1,34 +1,37 @@ package org.skywalking.apm.collector.agentstream.worker.segment.origin.dao; -import java.util.ArrayList; import java.util.Base64; import java.util.HashMap; -import java.util.List; import java.util.Map; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.skywalking.apm.collector.agentstream.worker.segment.origin.define.SegmentTable; import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author pengys5 */ -public class SegmentEsDAO extends EsDAO implements ISegmentDAO { +public class SegmentEsDAO extends EsDAO implements ISegmentDAO, IPersistenceDAO { private final Logger logger = LoggerFactory.getLogger(SegmentEsDAO.class); - @Override public List prepareBatch(Map dataMap) { - List indexRequestBuilders = new ArrayList<>(); - dataMap.forEach((id, data) -> { - logger.debug("segment prepareBatch, id: {}", id); - Map source = new HashMap(); - source.put(SegmentTable.COLUMN_DATA_BINARY, new String(Base64.getEncoder().encode(data.getDataBytes(0)))); - logger.debug("segment source: {}", source.toString()); - IndexRequestBuilder builder = getClient().prepareIndex(SegmentTable.TABLE, id).setSource(source); - indexRequestBuilders.add(builder); - }); - return indexRequestBuilders; + @Override public Data get(String id, DataDefine dataDefine) { + return null; + } + + @Override public UpdateRequestBuilder prepareBatchUpdate(Data data) { + return null; + } + + @Override public IndexRequestBuilder prepareBatchInsert(Data data) { + Map source = new HashMap<>(); + source.put(SegmentTable.COLUMN_DATA_BINARY, new String(Base64.getEncoder().encode(data.getDataBytes(0)))); + logger.debug("segment source: {}", source.toString()); + return getClient().prepareIndex(SegmentTable.TABLE, data.getDataString(0)).setSource(source); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/SegmentH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/SegmentH2DAO.java index 5e9918fe8..363088b87 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/SegmentH2DAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/SegmentH2DAO.java @@ -1,15 +1,9 @@ package org.skywalking.apm.collector.agentstream.worker.segment.origin.dao; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.storage.h2.dao.H2DAO; /** * @author pengys5 */ public class SegmentH2DAO extends H2DAO implements ISegmentDAO { - - @Override public List prepareBatch(Map map) { - return null; - } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/IDNameExchangeTimer.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/IDNameExchangeTimer.java deleted file mode 100644 index c1880a589..000000000 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/IDNameExchangeTimer.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.skywalking.apm.collector.agentstream.worker.storage; - -import java.util.List; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import org.skywalking.apm.collector.core.framework.Starter; -import org.skywalking.apm.collector.stream.worker.WorkerException; -import org.skywalking.apm.collector.stream.worker.impl.ExchangeWorker; -import org.skywalking.apm.collector.stream.worker.impl.ExchangeWorkerContainer; -import org.skywalking.apm.collector.stream.worker.impl.FlushAndSwitch; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author pengys5 - */ -public class IDNameExchangeTimer implements Starter { - - private final Logger logger = LoggerFactory.getLogger(IDNameExchangeTimer.class); - - public void start() { - logger.info("id and name exchange timer start"); - //TODO timer value config -// final long timeInterval = EsConfig.Es.Persistence.Timer.VALUE * 1000; - final long timeInterval = 3; - - Executors.newSingleThreadScheduledExecutor().schedule(() -> exchangeLastData(), timeInterval, TimeUnit.SECONDS); - } - - private void exchangeLastData() { - List workers = ExchangeWorkerContainer.INSTANCE.getExchangeWorkers(); - workers.forEach((ExchangeWorker worker) -> { - try { - worker.allocateJob(new FlushAndSwitch()); - worker.exchangeLastData(); - } catch (WorkerException e) { - logger.error(e.getMessage(), e); - } - }); - } -} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java index ac02fee13..aeb38e243 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java @@ -26,7 +26,7 @@ public class PersistenceTimer implements Starter { //TODO timer value config // final long timeInterval = EsConfig.Es.Persistence.Timer.VALUE * 1000; final long timeInterval = 3; - Executors.newSingleThreadScheduledExecutor().schedule(() -> extractDataAndSave(), timeInterval, TimeUnit.SECONDS); + Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> extractDataAndSave(), 1, timeInterval, TimeUnit.SECONDS); } private void extractDataAndSave() { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/util/ExchangeMarkUtils.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/util/ExchangeMarkUtils.java new file mode 100644 index 000000000..84faf171e --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/util/ExchangeMarkUtils.java @@ -0,0 +1,14 @@ +package org.skywalking.apm.collector.agentstream.worker.util; + +/** + * @author pengys5 + */ +public enum ExchangeMarkUtils { + INSTANCE; + + private static final String MARK_TAG = "M"; + + public String buildMarkedID(int id) { + return MARK_TAG + id; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define deleted file mode 100644 index c8abe9d72..000000000 --- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define +++ /dev/null @@ -1,4 +0,0 @@ -org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentDataDefine -org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine -org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceDataDefine -org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameDataDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_worker_provider.define similarity index 90% rename from apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define rename to apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_worker_provider.define index a1744eee4..fb133fbe0 100644 --- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define +++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_worker_provider.define @@ -2,7 +2,6 @@ org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentAggr org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentPersistenceWorker$Factory org.skywalking.apm.collector.agentstream.worker.node.mapping.NodeMappingAggregationWorker$Factory -org.skywalking.apm.collector.agentstream.worker.node.mapping.NodeComponentExchangeWorker$Factory org.skywalking.apm.collector.agentstream.worker.node.mapping.NodeMappingPersistenceWorker$Factory org.skywalking.apm.collector.agentstream.worker.noderef.reference.NodeRefAggregationWorker$Factory diff --git a/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandlerTestCase.java b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandlerTestCase.java index fc988f521..3949d8a15 100644 --- a/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandlerTestCase.java +++ b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandlerTestCase.java @@ -109,7 +109,7 @@ public class TraceSegmentServiceHandlerTestCase { ref_0.setParentServiceName(""); ref_0.setParentSpanId(2); ref_0.setParentTraceSegmentId(UniqueId.newBuilder().addIdParts(100).addIdParts(100).addIdParts(100).build()); - segmentBuilder.addRefs(ref_0); +// segmentBuilder.addRefs(ref_0); builder.setSegment(segmentBuilder.build().toByteString()); } diff --git a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java index 58cf145ef..1e4560af3 100644 --- a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java +++ b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.get.GetRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.client.IndicesAdminClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; @@ -111,9 +112,14 @@ public class ElasticSearchClient implements Client { } public IndexRequestBuilder prepareIndex(String indexName, String id) { + client.prepareUpdate(); return client.prepareIndex(indexName, "type", id); } + public UpdateRequestBuilder prepareUpdate(String indexName, String id) { + return client.prepareUpdate(indexName, "type", id); + } + public GetRequestBuilder prepareGet(String indexName, String id) { return client.prepareGet(indexName, "type", id); } diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageInstaller.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageInstaller.java index 0ae185794..3cd7559af 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageInstaller.java +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageInstaller.java @@ -23,8 +23,11 @@ public abstract class StorageInstaller { if (!isExists(client, tableDefine)) { logger.info("table: {} not exists", tableDefine.getName()); tableDefine.initialize(); - createTable(client, tableDefine); + } else { + logger.info("table: {} exists", tableDefine.getName()); + deleteTable(client, tableDefine); } + createTable(client, tableDefine); } } catch (DefineException e) { throw new StorageInstallException(e.getMessage(), e); @@ -35,7 +38,7 @@ public abstract class StorageInstaller { protected abstract boolean isExists(Client client, TableDefine tableDefine) throws StorageException; - protected abstract boolean deleteIndex(Client client, TableDefine tableDefine) throws StorageException; + protected abstract boolean deleteTable(Client client, TableDefine tableDefine) throws StorageException; protected abstract boolean createTable(Client client, TableDefine tableDefine) throws StorageException; } diff --git a/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/jetty/JettyHandler.java b/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/jetty/JettyHandler.java index 7b7c39bc5..7921df426 100644 --- a/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/jetty/JettyHandler.java +++ b/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/jetty/JettyHandler.java @@ -34,9 +34,16 @@ public abstract class JettyHandler extends HttpServlet implements Handler { @Override protected final void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - super.doPost(req, resp); + try { + doPost(req); + reply(resp); + } catch (ArgumentsParseException e) { + replyError(resp, e.getMessage(), HttpServletResponse.SC_BAD_REQUEST); + } } + protected abstract void doPost(HttpServletRequest req) throws ArgumentsParseException; + @Override protected final void doHead(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { super.doHead(req, resp); @@ -134,13 +141,23 @@ public abstract class JettyHandler extends HttpServlet implements Handler { out.close(); } + private void reply(HttpServletResponse response) throws IOException { + response.setContentType("text/json"); + response.setCharacterEncoding("utf-8"); + response.setStatus(HttpServletResponse.SC_OK); + + PrintWriter out = response.getWriter(); + out.flush(); + out.close(); + } + private void replyError(HttpServletResponse response, String errorMessage, int status) throws IOException { response.setContentType("text/plain"); response.setCharacterEncoding("utf-8"); response.setStatus(status); + response.setHeader("error-message", errorMessage); PrintWriter out = response.getWriter(); - out.print(errorMessage); out.flush(); out.close(); } diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/define/ElasticSearchStorageInstaller.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/define/ElasticSearchStorageInstaller.java index 25dd33d61..26c618f13 100644 --- a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/define/ElasticSearchStorageInstaller.java +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/define/ElasticSearchStorageInstaller.java @@ -81,7 +81,7 @@ public class ElasticSearchStorageInstaller extends StorageInstaller { return mappingBuilder; } - @Override protected boolean deleteIndex(Client client, TableDefine tableDefine) { + @Override protected boolean deleteTable(Client client, TableDefine tableDefine) { ElasticSearchClient esClient = (ElasticSearchClient)client; try { return esClient.deleteIndex(tableDefine.getName()); diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/define/H2StorageInstaller.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/define/H2StorageInstaller.java index d27a8ca74..3244808a5 100644 --- a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/define/H2StorageInstaller.java +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/define/H2StorageInstaller.java @@ -27,7 +27,7 @@ public class H2StorageInstaller extends StorageInstaller { return false; } - @Override protected boolean deleteIndex(Client client, TableDefine tableDefine) throws StorageException { + @Override protected boolean deleteTable(Client client, TableDefine tableDefine) throws StorageException { return false; } diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java index 908eecbb0..a1f22c2fa 100644 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java @@ -16,8 +16,6 @@ import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; import org.skywalking.apm.collector.stream.worker.LocalAsyncWorkerProviderDefineLoader; import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; import org.skywalking.apm.collector.stream.worker.RemoteWorkerProviderDefineLoader; -import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; -import org.skywalking.apm.collector.stream.worker.impl.data.DataDefineLoader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,10 +32,6 @@ public class StreamModuleInstaller implements ModuleInstaller { StreamModuleContext context = new StreamModuleContext(StreamModuleGroupDefine.GROUP_NAME); CollectorContextHelper.INSTANCE.putContext(context); - DataDefineLoader dataDefineLoader = new DataDefineLoader(); - Map dataDefineMap = dataDefineLoader.load(); - context.putAllDataDefine(dataDefineMap); - initializeWorker(context); logger.info("could not configure cluster module, use the default"); diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorker.java index e43561bee..896a2ad51 100644 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorker.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorker.java @@ -9,7 +9,9 @@ import org.skywalking.apm.collector.core.queue.QueueExecutor; * @author pengys5 * @since v3.0-2017 */ -public abstract class AbstractLocalAsyncWorker extends AbstractLocalWorker implements QueueExecutor { +public abstract class AbstractLocalAsyncWorker extends AbstractWorker implements QueueExecutor { + + private LocalAsyncWorkerRef workerRef; /** * Construct an AbstractLocalAsyncWorker with the worker role and context. @@ -32,6 +34,14 @@ public abstract class AbstractLocalAsyncWorker extends AbstractLocalWorker imple public void preStart() throws ProviderNotFoundException { } + @Override protected final LocalAsyncWorkerRef getSelf() { + return workerRef; + } + + @Override protected final void putSelfRef(LocalAsyncWorkerRef workerRef) { + this.workerRef = workerRef; + } + /** * Receive message * diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorkerProvider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorkerProvider.java index 7f00cb13c..fe702b1b7 100644 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorkerProvider.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorkerProvider.java @@ -6,15 +6,13 @@ import org.skywalking.apm.collector.core.queue.QueueEventHandler; import org.skywalking.apm.collector.core.queue.QueueExecutor; import org.skywalking.apm.collector.queue.QueueModuleContext; import org.skywalking.apm.collector.queue.QueueModuleGroupDefine; -import org.skywalking.apm.collector.stream.worker.impl.ExchangeWorker; -import org.skywalking.apm.collector.stream.worker.impl.ExchangeWorkerContainer; import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorkerContainer; /** * @author pengys5 */ -public abstract class AbstractLocalAsyncWorkerProvider extends AbstractLocalWorkerProvider { +public abstract class AbstractLocalAsyncWorkerProvider extends AbstractWorkerProvider { public abstract int queueSize(); @@ -24,8 +22,6 @@ public abstract class AbstractLocalAsyncWorkerProviderAbstractLocalSyncWorker defines workers who receive data from jvm inside call and response in real - * time. - * - * @author pengys5 - * @since v3.0-2017 - */ -public abstract class AbstractLocalSyncWorker extends AbstractLocalWorker { - public AbstractLocalSyncWorker(Role role, ClusterWorkerContext clusterContext) { - super(role, clusterContext); - } - - /** - * Called by the worker reference to execute the worker service. - * - * @param request {@link Object} is an input parameter - * @param response {@link Object} is an output parameter - */ - final public void allocateJob(Object request, Object response) throws WorkerInvokeException { - try { - onWork(request, response); - } catch (WorkerException e) { - throw new WorkerInvokeException(e.getMessage(), e.getCause()); - } - } - - /** - * Override this method to implement business logic. - * - * @param request {@link Object} is a in parameter - * @param response {@link Object} is a out parameter - */ - protected abstract void onWork(Object request, Object response) throws WorkerException; - - /** - * Prepare methods before this work starts to work. - *

Usually, create or find the workers reference should be call. - * - * @throws ProviderNotFoundException - */ - @Override - public void preStart() throws ProviderNotFoundException { - } -} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalSyncWorkerProvider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalSyncWorkerProvider.java deleted file mode 100644 index d3a542a61..000000000 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalSyncWorkerProvider.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.skywalking.apm.collector.stream.worker; - -/** - * @author pengys5 - */ -public abstract class AbstractLocalSyncWorkerProvider extends AbstractLocalWorkerProvider { - - @Override final public WorkerRef create() throws ProviderNotFoundException { - T localSyncWorker = workerInstance(getClusterContext()); - localSyncWorker.preStart(); - - LocalSyncWorkerRef workerRef = new LocalSyncWorkerRef(role(), localSyncWorker); - return workerRef; - } -} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorker.java deleted file mode 100644 index c92a23fc7..000000000 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorker.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.skywalking.apm.collector.stream.worker; - -/** - * @author pengys5 - */ -public abstract class AbstractLocalWorker extends AbstractWorker { - public AbstractLocalWorker(Role role, ClusterWorkerContext clusterContext) { - super(role, clusterContext); - } -} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorkerProvider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorkerProvider.java deleted file mode 100644 index 8cc96b71f..000000000 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorkerProvider.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.skywalking.apm.collector.stream.worker; - -/** - * @author pengys5 - */ -public abstract class AbstractLocalWorkerProvider extends AbstractWorkerProvider { -} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorker.java index 8dab25277..bba64efd2 100644 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorker.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorker.java @@ -9,7 +9,9 @@ package org.skywalking.apm.collector.stream.worker; * @author pengys5 * @since v3.0-2017 */ -public abstract class AbstractRemoteWorker extends AbstractWorker { +public abstract class AbstractRemoteWorker extends AbstractWorker { + + private RemoteWorkerRef workerRef; /** * Construct an AbstractRemoteWorker with the worker role and context. @@ -35,4 +37,12 @@ public abstract class AbstractRemoteWorker extends AbstractWorker { throw new WorkerInvokeException(e.getMessage(), e.getCause()); } } + + @Override protected final RemoteWorkerRef getSelf() { + return workerRef; + } + + @Override protected final void putSelfRef(RemoteWorkerRef workerRef) { + this.workerRef = workerRef; + } } diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractWorker.java index 42d69415a..01918b5e4 100644 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractWorker.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractWorker.java @@ -7,7 +7,7 @@ import org.slf4j.LoggerFactory; /** * @author pengys5 */ -public abstract class AbstractWorker implements Executor { +public abstract class AbstractWorker implements Executor { private final Logger logger = LoggerFactory.getLogger(AbstractWorker.class); @@ -45,4 +45,8 @@ public abstract class AbstractWorker implements Executor { final public Role getRole() { return role; } + + protected abstract S getSelf(); + + protected abstract void putSelfRef(S workerRef); } diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefineLoader.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefineLoader.java index 27fd2f1df..77ed5d4a5 100644 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefineLoader.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefineLoader.java @@ -17,7 +17,7 @@ public class LocalAsyncWorkerProviderDefineLoader implements Loader load() throws DefineException { List providers = new ArrayList<>(); - LocalAsyncWorkerProviderDefinitionFile definitionFile = new LocalAsyncWorkerProviderDefinitionFile(); + LocalWorkerProviderDefinitionFile definitionFile = new LocalWorkerProviderDefinitionFile(); logger.info("local async worker provider definition file name: {}", definitionFile.fileName()); DefinitionLoader definitionLoader = DefinitionLoader.load(AbstractLocalAsyncWorkerProvider.class, definitionFile); diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalSyncWorkerRef.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalSyncWorkerRef.java deleted file mode 100644 index d24d07779..000000000 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalSyncWorkerRef.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.skywalking.apm.collector.stream.worker; - -/** - * @author pengys5 - */ -public class LocalSyncWorkerRef extends WorkerRef { - - private AbstractLocalSyncWorker localSyncWorker; - - public LocalSyncWorkerRef(Role role, AbstractLocalSyncWorker localSyncWorker) { - super(role); - this.localSyncWorker = localSyncWorker; - } - - @Override - public void tell(Object message) throws WorkerInvokeException { - localSyncWorker.allocateJob(message, null); - } - - public void ask(Object request, Object response) throws WorkerInvokeException { - localSyncWorker.allocateJob(request, response); - } -} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefinitionFile.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalWorkerProviderDefinitionFile.java similarity index 60% rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefinitionFile.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalWorkerProviderDefinitionFile.java index 356cf9ffb..5dbfa09f7 100644 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefinitionFile.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalWorkerProviderDefinitionFile.java @@ -5,8 +5,8 @@ import org.skywalking.apm.collector.core.framework.DefinitionFile; /** * @author pengys5 */ -public class LocalAsyncWorkerProviderDefinitionFile extends DefinitionFile { +public class LocalWorkerProviderDefinitionFile extends DefinitionFile { @Override protected String fileName() { - return "local_async_worker_provider.define"; + return "local_worker_provider.define"; } } diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/ExchangeWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/ExchangeWorker.java deleted file mode 100644 index 3a2e8a098..000000000 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/ExchangeWorker.java +++ /dev/null @@ -1,74 +0,0 @@ -package org.skywalking.apm.collector.stream.worker.impl; - -import org.skywalking.apm.collector.core.queue.EndOfBatchCommand; -import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorker; -import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; -import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; -import org.skywalking.apm.collector.stream.worker.Role; -import org.skywalking.apm.collector.stream.worker.WorkerException; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; -import org.skywalking.apm.collector.stream.worker.impl.data.DataCache; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author pengys5 - */ -public abstract class ExchangeWorker extends AbstractLocalAsyncWorker { - - private final Logger logger = LoggerFactory.getLogger(ExchangeWorker.class); - - private DataCache dataCache; - - public ExchangeWorker(Role role, ClusterWorkerContext clusterContext) { - super(role, clusterContext); - dataCache = new DataCache(); - } - - @Override public void preStart() throws ProviderNotFoundException { - super.preStart(); - } - - @Override protected final void onWork(Object message) throws WorkerException { - if (message instanceof FlushAndSwitch) { - if (dataCache.trySwitchPointer()) { - dataCache.switchPointer(); - } - } else if (message instanceof EndOfBatchCommand) { - } else { - if (dataCache.currentCollectionSize() <= 1000) { - aggregate(message); - } - } - } - - protected abstract void exchange(Data data); - - public final void exchangeLastData() { - try { - while (dataCache.getLast().isHolding()) { - try { - Thread.sleep(10); - } catch (InterruptedException e) { - logger.warn("thread wake up"); - } - } - dataCache.getLast().asMap().values().forEach(data -> { - exchange(data); - }); - } finally { - dataCache.releaseLast(); - } - } - - protected final void aggregate(Object message) { - Data data = (Data)message; - dataCache.hold(); - if (dataCache.containsKey(data.id())) { - getRole().dataDefine().mergeData(data, dataCache.get(data.id())); - } else { - dataCache.put(data.id(), data); - } - dataCache.release(); - } -} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/ExchangeWorkerContainer.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/ExchangeWorkerContainer.java deleted file mode 100644 index 38ffea545..000000000 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/ExchangeWorkerContainer.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.skywalking.apm.collector.stream.worker.impl; - -import java.util.ArrayList; -import java.util.List; - -/** - * @author pengys5 - */ -public enum ExchangeWorkerContainer { - INSTANCE; - - private List exchangeWorkers = new ArrayList<>(); - - public void addWorker(ExchangeWorker worker) { - exchangeWorkers.add(worker); - } - - public List getExchangeWorkers() { - return exchangeWorkers; - } -} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/PersistenceWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/PersistenceWorker.java index f0f3bae3b..878c2c6e0 100644 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/PersistenceWorker.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/PersistenceWorker.java @@ -1,13 +1,16 @@ package org.skywalking.apm.collector.stream.worker.impl; +import java.util.ArrayList; import java.util.List; import java.util.Map; import org.skywalking.apm.collector.core.queue.EndOfBatchCommand; +import org.skywalking.apm.collector.core.util.ObjectUtils; import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorker; import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; import org.skywalking.apm.collector.stream.worker.Role; import org.skywalking.apm.collector.stream.worker.WorkerException; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.Data; import org.skywalking.apm.collector.stream.worker.impl.data.DataCache; import org.slf4j.Logger; @@ -46,7 +49,7 @@ public abstract class PersistenceWorker extends AbstractLocalAsyncWorker { } } - public List buildBatchCollection() throws WorkerException { + public final List buildBatchCollection() throws WorkerException { List batchCollection; try { while (dataCache.getLast().isHolding()) { @@ -56,6 +59,7 @@ public abstract class PersistenceWorker extends AbstractLocalAsyncWorker { logger.warn("thread wake up"); } } + batchCollection = prepareBatch(dataCache.getLast().asMap()); } finally { dataCache.releaseLast(); @@ -63,7 +67,26 @@ public abstract class PersistenceWorker extends AbstractLocalAsyncWorker { return batchCollection; } - protected abstract List prepareBatch(Map dataMap); + protected final List prepareBatch(Map dataMap) { + List insertBatchCollection = new ArrayList<>(); + List updateBatchCollection = new ArrayList<>(); + dataMap.forEach((id, data) -> { + if (needMergeDBData()) { + Data dbData = persistenceDAO().get(id, getRole().dataDefine()); + if (ObjectUtils.isNotEmpty(dbData)) { + getRole().dataDefine().mergeData(data, dbData); + updateBatchCollection.add(persistenceDAO().prepareBatchUpdate(data)); + } else { + insertBatchCollection.add(persistenceDAO().prepareBatchInsert(data)); + } + } else { + insertBatchCollection.add(persistenceDAO().prepareBatchInsert(data)); + } + }); + + insertBatchCollection.addAll(updateBatchCollection); + return insertBatchCollection; + } private void aggregate(Object message) { dataCache.hold(); @@ -79,4 +102,8 @@ public abstract class PersistenceWorker extends AbstractLocalAsyncWorker { dataCache.release(); } + + protected abstract IPersistenceDAO persistenceDAO(); + + protected abstract boolean needMergeDBData(); } diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/dao/IPersistenceDAO.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/dao/IPersistenceDAO.java new file mode 100644 index 000000000..956586265 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/dao/IPersistenceDAO.java @@ -0,0 +1,15 @@ +package org.skywalking.apm.collector.stream.worker.impl.dao; + +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; + +/** + * @author pengys5 + */ +public interface IPersistenceDAO { + Data get(String id, DataDefine dataDefine); + + I prepareBatchInsert(Data data); + + U prepareBatchUpdate(Data data); +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefineLoader.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefineLoader.java deleted file mode 100644 index 425fcb427..000000000 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefineLoader.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.skywalking.apm.collector.stream.worker.impl.data; - -import java.util.HashMap; -import java.util.Map; -import org.skywalking.apm.collector.core.framework.DefineException; -import org.skywalking.apm.collector.core.framework.Loader; -import org.skywalking.apm.collector.core.util.DefinitionLoader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author pengys5 - */ -public class DataDefineLoader implements Loader> { - - private final Logger logger = LoggerFactory.getLogger(DataDefineLoader.class); - - @Override public Map load() throws DefineException { - Map dataDefineMap = new HashMap<>(); - - DataDefinitionFile definitionFile = new DataDefinitionFile(); - DefinitionLoader definitionLoader = DefinitionLoader.load(DataDefine.class, definitionFile); - for (DataDefine dataDefine : definitionLoader) { - logger.info("loaded data definition class: {}", dataDefine.getClass().getName()); - dataDefineMap.put(dataDefine.defineId(), dataDefine); - } - return dataDefineMap; - } -} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefinitionFile.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefinitionFile.java deleted file mode 100644 index e7875d8b8..000000000 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefinitionFile.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.skywalking.apm.collector.stream.worker.impl.data; - -import org.skywalking.apm.collector.core.framework.DefinitionFile; - -/** - * @author pengys5 - */ -public class DataDefinitionFile extends DefinitionFile { - - @Override protected String fileName() { - return "data.define"; - } -} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Exchange.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Exchange.java deleted file mode 100644 index 3d37d4500..000000000 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Exchange.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.skywalking.apm.collector.stream.worker.impl.data; - -/** - * @author pengys5 - */ -public abstract class Exchange { - private int times; - - public Exchange(int times) { - this.times = times; - } - - public void increase() { - times++; - } - - public int getTimes() { - return times; - } - - public void setTimes(int times) { - this.times = times; - } -} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/SegmentCostEsDAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/SegmentCostEsDAO.java index c06df8079..4e1d05699 100644 --- a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/SegmentCostEsDAO.java +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/SegmentCostEsDAO.java @@ -45,7 +45,7 @@ public class SegmentCostEsDAO extends EsDAO implements ISegmentCostDAO { boolQueryBuilder.must().add(rangeQueryBuilder); } if (!StringUtils.isEmpty(operationName)) { - mustQueryList.add(QueryBuilders.matchQuery(SegmentCostTable.COLUMN_OPERATION_NAME, operationName)); + mustQueryList.add(QueryBuilders.matchQuery(SegmentCostTable.COLUMN_SERVICE_NAME, operationName)); } searchRequestBuilder.addSort(SegmentCostTable.COLUMN_COST, SortOrder.DESC); @@ -77,7 +77,7 @@ public class SegmentCostEsDAO extends EsDAO implements ISegmentCostDAO { topSegmentJson.addProperty(GlobalTraceTable.COLUMN_GLOBAL_TRACE_ID, globalTraces.get(0)); } - topSegmentJson.addProperty(SegmentCostTable.COLUMN_OPERATION_NAME, (String)searchHit.getSource().get(SegmentCostTable.COLUMN_OPERATION_NAME)); + topSegmentJson.addProperty(SegmentCostTable.COLUMN_SERVICE_NAME, (String)searchHit.getSource().get(SegmentCostTable.COLUMN_SERVICE_NAME)); topSegmentJson.addProperty(SegmentCostTable.COLUMN_COST, (Number)searchHit.getSource().get(SegmentCostTable.COLUMN_COST)); topSegmentJson.addProperty(SegmentCostTable.COLUMN_IS_ERROR, (Boolean)searchHit.getSource().get(SegmentCostTable.COLUMN_IS_ERROR)); From a0d1f5a752661c8d2f0768d66063b7c8d36625b6 Mon Sep 17 00:00:00 2001 From: pengys5 <8082209@qq.com> Date: Sat, 5 Aug 2017 13:21:24 +0800 Subject: [PATCH 2/8] Fixed package error --- .../apm/collector/ui/jetty/handler/SegmentTopGetHandler.java | 4 ++++ .../apm/collector/ui/jetty/handler/SpanGetHandler.java | 4 ++++ .../apm/collector/ui/jetty/handler/TraceDagGetHandler.java | 4 ++++ .../apm/collector/ui/jetty/handler/TraceStackGetHandler.java | 4 ++++ .../apm/collector/ui/jetty/handler/UIJettyServerHandler.java | 4 ++++ 5 files changed, 20 insertions(+) diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SegmentTopGetHandler.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SegmentTopGetHandler.java index 0feae9192..195638203 100644 --- a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SegmentTopGetHandler.java +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SegmentTopGetHandler.java @@ -79,4 +79,8 @@ public class SegmentTopGetHandler extends JettyHandler { return service.loadTop(startTime, endTime, minCost, maxCost, operationName, globalTraceId, limit, from); } + + @Override protected void doPost(HttpServletRequest req) throws ArgumentsParseException { + throw new UnsupportedOperationException(); + } } diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SpanGetHandler.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SpanGetHandler.java index eda5239f1..db1fa3a64 100644 --- a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SpanGetHandler.java +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SpanGetHandler.java @@ -35,4 +35,8 @@ public class SpanGetHandler extends JettyHandler { return service.load(segmentId, spanId); } + + @Override protected void doPost(HttpServletRequest req) throws ArgumentsParseException { + throw new UnsupportedOperationException(); + } } diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceDagGetHandler.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceDagGetHandler.java index 71a8a2c40..494b004ae 100644 --- a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceDagGetHandler.java +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceDagGetHandler.java @@ -31,4 +31,8 @@ public class TraceDagGetHandler extends JettyHandler { long endTime = Long.valueOf(endTimeStr); return service.load(startTime, endTime, timeBucketType); } + + @Override protected void doPost(HttpServletRequest req) throws ArgumentsParseException { + throw new UnsupportedOperationException(); + } } diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceStackGetHandler.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceStackGetHandler.java index 1792158b0..c24eb53af 100644 --- a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceStackGetHandler.java +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceStackGetHandler.java @@ -27,4 +27,8 @@ public class TraceStackGetHandler extends JettyHandler { return service.load(globalTraceId); } + + @Override protected void doPost(HttpServletRequest req) throws ArgumentsParseException { + throw new UnsupportedOperationException(); + } } diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/UIJettyServerHandler.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/UIJettyServerHandler.java index ca6f39340..c53e5112f 100644 --- a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/UIJettyServerHandler.java +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/UIJettyServerHandler.java @@ -30,4 +30,8 @@ public class UIJettyServerHandler extends JettyHandler { }); return serverArray; } + + @Override protected void doPost(HttpServletRequest req) throws ArgumentsParseException { + throw new UnsupportedOperationException(); + } } From 3492b0d86ee5d850404a478905c720f5599f8724 Mon Sep 17 00:00:00 2001 From: pengys5 <8082209@qq.com> Date: Sat, 5 Aug 2017 13:39:32 +0800 Subject: [PATCH 3/8] fixed package error --- .../jetty/handler/AgentStreamGRPCServerHandler.java | 4 ++++ .../jetty/handler/AgentStreamJettyServerHandler.java | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamGRPCServerHandler.java b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamGRPCServerHandler.java index 2c4e4bebb..db9b9362a 100644 --- a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamGRPCServerHandler.java +++ b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamGRPCServerHandler.java @@ -30,4 +30,8 @@ public class AgentStreamGRPCServerHandler extends JettyHandler { }); return serverArray; } + + @Override protected void doPost(HttpServletRequest req) throws ArgumentsParseException { + throw new UnsupportedOperationException(); + } } diff --git a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamJettyServerHandler.java b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamJettyServerHandler.java index dcaf05cfa..a463de01d 100644 --- a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamJettyServerHandler.java +++ b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamJettyServerHandler.java @@ -30,4 +30,8 @@ public class AgentStreamJettyServerHandler extends JettyHandler { }); return serverArray; } + + @Override protected void doPost(HttpServletRequest req) throws ArgumentsParseException { + throw new UnsupportedOperationException(); + } } From 911de3fe4a500ca01ea69f29192bbad3c0909d23 Mon Sep 17 00:00:00 2001 From: pengys5 <8082209@qq.com> Date: Sun, 6 Aug 2017 21:54:55 +0800 Subject: [PATCH 4/8] 1. Provide jetty http interface for application and instance register. 2. Provide jetty http interface for receive trace segment. 3. Test segment receive trace segment success. segment, node reference, node reference sum, node component, node mapping, global trace data are all correct. --- .../jetty/AgentRegisterJettyConfig.java | 10 ++ .../jetty/AgentRegisterJettyConfigParser.java | 36 +++++++ .../jetty/AgentRegisterJettyDataListener.java | 21 ++++ .../jetty/AgentRegisterJettyModuleDefine.java | 53 ++++++++++ .../AgentRegisterJettyModuleRegistration.java | 13 +++ .../ApplicationRegisterServletHandler.java | 52 ++++++++++ .../InstanceDiscoveryServletHandler.java | 53 ++++++++++ .../resources/META-INF/defines/module.define | 3 +- .../handler/AgentStreamGRPCServerHandler.java | 6 +- .../AgentStreamJettyServerHandler.java | 2 +- .../jetty/AgentStreamJettyModuleDefine.java | 6 +- .../handler/TraceSegmentServiceHandler.java | 34 ------- .../handler/TraceSegmentServletHandler.java | 54 ++++++++++ .../reader/KeyWithStringValueJsonReader.java | 36 +++++++ .../jetty/handler/reader/LogJsonReader.java | 37 +++++++ .../handler/reader/ReferenceJsonReader.java | 70 +++++++++++++ .../handler/reader/SegmentJsonReader.java | 69 +++++++++++++ .../jetty/handler/reader/SpanJsonReader.java | 99 +++++++++++++++++++ .../handler/reader/StreamJsonReader.java | 11 +++ .../jetty/handler/reader/TraceSegment.java | 35 +++++++ .../reader/TraceSegmentJsonReader.java | 54 ++++++++++ .../handler/reader/UniqueIdJsonReader.java | 22 +++++ .../collector/agentstream/worker/Const.java | 1 + .../worker/cache/InstanceCache.java | 25 +++++ .../component/NodeComponentSpanListener.java | 4 +- .../node/mapping/NodeMappingSpanListener.java | 4 +- .../reference/NodeRefSpanListener.java | 23 +++-- .../summary/NodeRefSumSpanListener.java | 27 ++++- .../ApplicationRegisterSerialWorker.java | 8 +- .../register/instance/dao/IInstanceDAO.java | 2 + .../register/instance/dao/InstanceEsDAO.java | 17 +++- .../register/instance/dao/InstanceH2DAO.java | 4 + .../worker/storage/PersistenceTimer.java | 32 +++--- .../agentstream/HttpClientTools.java | 77 +++++++++++++++ .../TraceSegmentJsonReaderTestCase.java | 28 ++++++ .../agentstream/mock/JsonFileReader.java | 24 +++++ .../agentstream/mock/SegmentPost.java | 39 ++++++++ .../segment/normal/application-register.json | 4 + .../json/segment/normal/dubbox-consumer.json | 71 +++++++++++++ .../json/segment/normal/dubbox-provider.json | 66 +++++++++++++ .../segment/normal/instance-register.json | 5 + .../elasticsearch/ElasticSearchClient.java | 1 - .../collector/server/jetty/JettyHandler.java | 22 ++--- .../storage/elasticsearch/dao/BatchEsDAO.java | 12 ++- .../jetty/handler/SegmentTopGetHandler.java | 2 +- .../ui/jetty/handler/SpanGetHandler.java | 2 +- .../ui/jetty/handler/TraceDagGetHandler.java | 2 +- .../jetty/handler/TraceStackGetHandler.java | 2 +- .../jetty/handler/UIJettyServerHandler.java | 2 +- 49 files changed, 1183 insertions(+), 99 deletions(-) create mode 100644 apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyConfig.java create mode 100644 apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyConfigParser.java create mode 100644 apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyDataListener.java create mode 100644 apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyModuleDefine.java create mode 100644 apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyModuleRegistration.java create mode 100644 apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/handler/ApplicationRegisterServletHandler.java create mode 100644 apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/handler/InstanceDiscoveryServletHandler.java delete mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/TraceSegmentServiceHandler.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/TraceSegmentServletHandler.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/KeyWithStringValueJsonReader.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/LogJsonReader.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/ReferenceJsonReader.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/SegmentJsonReader.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/SpanJsonReader.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/StreamJsonReader.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegment.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegmentJsonReader.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/UniqueIdJsonReader.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/InstanceCache.java create mode 100644 apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/HttpClientTools.java create mode 100644 apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegmentJsonReaderTestCase.java create mode 100644 apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/mock/JsonFileReader.java create mode 100644 apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/mock/SegmentPost.java create mode 100644 apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/application-register.json create mode 100644 apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/dubbox-consumer.json create mode 100644 apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/dubbox-provider.json create mode 100644 apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/instance-register.json diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyConfig.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyConfig.java new file mode 100644 index 000000000..cf1e45207 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyConfig.java @@ -0,0 +1,10 @@ +package org.skywalking.apm.collector.agentregister.jetty; + +/** + * @author pengys5 + */ +public class AgentRegisterJettyConfig { + public static String HOST; + public static int PORT; + public static String CONTEXT_PATH; +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyConfigParser.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyConfigParser.java new file mode 100644 index 000000000..65a10bebe --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyConfigParser.java @@ -0,0 +1,36 @@ +package org.skywalking.apm.collector.agentregister.jetty; + +import java.util.Map; +import org.skywalking.apm.collector.core.config.ConfigParseException; +import org.skywalking.apm.collector.core.module.ModuleConfigParser; +import org.skywalking.apm.collector.core.util.ObjectUtils; +import org.skywalking.apm.collector.core.util.StringUtils; + +/** + * @author pengys5 + */ +public class AgentRegisterJettyConfigParser implements ModuleConfigParser { + + private static final String HOST = "host"; + private static final String PORT = "port"; + public static final String CONTEXT_PATH = "contextPath"; + + @Override public void parse(Map config) throws ConfigParseException { + AgentRegisterJettyConfig.CONTEXT_PATH = "/"; + + if (ObjectUtils.isEmpty(config) || StringUtils.isEmpty(config.get(HOST))) { + AgentRegisterJettyConfig.HOST = "localhost"; + } else { + AgentRegisterJettyConfig.HOST = (String)config.get(HOST); + } + + if (ObjectUtils.isEmpty(config) || StringUtils.isEmpty(config.get(PORT))) { + AgentRegisterJettyConfig.PORT = 12800; + } else { + AgentRegisterJettyConfig.PORT = (Integer)config.get(PORT); + } + if (ObjectUtils.isNotEmpty(config) && StringUtils.isNotEmpty(config.get(CONTEXT_PATH))) { + AgentRegisterJettyConfig.CONTEXT_PATH = (String)config.get(CONTEXT_PATH); + } + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyDataListener.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyDataListener.java new file mode 100644 index 000000000..218e0f42d --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyDataListener.java @@ -0,0 +1,21 @@ +package org.skywalking.apm.collector.agentregister.jetty; + +import org.skywalking.apm.collector.agentregister.AgentRegisterModuleGroupDefine; +import org.skywalking.apm.collector.agentregister.grpc.AgentRegisterGRPCModuleDefine; +import org.skywalking.apm.collector.cluster.ClusterModuleDefine; +import org.skywalking.apm.collector.core.cluster.ClusterDataListener; + +/** + * @author pengys5 + */ +public class AgentRegisterJettyDataListener extends ClusterDataListener { + + public static final String PATH = ClusterModuleDefine.BASE_CATALOG + "." + AgentRegisterModuleGroupDefine.GROUP_NAME + "." + AgentRegisterJettyModuleDefine.MODULE_NAME; + + @Override public String path() { + return PATH; + } + + @Override public void addressChangedNotify() { + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyModuleDefine.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyModuleDefine.java new file mode 100644 index 000000000..413149d71 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyModuleDefine.java @@ -0,0 +1,53 @@ +package org.skywalking.apm.collector.agentregister.jetty; + +import java.util.LinkedList; +import java.util.List; +import org.skywalking.apm.collector.agentregister.AgentRegisterModuleDefine; +import org.skywalking.apm.collector.agentregister.AgentRegisterModuleGroupDefine; +import org.skywalking.apm.collector.agentregister.jetty.handler.ApplicationRegisterServletHandler; +import org.skywalking.apm.collector.agentregister.jetty.handler.InstanceDiscoveryServletHandler; +import org.skywalking.apm.collector.core.cluster.ClusterDataListener; +import org.skywalking.apm.collector.core.framework.Handler; +import org.skywalking.apm.collector.core.module.ModuleConfigParser; +import org.skywalking.apm.collector.core.module.ModuleRegistration; +import org.skywalking.apm.collector.core.server.Server; +import org.skywalking.apm.collector.server.jetty.JettyServer; + +/** + * @author pengys5 + */ +public class AgentRegisterJettyModuleDefine extends AgentRegisterModuleDefine { + + public static final String MODULE_NAME = "jetty"; + + @Override protected String group() { + return AgentRegisterModuleGroupDefine.GROUP_NAME; + } + + @Override public String name() { + return MODULE_NAME; + } + + @Override protected ModuleConfigParser configParser() { + return new AgentRegisterJettyConfigParser(); + } + + @Override protected Server server() { + return new JettyServer(AgentRegisterJettyConfig.HOST, AgentRegisterJettyConfig.PORT, AgentRegisterJettyConfig.CONTEXT_PATH); + } + + @Override protected ModuleRegistration registration() { + return new AgentRegisterJettyModuleRegistration(); + } + + @Override public ClusterDataListener listener() { + return new AgentRegisterJettyDataListener(); + } + + @Override public List handlerList() { + List handlers = new LinkedList<>(); + handlers.add(new ApplicationRegisterServletHandler()); + handlers.add(new InstanceDiscoveryServletHandler()); + return handlers; + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyModuleRegistration.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyModuleRegistration.java new file mode 100644 index 000000000..d6e2d260a --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyModuleRegistration.java @@ -0,0 +1,13 @@ +package org.skywalking.apm.collector.agentregister.jetty; + +import org.skywalking.apm.collector.core.module.ModuleRegistration; + +/** + * @author pengys5 + */ +public class AgentRegisterJettyModuleRegistration extends ModuleRegistration { + + @Override public Value buildValue() { + return new Value(AgentRegisterJettyConfig.HOST, AgentRegisterJettyConfig.PORT, AgentRegisterJettyConfig.CONTEXT_PATH); + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/handler/ApplicationRegisterServletHandler.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/handler/ApplicationRegisterServletHandler.java new file mode 100644 index 000000000..7c7f2a99b --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/handler/ApplicationRegisterServletHandler.java @@ -0,0 +1,52 @@ +package org.skywalking.apm.collector.agentregister.jetty.handler; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import java.io.IOException; +import javax.servlet.http.HttpServletRequest; +import org.skywalking.apm.collector.agentregister.application.ApplicationIDService; +import org.skywalking.apm.collector.server.jetty.ArgumentsParseException; +import org.skywalking.apm.collector.server.jetty.JettyHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class ApplicationRegisterServletHandler extends JettyHandler { + + private final Logger logger = LoggerFactory.getLogger(ApplicationRegisterServletHandler.class); + + private ApplicationIDService applicationIDService = new ApplicationIDService(); + private Gson gson = new Gson(); + private static final String APPLICATION_CODE = "c"; + private static final String APPLICATION_ID = "i"; + + @Override public String pathSpec() { + return "/application/register"; + } + + @Override protected JsonElement doGet(HttpServletRequest req) throws ArgumentsParseException { + throw new UnsupportedOperationException(); + } + + @Override protected JsonElement doPost(HttpServletRequest req) throws ArgumentsParseException { + JsonArray responseArray = new JsonArray(); + try { + JsonArray applicationCodes = gson.fromJson(req.getReader(), JsonArray.class); + for (int i = 0; i < applicationCodes.size(); i++) { + String applicationCode = applicationCodes.get(i).getAsString(); + int applicationId = applicationIDService.getOrCreate(applicationCode); + JsonObject mapping = new JsonObject(); + mapping.addProperty(APPLICATION_CODE, applicationCode); + mapping.addProperty(APPLICATION_ID, applicationId); + responseArray.add(mapping); + } + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + return responseArray; + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/handler/InstanceDiscoveryServletHandler.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/handler/InstanceDiscoveryServletHandler.java new file mode 100644 index 000000000..ffddfb94d --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/handler/InstanceDiscoveryServletHandler.java @@ -0,0 +1,53 @@ +package org.skywalking.apm.collector.agentregister.jetty.handler; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import java.io.IOException; +import javax.servlet.http.HttpServletRequest; +import org.skywalking.apm.collector.agentregister.instance.InstanceIDService; +import org.skywalking.apm.collector.server.jetty.ArgumentsParseException; +import org.skywalking.apm.collector.server.jetty.JettyHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class InstanceDiscoveryServletHandler extends JettyHandler { + + private final Logger logger = LoggerFactory.getLogger(InstanceDiscoveryServletHandler.class); + + private InstanceIDService instanceIDService = new InstanceIDService(); + private Gson gson = new Gson(); + + private static final String APPLICATION_ID = "ai"; + private static final String AGENT_UUID = "au"; + private static final String REGISTER_TIME = "rt"; + private static final String INSTANCE_ID = "ii"; + + @Override public String pathSpec() { + return "/instance/register"; + } + + @Override protected JsonElement doGet(HttpServletRequest req) throws ArgumentsParseException { + throw new UnsupportedOperationException(); + } + + @Override protected JsonElement doPost(HttpServletRequest req) throws ArgumentsParseException { + JsonObject responseJson = new JsonObject(); + try { + JsonObject instance = gson.fromJson(req.getReader(), JsonObject.class); + int applicationId = instance.get(APPLICATION_ID).getAsInt(); + String agentUUID = instance.get(AGENT_UUID).getAsString(); + long registerTime = instance.get(REGISTER_TIME).getAsLong(); + + int instanceId = instanceIDService.getOrCreate(applicationId, agentUUID, registerTime); + responseJson.addProperty(APPLICATION_ID, applicationId); + responseJson.addProperty(INSTANCE_ID, instanceId); + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + return responseJson; + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/module.define b/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/module.define index 70f99f0a2..e826d4160 100644 --- a/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/module.define +++ b/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/module.define @@ -1 +1,2 @@ -org.skywalking.apm.collector.agentregister.grpc.AgentRegisterGRPCModuleDefine \ No newline at end of file +org.skywalking.apm.collector.agentregister.grpc.AgentRegisterGRPCModuleDefine +org.skywalking.apm.collector.agentregister.jetty.AgentRegisterJettyModuleDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamGRPCServerHandler.java b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamGRPCServerHandler.java index db9b9362a..f1cd49461 100644 --- a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamGRPCServerHandler.java +++ b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamGRPCServerHandler.java @@ -25,13 +25,11 @@ public class AgentStreamGRPCServerHandler extends JettyHandler { ClusterModuleRegistrationReader reader = ((ClusterModuleContext)CollectorContextHelper.INSTANCE.getContext(ClusterModuleGroupDefine.GROUP_NAME)).getReader(); List servers = reader.read(AgentStreamGRPCDataListener.PATH); JsonArray serverArray = new JsonArray(); - servers.forEach(server -> { - serverArray.add(server); - }); + servers.forEach(server -> serverArray.add(server)); return serverArray; } - @Override protected void doPost(HttpServletRequest req) throws ArgumentsParseException { + @Override protected JsonElement doPost(HttpServletRequest req) throws ArgumentsParseException { throw new UnsupportedOperationException(); } } diff --git a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamJettyServerHandler.java b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamJettyServerHandler.java index a463de01d..bfcecdbf7 100644 --- a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamJettyServerHandler.java +++ b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamJettyServerHandler.java @@ -31,7 +31,7 @@ public class AgentStreamJettyServerHandler extends JettyHandler { return serverArray; } - @Override protected void doPost(HttpServletRequest req) throws ArgumentsParseException { + @Override protected JsonElement doPost(HttpServletRequest req) throws ArgumentsParseException { throw new UnsupportedOperationException(); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyModuleDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyModuleDefine.java index d20996ebd..87428c339 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyModuleDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyModuleDefine.java @@ -1,8 +1,10 @@ package org.skywalking.apm.collector.agentstream.jetty; +import java.util.LinkedList; import java.util.List; import org.skywalking.apm.collector.agentstream.AgentStreamModuleDefine; import org.skywalking.apm.collector.agentstream.AgentStreamModuleGroupDefine; +import org.skywalking.apm.collector.agentstream.jetty.handler.TraceSegmentServletHandler; import org.skywalking.apm.collector.core.cluster.ClusterDataListener; import org.skywalking.apm.collector.core.framework.Handler; import org.skywalking.apm.collector.core.module.ModuleConfigParser; @@ -42,6 +44,8 @@ public class AgentStreamJettyModuleDefine extends AgentStreamModuleDefine { } @Override public List handlerList() { - return null; + List handlers = new LinkedList<>(); + handlers.add(new TraceSegmentServletHandler()); + return handlers; } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/TraceSegmentServiceHandler.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/TraceSegmentServiceHandler.java deleted file mode 100644 index a92102319..000000000 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/TraceSegmentServiceHandler.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.skywalking.apm.collector.agentstream.jetty.handler; - -import com.google.gson.JsonElement; -import java.io.BufferedReader; -import java.io.IOException; -import javax.servlet.http.HttpServletRequest; -import org.skywalking.apm.collector.server.jetty.ArgumentsParseException; -import org.skywalking.apm.collector.server.jetty.JettyHandler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author pengys5 - */ -public class TraceSegmentServiceHandler extends JettyHandler { - - private final Logger logger = LoggerFactory.getLogger(TraceSegmentServiceHandler.class); - - @Override public String pathSpec() { - return null; - } - - @Override protected JsonElement doGet(HttpServletRequest req) throws ArgumentsParseException { - throw new UnsupportedOperationException(); - } - - @Override protected void doPost(HttpServletRequest req) throws ArgumentsParseException { - try { - BufferedReader reader = req.getReader(); - } catch (IOException e) { - logger.error(e.getMessage(), e); - } - } -} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/TraceSegmentServletHandler.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/TraceSegmentServletHandler.java new file mode 100644 index 000000000..7622aac48 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/TraceSegmentServletHandler.java @@ -0,0 +1,54 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler; + +import com.google.gson.JsonElement; +import com.google.gson.stream.JsonReader; +import java.io.BufferedReader; +import java.io.IOException; +import javax.servlet.http.HttpServletRequest; +import org.skywalking.apm.collector.agentstream.jetty.handler.reader.TraceSegment; +import org.skywalking.apm.collector.agentstream.jetty.handler.reader.TraceSegmentJsonReader; +import org.skywalking.apm.collector.agentstream.worker.segment.SegmentParse; +import org.skywalking.apm.collector.server.jetty.ArgumentsParseException; +import org.skywalking.apm.collector.server.jetty.JettyHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class TraceSegmentServletHandler extends JettyHandler { + + private final Logger logger = LoggerFactory.getLogger(TraceSegmentServletHandler.class); + + @Override public String pathSpec() { + return "/segments"; + } + + @Override protected JsonElement doGet(HttpServletRequest req) throws ArgumentsParseException { + throw new UnsupportedOperationException(); + } + + @Override protected JsonElement doPost(HttpServletRequest req) throws ArgumentsParseException { + logger.debug("receive stream segment"); + try { + BufferedReader bufferedReader = req.getReader(); + read(bufferedReader); + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + return null; + } + + private TraceSegmentJsonReader jsonReader = new TraceSegmentJsonReader(); + + private void read(BufferedReader bufferedReader) throws IOException { + JsonReader reader = new JsonReader(bufferedReader); + reader.beginArray(); + while (reader.hasNext()) { + SegmentParse segmentParse = new SegmentParse(); + TraceSegment traceSegment = jsonReader.read(reader); + segmentParse.parse(traceSegment.getGlobalTraceIds(), traceSegment.getTraceSegmentObject()); + } + reader.endArray(); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/KeyWithStringValueJsonReader.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/KeyWithStringValueJsonReader.java new file mode 100644 index 000000000..f0f1bb144 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/KeyWithStringValueJsonReader.java @@ -0,0 +1,36 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler.reader; + +import com.google.gson.stream.JsonReader; +import java.io.IOException; +import org.skywalking.apm.network.proto.KeyWithStringValue; + +/** + * @author pengys5 + */ +public class KeyWithStringValueJsonReader implements StreamJsonReader { + + private static final String KEY = "k"; + private static final String VALUE = "v"; + + @Override public KeyWithStringValue read(JsonReader reader) throws IOException { + KeyWithStringValue.Builder builder = KeyWithStringValue.newBuilder(); + + reader.beginObject(); + while (reader.hasNext()) { + switch (reader.nextName()) { + case KEY: + builder.setKey(reader.nextString()); + break; + case VALUE: + builder.setValue(reader.nextString()); + break; + default: + reader.skipValue(); + break; + } + } + reader.endObject(); + + return builder.build(); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/LogJsonReader.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/LogJsonReader.java new file mode 100644 index 000000000..f8d6526b2 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/LogJsonReader.java @@ -0,0 +1,37 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler.reader; + +import com.google.gson.stream.JsonReader; +import java.io.IOException; +import org.skywalking.apm.network.proto.LogMessage; + +/** + * @author pengys5 + */ +public class LogJsonReader implements StreamJsonReader { + + private KeyWithStringValueJsonReader keyWithStringValueJsonReader = new KeyWithStringValueJsonReader(); + + private static final String TI = "ti"; + private static final String LD = "ld"; + + @Override public LogMessage read(JsonReader reader) throws IOException { + LogMessage.Builder builder = LogMessage.newBuilder(); + + while (reader.hasNext()) { + switch (reader.nextName()) { + case TI: + builder.setTime(reader.nextLong()); + case LD: + reader.beginArray(); + while (reader.hasNext()) { + builder.addData(keyWithStringValueJsonReader.read(reader)); + } + reader.endArray(); + default: + reader.skipValue(); + } + } + + return builder.build(); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/ReferenceJsonReader.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/ReferenceJsonReader.java new file mode 100644 index 000000000..1398c4f53 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/ReferenceJsonReader.java @@ -0,0 +1,70 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler.reader; + +import com.google.gson.stream.JsonReader; +import java.io.IOException; +import org.skywalking.apm.network.proto.TraceSegmentReference; + +/** + * @author pengys5 + */ +public class ReferenceJsonReader implements StreamJsonReader { + + private UniqueIdJsonReader uniqueIdJsonReader = new UniqueIdJsonReader(); + + private static final String TS = "ts"; + private static final String AI = "ai"; + private static final String SI = "si"; + private static final String VI = "vi"; + private static final String VN = "vn"; + private static final String NI = "ni"; + private static final String NN = "nn"; + private static final String EI = "ei"; + private static final String EN = "en"; + private static final String RV = "rv"; + + @Override public TraceSegmentReference read(JsonReader reader) throws IOException { + TraceSegmentReference.Builder builder = TraceSegmentReference.newBuilder(); + + reader.beginObject(); + while (reader.hasNext()) { + switch (reader.nextName()) { + case TS: + builder.setParentTraceSegmentId(uniqueIdJsonReader.read(reader)); + break; + case AI: + builder.setParentApplicationInstanceId(reader.nextInt()); + break; + case SI: + builder.setParentSpanId(reader.nextInt()); + break; + case VI: + builder.setParentServiceId(reader.nextInt()); + break; + case VN: + builder.setParentServiceName(reader.nextString()); + break; + case NI: + builder.setNetworkAddressId(reader.nextInt()); + break; + case NN: + builder.setNetworkAddress(reader.nextString()); + break; + case EI: + builder.setEntryServiceId(reader.nextInt()); + break; + case EN: + builder.setEntryServiceName(reader.nextString()); + break; + case RV: + builder.setRefTypeValue(reader.nextInt()); + break; + default: + reader.skipValue(); + break; + } + } + reader.endObject(); + + return builder.build(); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/SegmentJsonReader.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/SegmentJsonReader.java new file mode 100644 index 000000000..c88a18f4b --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/SegmentJsonReader.java @@ -0,0 +1,69 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler.reader; + +import com.google.gson.stream.JsonReader; +import java.io.IOException; +import org.skywalking.apm.network.proto.TraceSegmentObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class SegmentJsonReader implements StreamJsonReader { + + private final Logger logger = LoggerFactory.getLogger(SegmentJsonReader.class); + + private UniqueIdJsonReader uniqueIdJsonReader = new UniqueIdJsonReader(); + private ReferenceJsonReader referenceJsonReader = new ReferenceJsonReader(); + private SpanJsonReader spanJsonReader = new SpanJsonReader(); + + private static final String TS = "ts"; + private static final String AI = "ai"; + private static final String II = "ii"; + private static final String RS = "rs"; + private static final String SS = "ss"; + + @Override public TraceSegmentObject read(JsonReader reader) throws IOException { + TraceSegmentObject.Builder builder = TraceSegmentObject.newBuilder(); + + reader.beginObject(); + while (reader.hasNext()) { + switch (reader.nextName()) { + case TS: + builder.setTraceSegmentId(uniqueIdJsonReader.read(reader)); + if (logger.isDebugEnabled()) { + StringBuilder segmentId = new StringBuilder(); + builder.getTraceSegmentId().getIdPartsList().forEach(idPart -> segmentId.append(idPart)); + logger.debug("segment id: {}", segmentId); + } + break; + case AI: + builder.setApplicationId(reader.nextInt()); + break; + case II: + builder.setApplicationInstanceId(reader.nextInt()); + break; + case RS: + reader.beginArray(); + while (reader.hasNext()) { + builder.addRefs(referenceJsonReader.read(reader)); + } + reader.endArray(); + break; + case SS: + reader.beginArray(); + while (reader.hasNext()) { + builder.addSpans(spanJsonReader.read(reader)); + } + reader.endArray(); + break; + default: + reader.skipValue(); + break; + } + } + reader.endObject(); + + return builder.build(); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/SpanJsonReader.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/SpanJsonReader.java new file mode 100644 index 000000000..074929525 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/SpanJsonReader.java @@ -0,0 +1,99 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler.reader; + +import com.google.gson.stream.JsonReader; +import java.io.IOException; +import org.skywalking.apm.network.proto.SpanObject; + +/** + * @author pengys5 + */ +public class SpanJsonReader implements StreamJsonReader { + + private KeyWithStringValueJsonReader keyWithStringValueJsonReader = new KeyWithStringValueJsonReader(); + private LogJsonReader logJsonReader = new LogJsonReader(); + + private static final String SI = "si"; + private static final String TV = "tv"; + private static final String LV = "lv"; + private static final String PS = "ps"; + private static final String ST = "st"; + private static final String ET = "et"; + private static final String CI = "ci"; + private static final String CN = "cn"; + private static final String OI = "oi"; + private static final String ON = "on"; + private static final String PI = "pi"; + private static final String PN = "pn"; + private static final String IE = "ie"; + private static final String TO = "to"; + private static final String LO = "lo"; + + @Override public SpanObject read(JsonReader reader) throws IOException { + SpanObject.Builder builder = SpanObject.newBuilder(); + + reader.beginObject(); + while (reader.hasNext()) { + switch (reader.nextName()) { + case SI: + builder.setSpanId(reader.nextInt()); + break; + case TV: + builder.setSpanTypeValue(reader.nextInt()); + break; + case LV: + builder.setSpanLayerValue(reader.nextInt()); + break; + case PS: + builder.setParentSpanId(reader.nextInt()); + break; + case ST: + builder.setStartTime(reader.nextLong()); + break; + case ET: + builder.setEndTime(reader.nextLong()); + break; + case CI: + builder.setComponentId(reader.nextInt()); + break; + case CN: + builder.setComponent(reader.nextString()); + break; + case OI: + builder.setOperationNameId(reader.nextInt()); + break; + case ON: + builder.setOperationName(reader.nextString()); + break; + case PI: + builder.setPeerId(reader.nextInt()); + break; + case PN: + builder.setPeer(reader.nextString()); + break; + case IE: + builder.setIsError(reader.nextBoolean()); + break; + case TO: + reader.beginArray(); + while (reader.hasNext()) { + builder.addTags(keyWithStringValueJsonReader.read(reader)); + } + reader.endArray(); + break; + case LO: + reader.beginArray(); + while (reader.hasNext()) { + builder.addLogs(logJsonReader.read(reader)); + } + reader.endArray(); + break; + default: + reader.skipValue(); + break; + } + } + reader.endObject(); + + return builder.build(); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/StreamJsonReader.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/StreamJsonReader.java new file mode 100644 index 000000000..95095a731 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/StreamJsonReader.java @@ -0,0 +1,11 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler.reader; + +import com.google.gson.stream.JsonReader; +import java.io.IOException; + +/** + * @author pengys5 + */ +public interface StreamJsonReader { + T read(JsonReader reader) throws IOException; +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegment.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegment.java new file mode 100644 index 000000000..fa56b0f04 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegment.java @@ -0,0 +1,35 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler.reader; + +import java.util.ArrayList; +import java.util.List; +import org.skywalking.apm.network.proto.TraceSegmentObject; +import org.skywalking.apm.network.proto.UniqueId; + +/** + * @author pengys5 + */ +public class TraceSegment { + + private List uniqueIds; + private TraceSegmentObject traceSegmentObject; + + public TraceSegment() { + uniqueIds = new ArrayList<>(); + } + + public List getGlobalTraceIds() { + return uniqueIds; + } + + public void addGlobalTraceId(UniqueId globalTraceId) { + uniqueIds.add(globalTraceId); + } + + public TraceSegmentObject getTraceSegmentObject() { + return traceSegmentObject; + } + + public void setTraceSegmentObject(TraceSegmentObject traceSegmentObject) { + this.traceSegmentObject = traceSegmentObject; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegmentJsonReader.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegmentJsonReader.java new file mode 100644 index 000000000..26045e3c9 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegmentJsonReader.java @@ -0,0 +1,54 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler.reader; + +import com.google.gson.stream.JsonReader; +import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class TraceSegmentJsonReader implements StreamJsonReader { + + private final Logger logger = LoggerFactory.getLogger(TraceSegmentJsonReader.class); + + private UniqueIdJsonReader uniqueIdJsonReader = new UniqueIdJsonReader(); + private SegmentJsonReader segmentJsonReader = new SegmentJsonReader(); + + private static final String GT = "gt"; + private static final String SG = "sg"; + + @Override public TraceSegment read(JsonReader reader) throws IOException { + TraceSegment traceSegment = new TraceSegment(); + + reader.beginObject(); + while (reader.hasNext()) { + switch (reader.nextName()) { + case GT: + reader.beginArray(); + while (reader.hasNext()) { + traceSegment.addGlobalTraceId(uniqueIdJsonReader.read(reader)); + } + reader.endArray(); + + if (logger.isDebugEnabled()) { + traceSegment.getGlobalTraceIds().forEach(uniqueId -> { + StringBuilder globalTraceId = new StringBuilder(); + uniqueId.getIdPartsList().forEach(idPart -> globalTraceId.append(idPart)); + logger.debug("global trace id: {}", globalTraceId.toString()); + }); + } + break; + case SG: + traceSegment.setTraceSegmentObject(segmentJsonReader.read(reader)); + break; + default: + reader.skipValue(); + break; + } + } + reader.endObject(); + + return traceSegment; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/UniqueIdJsonReader.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/UniqueIdJsonReader.java new file mode 100644 index 000000000..8df8e1e0b --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/UniqueIdJsonReader.java @@ -0,0 +1,22 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler.reader; + +import com.google.gson.stream.JsonReader; +import java.io.IOException; +import org.skywalking.apm.network.proto.UniqueId; + +/** + * @author pengys5 + */ +public class UniqueIdJsonReader implements StreamJsonReader { + + @Override public UniqueId read(JsonReader reader) throws IOException { + UniqueId.Builder builder = UniqueId.newBuilder(); + + reader.beginArray(); + while (reader.hasNext()) { + builder.addIdParts(reader.nextLong()); + } + reader.endArray(); + return builder.build(); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/Const.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/Const.java index b0d8ec415..df48bd837 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/Const.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/Const.java @@ -8,6 +8,7 @@ public class Const { public static final String IDS_SPLIT = "\\.\\.-\\.\\."; public static final String PEERS_FRONT_SPLIT = "["; public static final String PEERS_BEHIND_SPLIT = "]"; + public static final int USER_ID = 1; public static final String USER_CODE = "User"; public static final String SEGMENT_SPAN_SPLIT = "S"; } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/InstanceCache.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/InstanceCache.java new file mode 100644 index 000000000..54ac23020 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/InstanceCache.java @@ -0,0 +1,25 @@ +package org.skywalking.apm.collector.agentstream.worker.cache; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.skywalking.apm.collector.agentstream.worker.register.instance.dao.IInstanceDAO; +import org.skywalking.apm.collector.storage.dao.DAOContainer; + +/** + * @author pengys5 + */ +public class InstanceCache { + + private static Cache CACHE = CacheBuilder.newBuilder().maximumSize(1000).build(); + + public static int get(int applicationInstanceId) { + try { + return CACHE.get(applicationInstanceId, () -> { + IInstanceDAO dao = (IInstanceDAO)DAOContainer.INSTANCE.get(IInstanceDAO.class.getName()); + return dao.getApplicationId(applicationInstanceId); + }); + } catch (Throwable e) { + return 0; + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java index 4f4691802..6f83321d4 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java @@ -40,7 +40,7 @@ public class NodeComponentSpanListener implements EntrySpanListener, ExitSpanLis peer = spanObject.getPeer(); } - String agg = componentName + Const.ID_SPLIT + peer; + String agg = peer + Const.ID_SPLIT + componentName; nodeComponents.add(agg); } @@ -62,7 +62,7 @@ public class NodeComponentSpanListener implements EntrySpanListener, ExitSpanLis } String peer = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId); - String agg = componentName + Const.ID_SPLIT + peer; + String agg = peer + Const.ID_SPLIT + componentName; nodeComponents.add(agg); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingSpanListener.java index 2c5ebe600..38230c29b 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingSpanListener.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingSpanListener.java @@ -32,11 +32,11 @@ public class NodeMappingSpanListener implements RefsListener, FirstSpanListener String segmentId) { logger.debug("node mapping listener parse reference"); String peers = reference.getNetworkAddress(); - if (reference.getNetworkAddressId() == 0) { + if (reference.getNetworkAddressId() != 0) { peers = ExchangeMarkUtils.INSTANCE.buildMarkedID(reference.getNetworkAddressId()); } - String agg = applicationId + Const.ID_SPLIT + peers; + String agg = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId) + Const.ID_SPLIT + peers; nodeMappings.add(agg); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefSpanListener.java index a57e678b1..ba89e24be 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefSpanListener.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefSpanListener.java @@ -3,6 +3,7 @@ package org.skywalking.apm.collector.agentstream.worker.noderef.reference; import java.util.ArrayList; import java.util.List; import org.skywalking.apm.collector.agentstream.worker.Const; +import org.skywalking.apm.collector.agentstream.worker.cache.InstanceCache; import org.skywalking.apm.collector.agentstream.worker.noderef.reference.define.NodeRefDataDefine; import org.skywalking.apm.collector.agentstream.worker.segment.EntrySpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.ExitSpanListener; @@ -27,27 +28,27 @@ public class NodeRefSpanListener implements EntrySpanListener, ExitSpanListener, private final Logger logger = LoggerFactory.getLogger(NodeRefSpanListener.class); - private List nodeExitReferences = new ArrayList<>(); + private List nodeReferences = new ArrayList<>(); private List nodeEntryReferences = new ArrayList<>(); private long timeBucket; private boolean hasReference = false; @Override public void parseExit(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { - String front = String.valueOf(applicationId); + String front = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId); String behind = spanObject.getPeer(); - if (spanObject.getPeerId() == 0) { + if (spanObject.getPeerId() != 0) { behind = ExchangeMarkUtils.INSTANCE.buildMarkedID(spanObject.getPeerId()); } String agg = front + Const.ID_SPLIT + behind; - nodeExitReferences.add(agg); + nodeReferences.add(agg); } @Override public void parseEntry(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { String behind = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId); - String front = Const.USER_CODE; + String front = ExchangeMarkUtils.INSTANCE.buildMarkedID(Const.USER_ID); String agg = front + Const.ID_SPLIT + behind; nodeEntryReferences.add(agg); } @@ -59,6 +60,14 @@ public class NodeRefSpanListener implements EntrySpanListener, ExitSpanListener, @Override public void parseRef(TraceSegmentReference reference, int applicationId, int applicationInstanceId, String segmentId) { + int parentApplicationId = InstanceCache.get(reference.getParentApplicationInstanceId()); + + String front = ExchangeMarkUtils.INSTANCE.buildMarkedID(parentApplicationId); + String behind = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId); + + String agg = front + Const.ID_SPLIT + behind; + nodeReferences.add(agg); + hasReference = true; } @@ -66,10 +75,10 @@ public class NodeRefSpanListener implements EntrySpanListener, ExitSpanListener, logger.debug("node reference listener build"); StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); if (!hasReference) { - nodeExitReferences.addAll(nodeEntryReferences); + nodeReferences.addAll(nodeEntryReferences); } - for (String agg : nodeExitReferences) { + for (String agg : nodeReferences) { NodeRefDataDefine.NodeReference nodeReference = new NodeRefDataDefine.NodeReference(); nodeReference.setId(timeBucket + Const.ID_SPLIT + agg); nodeReference.setAgg(agg); diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumSpanListener.java index f3251d032..aedfdeceb 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumSpanListener.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumSpanListener.java @@ -3,6 +3,7 @@ package org.skywalking.apm.collector.agentstream.worker.noderef.summary; import java.util.ArrayList; import java.util.List; import org.skywalking.apm.collector.agentstream.worker.Const; +import org.skywalking.apm.collector.agentstream.worker.cache.InstanceCache; import org.skywalking.apm.collector.agentstream.worker.noderef.summary.define.NodeRefSumDataDefine; import org.skywalking.apm.collector.agentstream.worker.segment.EntrySpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.ExitSpanListener; @@ -29,14 +30,18 @@ public class NodeRefSumSpanListener implements EntrySpanListener, ExitSpanListen private List nodeExitReferences = new ArrayList<>(); private List nodeEntryReferences = new ArrayList<>(); + private List nodeReferences = new ArrayList<>(); private long timeBucket; private boolean hasReference = false; + private long startTime; + private long endTime; + private boolean isError; @Override public void parseExit(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { - String front = String.valueOf(applicationId); + String front = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId); String behind = spanObject.getPeer(); - if (spanObject.getPeerId() == 0) { + if (spanObject.getPeerId() != 0) { behind = ExchangeMarkUtils.INSTANCE.buildMarkedID(spanObject.getPeerId()); } @@ -47,8 +52,7 @@ public class NodeRefSumSpanListener implements EntrySpanListener, ExitSpanListen @Override public void parseEntry(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { String behind = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId); - String front = Const.USER_CODE; - + String front = ExchangeMarkUtils.INSTANCE.buildMarkedID(Const.USER_ID); String agg = front + Const.ID_SPLIT + behind; nodeEntryReferences.add(buildNodeRefSum(spanObject.getStartTime(), spanObject.getEndTime(), agg, spanObject.getIsError())); } @@ -77,11 +81,22 @@ public class NodeRefSumSpanListener implements EntrySpanListener, ExitSpanListen @Override public void parseFirst(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { timeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(spanObject.getStartTime()); + startTime = spanObject.getStartTime(); + endTime = spanObject.getEndTime(); + isError = spanObject.getIsError(); } @Override public void parseRef(TraceSegmentReference reference, int applicationId, int applicationInstanceId, String segmentId) { + int parentApplicationId = InstanceCache.get(reference.getParentApplicationInstanceId()); + + String front = ExchangeMarkUtils.INSTANCE.buildMarkedID(parentApplicationId); + String behind = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId); + + String agg = front + Const.ID_SPLIT + behind; + hasReference = true; + nodeReferences.add(agg); } @Override public void build() { @@ -89,6 +104,10 @@ public class NodeRefSumSpanListener implements EntrySpanListener, ExitSpanListen StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); if (!hasReference) { nodeExitReferences.addAll(nodeEntryReferences); + } else { + nodeReferences.forEach(agg -> { + nodeExitReferences.add(buildNodeRefSum(startTime, endTime, agg, isError)); + }); } for (NodeRefSumDataDefine.NodeReferenceSum referenceSum : nodeExitReferences) { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java index 791b3768e..ba2811a42 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java @@ -1,5 +1,6 @@ package org.skywalking.apm.collector.agentstream.worker.register.application; +import org.skywalking.apm.collector.agentstream.worker.Const; import org.skywalking.apm.collector.agentstream.worker.register.IdAutoIncrement; import org.skywalking.apm.collector.agentstream.worker.register.application.dao.IApplicationDAO; import org.skywalking.apm.collector.storage.dao.DAOContainer; @@ -40,8 +41,11 @@ public class ApplicationRegisterSerialWorker extends AbstractLocalAsyncWorker { if (applicationId == 0) { int min = dao.getMinApplicationId(); if (min == 0) { - application.setApplicationId(1); - application.setId("1"); + ApplicationDataDefine.Application userApplication = new ApplicationDataDefine.Application(String.valueOf(Const.USER_ID), Const.USER_CODE, Const.USER_ID); + dao.save(userApplication); + + application.setApplicationId(-1); + application.setId("-1"); } else { int max = dao.getMaxApplicationId(); applicationId = IdAutoIncrement.INSTANCE.increment(min, max); diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/IInstanceDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/IInstanceDAO.java index f0c311146..79d22d27e 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/IInstanceDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/IInstanceDAO.java @@ -15,4 +15,6 @@ public interface IInstanceDAO { void save(InstanceDataDefine.Instance instance); void updateHeartbeatTime(int instanceId, long heartbeatTime); + + int getApplicationId(int applicationInstanceId); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceEsDAO.java index 237f50009..3b56987d9 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceEsDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceEsDAO.java @@ -2,6 +2,7 @@ package org.skywalking.apm.collector.agentstream.worker.register.instance.dao; import java.util.HashMap; import java.util.Map; +import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; @@ -40,8 +41,7 @@ public class InstanceEsDAO extends EsDAO implements IInstanceDAO { SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); if (searchResponse.getHits().totalHits > 0) { SearchHit searchHit = searchResponse.getHits().iterator().next(); - int instanceId = (int)searchHit.getSource().get(InstanceTable.COLUMN_INSTANCE_ID); - return instanceId; + return (int)searchHit.getSource().get(InstanceTable.COLUMN_INSTANCE_ID); } return 0; } @@ -57,7 +57,7 @@ public class InstanceEsDAO extends EsDAO implements IInstanceDAO { @Override public void save(InstanceDataDefine.Instance instance) { logger.debug("save instance register info, application id: {}, agentUUID: {}", instance.getApplicationId(), instance.getAgentUUID()); ElasticSearchClient client = getClient(); - Map source = new HashMap(); + Map source = new HashMap<>(); source.put(InstanceTable.COLUMN_INSTANCE_ID, instance.getInstanceId()); source.put(InstanceTable.COLUMN_APPLICATION_ID, instance.getApplicationId()); source.put(InstanceTable.COLUMN_AGENTUUID, instance.getAgentUUID()); @@ -75,10 +75,19 @@ public class InstanceEsDAO extends EsDAO implements IInstanceDAO { updateRequest.id(String.valueOf(instanceId)); updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - Map source = new HashMap(); + Map source = new HashMap<>(); source.put(InstanceTable.COLUMN_HEARTBEAT_TIME, heartbeatTime); updateRequest.doc(source); client.update(updateRequest); } + + @Override public int getApplicationId(int applicationInstanceId) { + GetResponse response = getClient().prepareGet(InstanceTable.TABLE, String.valueOf(applicationInstanceId)).get(); + if (response.isExists()) { + return (int)response.getSource().get(InstanceTable.COLUMN_APPLICATION_ID); + } else { + return 0; + } + } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceH2DAO.java index 64070090e..d46aaaaec 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceH2DAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceH2DAO.java @@ -26,4 +26,8 @@ public class InstanceH2DAO extends H2DAO implements IInstanceDAO { @Override public void updateHeartbeatTime(int instanceId, long heartbeatTime) { } + + @Override public int getApplicationId(int applicationInstanceId) { + return 0; + } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java index aeb38e243..6d17a4dfc 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java @@ -30,19 +30,25 @@ public class PersistenceTimer implements Starter { } private void extractDataAndSave() { - List workers = PersistenceWorkerContainer.INSTANCE.getPersistenceWorkers(); - List batchAllCollection = new ArrayList<>(); - workers.forEach((PersistenceWorker worker) -> { - try { - worker.allocateJob(new FlushAndSwitch()); - List batchCollection = worker.buildBatchCollection(); - batchAllCollection.addAll(batchCollection); - } catch (WorkerException e) { - logger.error(e.getMessage(), e); - } - }); + try { + List workers = PersistenceWorkerContainer.INSTANCE.getPersistenceWorkers(); + List batchAllCollection = new ArrayList<>(); + workers.forEach((PersistenceWorker worker) -> { + try { + worker.allocateJob(new FlushAndSwitch()); + List batchCollection = worker.buildBatchCollection(); + batchAllCollection.addAll(batchCollection); + } catch (WorkerException e) { + logger.error(e.getMessage(), e); + } + }); - IBatchDAO dao = (IBatchDAO)DAOContainer.INSTANCE.get(IBatchDAO.class.getName()); - dao.batchPersistence(batchAllCollection); + IBatchDAO dao = (IBatchDAO)DAOContainer.INSTANCE.get(IBatchDAO.class.getName()); + dao.batchPersistence(batchAllCollection); + } catch (Throwable e) { + logger.error(e.getMessage(), e); + } finally { + logger.debug("persistence data save finish"); + } } } diff --git a/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/HttpClientTools.java b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/HttpClientTools.java new file mode 100644 index 000000000..ccaf9184c --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/HttpClientTools.java @@ -0,0 +1,77 @@ +package org.skywalking.apm.collector.agentstream; + +import java.io.IOException; +import java.net.URI; +import java.util.List; +import org.apache.http.Consts; +import org.apache.http.HttpEntity; +import org.apache.http.NameValuePair; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public enum HttpClientTools { + INSTANCE; + + private final Logger logger = LoggerFactory.getLogger(HttpClientTools.class); + + public String get(String url, List params) throws IOException { + CloseableHttpClient httpClient = HttpClients.createDefault(); + try { + HttpGet httpget = new HttpGet(url); + String paramStr = EntityUtils.toString(new UrlEncodedFormEntity(params)); + httpget.setURI(new URI(httpget.getURI().toString() + "?" + paramStr)); + logger.debug("executing get request {}", httpget.getURI()); + + try (CloseableHttpResponse response = httpClient.execute(httpget)) { + HttpEntity entity = response.getEntity(); + if (entity != null) { + return EntityUtils.toString(entity); + } + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } finally { + try { + httpClient.close(); + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + } + return null; + } + + public String post(String url, String data) throws IOException { + CloseableHttpClient httpClient = HttpClients.createDefault(); + try { + HttpPost httppost = new HttpPost(url); + httppost.setEntity(new StringEntity(data, Consts.UTF_8)); + logger.debug("executing post request {}", httppost.getURI()); + try (CloseableHttpResponse response = httpClient.execute(httppost)) { + HttpEntity entity = response.getEntity(); + if (entity != null) { + return EntityUtils.toString(entity); + } + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } finally { + try { + httpClient.close(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + return null; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegmentJsonReaderTestCase.java b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegmentJsonReaderTestCase.java new file mode 100644 index 000000000..5e42bec96 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegmentJsonReaderTestCase.java @@ -0,0 +1,28 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler.reader; + +import com.google.gson.JsonElement; +import com.google.gson.stream.JsonReader; +import java.io.IOException; +import java.io.StringReader; +import org.junit.Test; +import org.skywalking.apm.collector.agentstream.mock.JsonFileReader; + +/** + * @author pengys5 + */ +public class TraceSegmentJsonReaderTestCase { + + @Test + public void testRead() throws IOException { + TraceSegmentJsonReader reader = new TraceSegmentJsonReader(); + JsonElement jsonElement = JsonFileReader.INSTANCE.read("json/segment/normal/dubbox-consumer.json"); + System.out.println(jsonElement.toString()); + + JsonReader jsonReader = new JsonReader(new StringReader(jsonElement.toString())); + jsonReader.beginArray(); + while (jsonReader.hasNext()) { + reader.read(jsonReader); + } + jsonReader.endArray(); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/mock/JsonFileReader.java b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/mock/JsonFileReader.java new file mode 100644 index 000000000..b22c8edfc --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/mock/JsonFileReader.java @@ -0,0 +1,24 @@ +package org.skywalking.apm.collector.agentstream.mock; + +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; +import java.io.FileNotFoundException; +import java.io.FileReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public enum JsonFileReader { + INSTANCE; + + private final Logger logger = LoggerFactory.getLogger(JsonFileReader.class); + + public JsonElement read(String fileName) throws FileNotFoundException { + String path = this.getClass().getClassLoader().getResource(fileName).getFile(); + logger.debug("path: {}", path); + JsonParser jsonParser = new JsonParser(); + return jsonParser.parse(new FileReader(path)); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/mock/SegmentPost.java b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/mock/SegmentPost.java new file mode 100644 index 000000000..80228cd16 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/mock/SegmentPost.java @@ -0,0 +1,39 @@ +package org.skywalking.apm.collector.agentstream.mock; + +import com.google.gson.JsonElement; +import java.io.IOException; +import org.skywalking.apm.collector.agentstream.HttpClientTools; +import org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceDataDefine; +import org.skywalking.apm.collector.agentstream.worker.register.instance.dao.InstanceEsDAO; +import org.skywalking.apm.collector.client.elasticsearch.ElasticSearchClient; +import org.skywalking.apm.collector.core.CollectorException; + +/** + * @author pengys5 + */ +public class SegmentPost { + + // @Test + public void test() throws IOException, InterruptedException, CollectorException { + ElasticSearchClient client = new ElasticSearchClient("CollectorDBCluster", true, "127.0.0.1:9300"); + client.initialize(); + + InstanceEsDAO instanceEsDAO = new InstanceEsDAO(); + instanceEsDAO.setClient(client); + + InstanceDataDefine.Instance consumerInstance = new InstanceDataDefine.Instance("2", 2, "dubbox-consumer", 1501858094526L, 2); + instanceEsDAO.save(consumerInstance); + InstanceDataDefine.Instance providerInstance = new InstanceDataDefine.Instance("3", 3, "dubbox-provider", 1501858094526L, 3); + instanceEsDAO.save(providerInstance); + + JsonElement consumer = JsonFileReader.INSTANCE.read("json/segment/normal/dubbox-consumer.json"); + HttpClientTools.INSTANCE.post("http://localhost:12800/segments", consumer.toString()); + + Thread.sleep(5000); + + JsonElement provider = JsonFileReader.INSTANCE.read("json/segment/normal/dubbox-provider.json"); + HttpClientTools.INSTANCE.post("http://localhost:12800/segments", provider.toString()); + + Thread.sleep(5000); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/application-register.json b/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/application-register.json new file mode 100644 index 000000000..eb829c2cb --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/application-register.json @@ -0,0 +1,4 @@ +[ + "dubbox-consumer", + "dubbox-provider" +] \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/dubbox-consumer.json b/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/dubbox-consumer.json new file mode 100644 index 000000000..57c79c9c3 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/dubbox-consumer.json @@ -0,0 +1,71 @@ +[ + { + "gt": [ + [ + 230150, + 185809, + 24040000 + ] + ], + "sg": { + "ts": [ + 230150, + 185809, + 24040000 + ], + "ai": 2, + "ii": 2, + "rs": [], + "ss": [ + { + "si": 1, + "tv": 1, + "lv": 1, + "ps": 0, + "st": 1501858094526, + "et": 1501858097004, + "ci": 3, + "cn": "", + "oi": 0, + "on": "org.skywaking.apm.testcase.dubbo.services.GreetService.doBusiness()", + "pi": 0, + "pn": "172.25.0.4:20880", + "ie": false, + "to": [ + { + "k": "url", + "v": "rest://172.25.0.4:20880/org.skywaking.apm.testcase.dubbo.services.GreetService.doBusiness()" + } + ], + "lo": [] + }, + { + "si": 0, + "tv": 0, + "lv": 2, + "ps": -1, + "st": 1501858092409, + "et": 1501858097033, + "ci": 1, + "cn": "", + "oi": 0, + "on": "/dubbox-case/case/dubbox-rest", + "pi": 0, + "pn": "", + "ie": false, + "to": [ + { + "k": "url", + "v": "http://localhost:18080/dubbox-case/case/dubbox-rest" + }, + { + "k": "http.method", + "v": "GET" + } + ], + "lo": [] + } + ] + } + } +] \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/dubbox-provider.json b/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/dubbox-provider.json new file mode 100644 index 000000000..0e47af810 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/dubbox-provider.json @@ -0,0 +1,66 @@ +[ + { + "gt": [ + [ + 230150, + 185809, + 24040000 + ] + ], + "sg": { + "ts": [ + 137150, + 185809, + 48780000 + ], + "ai": 3, + "ii": 3, + "rs": [ + { + "ts": [ + 230150, + 185809, + 24040000 + ], + "ai": 2, + "si": 1, + "vi": 0, + "vn": "org.skywaking.apm.testcase.dubbo.services.GreetService.doBusiness()", + "ni": 0, + "nn": "172.25.0.4:20880", + "ei": 0, + "en": "/dubbox-case/case/dubbox-rest", + "rn": 0 + } + ], + "ss": [ + { + "si": 0, + "tv": 0, + "lv": 1, + "ps": -1, + "st": 1501858094883, + "et": 1501858096950, + "ci": 3, + "cn": "", + "oi": 0, + "on": "org.skywaking.apm.testcase.dubbo.services.GreetService.doBusiness()", + "pi": 0, + "pn": "", + "ie": false, + "to": [ + { + "k": "url", + "v": "rest://172.25.0.4:20880/org.skywaking.apm.testcase.dubbo.services.GreetService.doBusiness()" + }, + { + "k": "http.method", + "v": "GET" + } + ], + "lo": [] + } + ] + } + } +] \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/instance-register.json b/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/instance-register.json new file mode 100644 index 000000000..f29ab121d --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/instance-register.json @@ -0,0 +1,5 @@ +{ + "ai": 0, + "au": "dubbox-consumer", + "rt": 1501858094526 +} \ No newline at end of file diff --git a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java index 1e4560af3..c2362ff5e 100644 --- a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java +++ b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java @@ -112,7 +112,6 @@ public class ElasticSearchClient implements Client { } public IndexRequestBuilder prepareIndex(String indexName, String id) { - client.prepareUpdate(); return client.prepareIndex(indexName, "type", id); } diff --git a/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/jetty/JettyHandler.java b/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/jetty/JettyHandler.java index 7921df426..22e21afba 100644 --- a/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/jetty/JettyHandler.java +++ b/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/jetty/JettyHandler.java @@ -13,6 +13,7 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.skywalking.apm.collector.core.framework.Handler; +import org.skywalking.apm.collector.core.util.ObjectUtils; /** * @author pengys5 @@ -35,14 +36,13 @@ public abstract class JettyHandler extends HttpServlet implements Handler { @Override protected final void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { try { - doPost(req); - reply(resp); + reply(resp, doPost(req)); } catch (ArgumentsParseException e) { replyError(resp, e.getMessage(), HttpServletResponse.SC_BAD_REQUEST); } } - protected abstract void doPost(HttpServletRequest req) throws ArgumentsParseException; + protected abstract JsonElement doPost(HttpServletRequest req) throws ArgumentsParseException; @Override protected final void doHead(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { @@ -136,17 +136,9 @@ public abstract class JettyHandler extends HttpServlet implements Handler { response.setStatus(HttpServletResponse.SC_OK); PrintWriter out = response.getWriter(); - out.print(resJson); - out.flush(); - out.close(); - } - - private void reply(HttpServletResponse response) throws IOException { - response.setContentType("text/json"); - response.setCharacterEncoding("utf-8"); - response.setStatus(HttpServletResponse.SC_OK); - - PrintWriter out = response.getWriter(); + if (ObjectUtils.isNotEmpty(resJson)) { + out.print(resJson); + } out.flush(); out.close(); } @@ -161,4 +153,4 @@ public abstract class JettyHandler extends HttpServlet implements Handler { out.flush(); out.close(); } -} +} \ No newline at end of file diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/BatchEsDAO.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/BatchEsDAO.java index 74c06e634..5a222145a 100644 --- a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/BatchEsDAO.java +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/BatchEsDAO.java @@ -4,6 +4,7 @@ import java.util.List; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.skywalking.apm.collector.core.util.CollectionUtils; import org.skywalking.apm.collector.storage.dao.IBatchDAO; import org.slf4j.Logger; @@ -19,11 +20,16 @@ public class BatchEsDAO extends EsDAO implements IBatchDAO { @Override public void batchPersistence(List batchCollection) { BulkRequestBuilder bulkRequest = getClient().prepareBulk(); - logger.info("bulk data size: {}", batchCollection.size()); + logger.debug("bulk data size: {}", batchCollection.size()); if (CollectionUtils.isNotEmpty(batchCollection)) { for (int i = 0; i < batchCollection.size(); i++) { - IndexRequestBuilder builder = (IndexRequestBuilder)batchCollection.get(i); - bulkRequest.add(builder); + Object builder = batchCollection.get(i); + if (builder instanceof IndexRequestBuilder) { + bulkRequest.add((IndexRequestBuilder)builder); + } + if (builder instanceof UpdateRequestBuilder) { + bulkRequest.add((UpdateRequestBuilder)builder); + } } BulkResponse bulkResponse = bulkRequest.execute().actionGet(); diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SegmentTopGetHandler.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SegmentTopGetHandler.java index 195638203..003d37e55 100644 --- a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SegmentTopGetHandler.java +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SegmentTopGetHandler.java @@ -80,7 +80,7 @@ public class SegmentTopGetHandler extends JettyHandler { return service.loadTop(startTime, endTime, minCost, maxCost, operationName, globalTraceId, limit, from); } - @Override protected void doPost(HttpServletRequest req) throws ArgumentsParseException { + @Override protected JsonElement doPost(HttpServletRequest req) throws ArgumentsParseException { throw new UnsupportedOperationException(); } } diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SpanGetHandler.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SpanGetHandler.java index db1fa3a64..241e5bea5 100644 --- a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SpanGetHandler.java +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SpanGetHandler.java @@ -36,7 +36,7 @@ public class SpanGetHandler extends JettyHandler { return service.load(segmentId, spanId); } - @Override protected void doPost(HttpServletRequest req) throws ArgumentsParseException { + @Override protected JsonElement doPost(HttpServletRequest req) throws ArgumentsParseException { throw new UnsupportedOperationException(); } } diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceDagGetHandler.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceDagGetHandler.java index 494b004ae..a11540d83 100644 --- a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceDagGetHandler.java +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceDagGetHandler.java @@ -32,7 +32,7 @@ public class TraceDagGetHandler extends JettyHandler { return service.load(startTime, endTime, timeBucketType); } - @Override protected void doPost(HttpServletRequest req) throws ArgumentsParseException { + @Override protected JsonElement doPost(HttpServletRequest req) throws ArgumentsParseException { throw new UnsupportedOperationException(); } } diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceStackGetHandler.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceStackGetHandler.java index c24eb53af..b497e2580 100644 --- a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceStackGetHandler.java +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceStackGetHandler.java @@ -28,7 +28,7 @@ public class TraceStackGetHandler extends JettyHandler { return service.load(globalTraceId); } - @Override protected void doPost(HttpServletRequest req) throws ArgumentsParseException { + @Override protected JsonElement doPost(HttpServletRequest req) throws ArgumentsParseException { throw new UnsupportedOperationException(); } } diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/UIJettyServerHandler.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/UIJettyServerHandler.java index c53e5112f..78b49cd42 100644 --- a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/UIJettyServerHandler.java +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/UIJettyServerHandler.java @@ -31,7 +31,7 @@ public class UIJettyServerHandler extends JettyHandler { return serverArray; } - @Override protected void doPost(HttpServletRequest req) throws ArgumentsParseException { + @Override protected JsonElement doPost(HttpServletRequest req) throws ArgumentsParseException { throw new UnsupportedOperationException(); } } From 6856b896e1b556708a9b88f4a08793531782fc1f Mon Sep 17 00:00:00 2001 From: pengys5 <8082209@qq.com> Date: Sun, 6 Aug 2017 22:15:12 +0800 Subject: [PATCH 5/8] Fixed style check error --- .../agentregister/jetty/AgentRegisterJettyDataListener.java | 1 - 1 file changed, 1 deletion(-) diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyDataListener.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyDataListener.java index 218e0f42d..a15ff860e 100644 --- a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyDataListener.java +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyDataListener.java @@ -1,7 +1,6 @@ package org.skywalking.apm.collector.agentregister.jetty; import org.skywalking.apm.collector.agentregister.AgentRegisterModuleGroupDefine; -import org.skywalking.apm.collector.agentregister.grpc.AgentRegisterGRPCModuleDefine; import org.skywalking.apm.collector.cluster.ClusterModuleDefine; import org.skywalking.apm.collector.core.cluster.ClusterDataListener; From 119d21d5252b466a269dbc793f8b6186d96dc8ce Mon Sep 17 00:00:00 2001 From: pengys5 <8082209@qq.com> Date: Sun, 6 Aug 2017 23:26:17 +0800 Subject: [PATCH 6/8] Service entry test success. --- .../noderef/summary/dao/NodeRefSumEsDAO.java | 14 +-- .../worker/segment/SegmentParse.java | 2 + .../entry/ServiceEntryAggregationWorker.java | 66 +++++++++++ .../entry/ServiceEntryPersistenceWorker.java | 71 ++++++++++++ .../entry/ServiceEntryRemoteWorker.java | 60 ++++++++++ .../entry/ServiceEntrySpanListener.java | 70 ++++++++++++ .../service/entry/dao/IServiceEntryDAO.java | 7 ++ .../service/entry/dao/ServiceEntryEsDAO.java | 50 +++++++++ .../service/entry/dao/ServiceEntryH2DAO.java | 9 ++ .../entry/define/ServiceEntryDataDefine.java | 106 ++++++++++++++++++ .../define/ServiceEntryEsTableDefine.java | 31 +++++ .../define/ServiceEntryH2TableDefine.java | 20 ++++ .../entry/define/ServiceEntryTable.java | 11 ++ .../worker/storage/PersistenceTimer.java | 1 + .../resources/META-INF/defines/es_dao.define | 3 +- .../resources/META-INF/defines/h2_dao.define | 3 +- .../defines/local_worker_provider.define | 3 + .../defines/remote_worker_provider.define | 4 +- .../resources/META-INF/defines/storage.define | 5 +- .../agentstream/mock/SegmentPost.java | 3 +- 20 files changed, 526 insertions(+), 13 deletions(-) create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntryAggregationWorker.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntryPersistenceWorker.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntryRemoteWorker.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntrySpanListener.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/dao/IServiceEntryDAO.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/dao/ServiceEntryEsDAO.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/dao/ServiceEntryH2DAO.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryDataDefine.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryEsTableDefine.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryH2TableDefine.java create mode 100644 apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryTable.java diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumEsDAO.java index c117e70a9..1810412d3 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumEsDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumEsDAO.java @@ -22,13 +22,13 @@ public class NodeRefSumEsDAO extends EsDAO implements INodeRefSumDAO, IPersisten if (getResponse.isExists()) { Data data = dataDefine.build(id); Map source = getResponse.getSource(); - data.setDataLong(0, (Long)source.get(NodeRefSumTable.COLUMN_ONE_SECOND_LESS)); - data.setDataLong(1, (Long)source.get(NodeRefSumTable.COLUMN_THREE_SECOND_LESS)); - data.setDataLong(2, (Long)source.get(NodeRefSumTable.COLUMN_FIVE_SECOND_LESS)); - data.setDataLong(3, (Long)source.get(NodeRefSumTable.COLUMN_FIVE_SECOND_GREATER)); - data.setDataLong(4, (Long)source.get(NodeRefSumTable.COLUMN_ERROR)); - data.setDataLong(5, (Long)source.get(NodeRefSumTable.COLUMN_SUMMARY)); - data.setDataLong(6, (Long)source.get(NodeRefSumTable.COLUMN_TIME_BUCKET)); + data.setDataLong(0, ((Number)source.get(NodeRefSumTable.COLUMN_ONE_SECOND_LESS)).longValue()); + data.setDataLong(1, ((Number)source.get(NodeRefSumTable.COLUMN_THREE_SECOND_LESS)).longValue()); + data.setDataLong(2, ((Number)source.get(NodeRefSumTable.COLUMN_FIVE_SECOND_LESS)).longValue()); + data.setDataLong(3, ((Number)source.get(NodeRefSumTable.COLUMN_FIVE_SECOND_GREATER)).longValue()); + data.setDataLong(4, ((Number)source.get(NodeRefSumTable.COLUMN_ERROR)).longValue()); + data.setDataLong(5, ((Number)source.get(NodeRefSumTable.COLUMN_SUMMARY)).longValue()); + data.setDataLong(6, ((Number)source.get(NodeRefSumTable.COLUMN_TIME_BUCKET)).longValue()); data.setDataString(1, (String)source.get(NodeRefSumTable.COLUMN_AGG)); return data; } else { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java index 6558a2615..0b4d436b9 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java @@ -10,6 +10,7 @@ import org.skywalking.apm.collector.agentstream.worker.noderef.summary.NodeRefSu import org.skywalking.apm.collector.agentstream.worker.segment.cost.SegmentCostSpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.origin.SegmentPersistenceWorker; import org.skywalking.apm.collector.agentstream.worker.segment.origin.define.SegmentDataDefine; +import org.skywalking.apm.collector.agentstream.worker.service.entry.ServiceEntrySpanListener; import org.skywalking.apm.collector.core.framework.CollectorContextHelper; import org.skywalking.apm.collector.core.util.CollectionUtils; import org.skywalking.apm.collector.stream.StreamModuleContext; @@ -41,6 +42,7 @@ public class SegmentParse { spanListeners.add(new NodeRefSumSpanListener()); spanListeners.add(new SegmentCostSpanListener()); spanListeners.add(new GlobalTraceSpanListener()); + spanListeners.add(new ServiceEntrySpanListener()); } public void parse(List traceIds, TraceSegmentObject segmentObject) { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntryAggregationWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntryAggregationWorker.java new file mode 100644 index 000000000..c74fce3f8 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntryAggregationWorker.java @@ -0,0 +1,66 @@ +package org.skywalking.apm.collector.agentstream.worker.service.entry; + +import org.skywalking.apm.collector.agentstream.worker.service.entry.define.ServiceEntryDataDefine; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; +import org.skywalking.apm.collector.stream.worker.WorkerRefs; +import org.skywalking.apm.collector.stream.worker.impl.AggregationWorker; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class ServiceEntryAggregationWorker extends AggregationWorker { + + public ServiceEntryAggregationWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + super.preStart(); + } + + @Override protected WorkerRefs nextWorkRef(String id) throws WorkerNotFoundException { + return getClusterContext().lookup(ServiceEntryRemoteWorker.WorkerRole.INSTANCE); + } + + public static class Factory extends AbstractLocalAsyncWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public ServiceEntryAggregationWorker workerInstance(ClusterWorkerContext clusterContext) { + return new ServiceEntryAggregationWorker(role(), clusterContext); + } + + @Override + public int queueSize() { + return 1024; + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return ServiceEntryAggregationWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new HashCodeSelector(); + } + + @Override public DataDefine dataDefine() { + return new ServiceEntryDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntryPersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntryPersistenceWorker.java new file mode 100644 index 000000000..a5caec87a --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntryPersistenceWorker.java @@ -0,0 +1,71 @@ +package org.skywalking.apm.collector.agentstream.worker.service.entry; + +import org.skywalking.apm.collector.agentstream.worker.service.entry.dao.IServiceEntryDAO; +import org.skywalking.apm.collector.agentstream.worker.service.entry.define.ServiceEntryDataDefine; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class ServiceEntryPersistenceWorker extends PersistenceWorker { + + public ServiceEntryPersistenceWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + super.preStart(); + } + + @Override protected boolean needMergeDBData() { + return true; + } + + @Override protected IPersistenceDAO persistenceDAO() { + return (IPersistenceDAO)DAOContainer.INSTANCE.get(IServiceEntryDAO.class.getName()); + } + + public static class Factory extends AbstractLocalAsyncWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public ServiceEntryPersistenceWorker workerInstance(ClusterWorkerContext clusterContext) { + return new ServiceEntryPersistenceWorker(role(), clusterContext); + } + + @Override + public int queueSize() { + return 1024; + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return ServiceEntryPersistenceWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new HashCodeSelector(); + } + + @Override public DataDefine dataDefine() { + return new ServiceEntryDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntryRemoteWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntryRemoteWorker.java new file mode 100644 index 000000000..3ca0a8195 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntryRemoteWorker.java @@ -0,0 +1,60 @@ +package org.skywalking.apm.collector.agentstream.worker.service.entry; + +import org.skywalking.apm.collector.agentstream.worker.service.entry.define.ServiceEntryDataDefine; +import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorker; +import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerException; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class ServiceEntryRemoteWorker extends AbstractRemoteWorker { + + protected ServiceEntryRemoteWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + + } + + @Override protected void onWork(Object message) throws WorkerException { + getClusterContext().lookup(ServiceEntryPersistenceWorker.WorkerRole.INSTANCE).tell(message); + } + + public static class Factory extends AbstractRemoteWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public ServiceEntryRemoteWorker workerInstance(ClusterWorkerContext clusterContext) { + return new ServiceEntryRemoteWorker(role(), clusterContext); + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return ServiceEntryRemoteWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new HashCodeSelector(); + } + + @Override public DataDefine dataDefine() { + return new ServiceEntryDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntrySpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntrySpanListener.java new file mode 100644 index 000000000..aaeba441b --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntrySpanListener.java @@ -0,0 +1,70 @@ +package org.skywalking.apm.collector.agentstream.worker.service.entry; + +import org.skywalking.apm.collector.agentstream.worker.Const; +import org.skywalking.apm.collector.agentstream.worker.segment.EntrySpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.RefsListener; +import org.skywalking.apm.collector.agentstream.worker.service.entry.define.ServiceEntryDataDefine; +import org.skywalking.apm.collector.agentstream.worker.util.ExchangeMarkUtils; +import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.stream.StreamModuleContext; +import org.skywalking.apm.collector.stream.StreamModuleGroupDefine; +import org.skywalking.apm.collector.stream.worker.WorkerInvokeException; +import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; +import org.skywalking.apm.network.proto.SpanObject; +import org.skywalking.apm.network.proto.TraceSegmentReference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class ServiceEntrySpanListener implements RefsListener, FirstSpanListener, EntrySpanListener { + + private final Logger logger = LoggerFactory.getLogger(ServiceEntrySpanListener.class); + + private long timeBucket; + private boolean hasReference = false; + private String agg; + private int applicationId; + + @Override + public void parseEntry(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + String entryServiceName = spanObject.getOperationName(); + if (spanObject.getOperationNameId() != 0) { + entryServiceName = ExchangeMarkUtils.INSTANCE.buildMarkedID(spanObject.getOperationNameId()); + } + this.agg = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId) + Const.ID_SPLIT + entryServiceName; + this.applicationId = applicationId; + } + + @Override public void parseRef(TraceSegmentReference reference, int applicationId, int applicationInstanceId, + String segmentId) { + hasReference = true; + } + + @Override + public void parseFirst(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + timeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(spanObject.getStartTime()); + } + + @Override public void build() { + logger.debug("entry service listener build"); + StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); + if (!hasReference) { + ServiceEntryDataDefine.ServiceEntry serviceEntry = new ServiceEntryDataDefine.ServiceEntry(); + serviceEntry.setId(timeBucket + Const.ID_SPLIT + agg); + serviceEntry.setApplicationId(applicationId); + serviceEntry.setAgg(agg); + serviceEntry.setTimeBucket(timeBucket); + + try { + logger.debug("send to service entry aggregation worker, id: {}", serviceEntry.getId()); + context.getClusterWorkerContext().lookup(ServiceEntryAggregationWorker.WorkerRole.INSTANCE).tell(serviceEntry.toData()); + } catch (WorkerInvokeException | WorkerNotFoundException e) { + logger.error(e.getMessage(), e); + } + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/dao/IServiceEntryDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/dao/IServiceEntryDAO.java new file mode 100644 index 000000000..fcfe8f572 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/dao/IServiceEntryDAO.java @@ -0,0 +1,7 @@ +package org.skywalking.apm.collector.agentstream.worker.service.entry.dao; + +/** + * @author pengys5 + */ +public interface IServiceEntryDAO { +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/dao/ServiceEntryEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/dao/ServiceEntryEsDAO.java new file mode 100644 index 000000000..b35edd115 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/dao/ServiceEntryEsDAO.java @@ -0,0 +1,50 @@ +package org.skywalking.apm.collector.agentstream.worker.service.entry.dao; + +import java.util.HashMap; +import java.util.Map; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.update.UpdateRequestBuilder; +import org.skywalking.apm.collector.agentstream.worker.service.entry.define.ServiceEntryTable; +import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; + +/** + * @author pengys5 + */ +public class ServiceEntryEsDAO extends EsDAO implements IServiceEntryDAO, IPersistenceDAO { + + @Override public Data get(String id, DataDefine dataDefine) { + GetResponse getResponse = getClient().prepareGet(ServiceEntryTable.TABLE, id).get(); + if (getResponse.isExists()) { + Data data = dataDefine.build(id); + Map source = getResponse.getSource(); + data.setDataInteger(0, (Integer)source.get(ServiceEntryTable.COLUMN_APPLICATION_ID)); + data.setDataString(1, (String)source.get(ServiceEntryTable.COLUMN_AGG)); + data.setDataLong(0, (Long)source.get(ServiceEntryTable.COLUMN_TIME_BUCKET)); + return data; + } else { + return null; + } + } + + @Override public IndexRequestBuilder prepareBatchInsert(Data data) { + Map source = new HashMap<>(); + source.put(ServiceEntryTable.COLUMN_APPLICATION_ID, data.getDataInteger(0)); + source.put(ServiceEntryTable.COLUMN_AGG, data.getDataString(1)); + source.put(ServiceEntryTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + + return getClient().prepareIndex(ServiceEntryTable.TABLE, data.getDataString(0)).setSource(source); + } + + @Override public UpdateRequestBuilder prepareBatchUpdate(Data data) { + Map source = new HashMap<>(); + source.put(ServiceEntryTable.COLUMN_APPLICATION_ID, data.getDataInteger(0)); + source.put(ServiceEntryTable.COLUMN_AGG, data.getDataString(1)); + source.put(ServiceEntryTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + + return getClient().prepareUpdate(ServiceEntryTable.TABLE, data.getDataString(0)).setDoc(source); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/dao/ServiceEntryH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/dao/ServiceEntryH2DAO.java new file mode 100644 index 000000000..fb2cd96c7 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/dao/ServiceEntryH2DAO.java @@ -0,0 +1,9 @@ +package org.skywalking.apm.collector.agentstream.worker.service.entry.dao; + +import org.skywalking.apm.collector.storage.h2.dao.H2DAO; + +/** + * @author pengys5 + */ +public class ServiceEntryH2DAO extends H2DAO implements IServiceEntryDAO { +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryDataDefine.java new file mode 100644 index 000000000..ed044eb5d --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryDataDefine.java @@ -0,0 +1,106 @@ +package org.skywalking.apm.collector.agentstream.worker.service.entry.define; + +import org.skywalking.apm.collector.remote.grpc.proto.RemoteData; +import org.skywalking.apm.collector.stream.worker.impl.data.Attribute; +import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.impl.data.Transform; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation; + +/** + * @author pengys5 + */ +public class ServiceEntryDataDefine extends DataDefine { + + @Override public int defineId() { + return 501; + } + + @Override protected int initialCapacity() { + return 4; + } + + @Override protected void attributeDefine() { + addAttribute(0, new Attribute(ServiceEntryTable.COLUMN_ID, AttributeType.STRING, new NonOperation())); + addAttribute(1, new Attribute(ServiceEntryTable.COLUMN_APPLICATION_ID, AttributeType.INTEGER, new NonOperation())); + addAttribute(2, new Attribute(ServiceEntryTable.COLUMN_AGG, AttributeType.STRING, new CoverOperation())); + addAttribute(3, new Attribute(ServiceEntryTable.COLUMN_TIME_BUCKET, AttributeType.LONG, new CoverOperation())); + } + + @Override public Object deserialize(RemoteData remoteData) { + return null; + } + + @Override public RemoteData serialize(Object object) { + return null; + } + + public static class ServiceEntry implements Transform { + private String id; + private int applicationId; + private String agg; + private long timeBucket; + + ServiceEntry(String id, int applicationId, String agg, long timeBucket) { + this.id = id; + this.applicationId = applicationId; + this.agg = agg; + this.timeBucket = timeBucket; + } + + public ServiceEntry() { + } + + @Override public Data toData() { + ServiceEntryDataDefine define = new ServiceEntryDataDefine(); + Data data = define.build(id); + data.setDataString(0, this.id); + data.setDataInteger(0, this.applicationId); + data.setDataString(1, this.agg); + data.setDataLong(0, this.timeBucket); + return data; + } + + @Override public ServiceEntry toSelf(Data data) { + this.id = data.getDataString(0); + this.applicationId = data.getDataInteger(0); + this.agg = data.getDataString(1); + this.timeBucket = data.getDataLong(0); + return this; + } + + public String getId() { + return id; + } + + public String getAgg() { + return agg; + } + + public long getTimeBucket() { + return timeBucket; + } + + public void setId(String id) { + this.id = id; + } + + public void setAgg(String agg) { + this.agg = agg; + } + + public void setTimeBucket(long timeBucket) { + this.timeBucket = timeBucket; + } + + public int getApplicationId() { + return applicationId; + } + + public void setApplicationId(int applicationId) { + this.applicationId = applicationId; + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryEsTableDefine.java new file mode 100644 index 000000000..54191227f --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryEsTableDefine.java @@ -0,0 +1,31 @@ +package org.skywalking.apm.collector.agentstream.worker.service.entry.define; + +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine; +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine; + +/** + * @author pengys5 + */ +public class ServiceEntryEsTableDefine extends ElasticSearchTableDefine { + + public ServiceEntryEsTableDefine() { + super(ServiceEntryTable.TABLE); + } + + @Override public int refreshInterval() { + return 2; + } + + @Override public int numberOfShards() { + return 2; + } + + @Override public int numberOfReplicas() { + return 0; + } + + @Override public void initialize() { + addColumn(new ElasticSearchColumnDefine(ServiceEntryTable.COLUMN_AGG, ElasticSearchColumnDefine.Type.Keyword.name())); + addColumn(new ElasticSearchColumnDefine(ServiceEntryTable.COLUMN_TIME_BUCKET, ElasticSearchColumnDefine.Type.Long.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryH2TableDefine.java new file mode 100644 index 000000000..e38d6e726 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryH2TableDefine.java @@ -0,0 +1,20 @@ +package org.skywalking.apm.collector.agentstream.worker.service.entry.define; + +import org.skywalking.apm.collector.storage.h2.define.H2ColumnDefine; +import org.skywalking.apm.collector.storage.h2.define.H2TableDefine; + +/** + * @author pengys5 + */ +public class ServiceEntryH2TableDefine extends H2TableDefine { + + public ServiceEntryH2TableDefine() { + super(ServiceEntryTable.TABLE); + } + + @Override public void initialize() { + addColumn(new H2ColumnDefine(ServiceEntryTable.COLUMN_ID, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(ServiceEntryTable.COLUMN_AGG, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(ServiceEntryTable.COLUMN_TIME_BUCKET, H2ColumnDefine.Type.Bigint.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryTable.java new file mode 100644 index 000000000..5ec9957f8 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryTable.java @@ -0,0 +1,11 @@ +package org.skywalking.apm.collector.agentstream.worker.service.entry.define; + +import org.skywalking.apm.collector.agentstream.worker.CommonTable; + +/** + * @author pengys5 + */ +public class ServiceEntryTable extends CommonTable { + public static final String TABLE = "service_entry"; + public static final String COLUMN_APPLICATION_ID = "application_id"; +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java index 6d17a4dfc..92825552d 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java @@ -34,6 +34,7 @@ public class PersistenceTimer implements Starter { List workers = PersistenceWorkerContainer.INSTANCE.getPersistenceWorkers(); List batchAllCollection = new ArrayList<>(); workers.forEach((PersistenceWorker worker) -> { + logger.debug("extract {} worker data and save", worker.getRole().roleName()); try { worker.allocateJob(new FlushAndSwitch()); List batchCollection = worker.buildBatchCollection(); diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define index dd2e4e545..0a9b11a86 100644 --- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define +++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define @@ -7,4 +7,5 @@ org.skywalking.apm.collector.agentstream.worker.noderef.reference.dao.NodeRefere org.skywalking.apm.collector.agentstream.worker.segment.origin.dao.SegmentEsDAO org.skywalking.apm.collector.agentstream.worker.noderef.summary.dao.NodeRefSumEsDAO org.skywalking.apm.collector.agentstream.worker.segment.cost.dao.SegmentCostEsDAO -org.skywalking.apm.collector.agentstream.worker.global.dao.GlobalTraceEsDAO \ No newline at end of file +org.skywalking.apm.collector.agentstream.worker.global.dao.GlobalTraceEsDAO +org.skywalking.apm.collector.agentstream.worker.service.entry.dao.ServiceEntryEsDAO \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define index da2352e37..c02664268 100644 --- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define +++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define @@ -7,4 +7,5 @@ org.skywalking.apm.collector.agentstream.worker.noderef.reference.dao.NodeRefere org.skywalking.apm.collector.agentstream.worker.segment.origin.dao.SegmentH2DAO org.skywalking.apm.collector.agentstream.worker.noderef.summary.dao.NodeRefSumH2DAO org.skywalking.apm.collector.agentstream.worker.segment.cost.dao.SegmentCostH2DAO -org.skywalking.apm.collector.agentstream.worker.global.dao.GlobalTraceH2DAO \ No newline at end of file +org.skywalking.apm.collector.agentstream.worker.global.dao.GlobalTraceH2DAO +org.skywalking.apm.collector.agentstream.worker.service.entry.dao.ServiceEntryH2DAO \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_worker_provider.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_worker_provider.define index fb133fbe0..34e6bdd71 100644 --- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_worker_provider.define +++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_worker_provider.define @@ -10,6 +10,9 @@ org.skywalking.apm.collector.agentstream.worker.noderef.reference.NodeRefPersist org.skywalking.apm.collector.agentstream.worker.noderef.summary.NodeRefSumAggregationWorker$Factory org.skywalking.apm.collector.agentstream.worker.noderef.summary.NodeRefSumPersistenceWorker$Factory +org.skywalking.apm.collector.agentstream.worker.service.entry.ServiceEntryAggregationWorker$Factory +org.skywalking.apm.collector.agentstream.worker.service.entry.ServiceEntryPersistenceWorker$Factory + org.skywalking.apm.collector.agentstream.worker.segment.origin.SegmentPersistenceWorker$Factory org.skywalking.apm.collector.agentstream.worker.segment.cost.SegmentCostPersistenceWorker$Factory org.skywalking.apm.collector.agentstream.worker.global.GlobalTracePersistenceWorker$Factory diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define index b82aef6bd..db8d6bb14 100644 --- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define +++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define @@ -5,4 +5,6 @@ org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceName org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentRemoteWorker$Factory org.skywalking.apm.collector.agentstream.worker.node.mapping.NodeMappingRemoteWorker$Factory org.skywalking.apm.collector.agentstream.worker.noderef.reference.NodeRefRemoteWorker$Factory -org.skywalking.apm.collector.agentstream.worker.noderef.summary.NodeRefSumRemoteWorker$Factory \ No newline at end of file +org.skywalking.apm.collector.agentstream.worker.noderef.summary.NodeRefSumRemoteWorker$Factory + +org.skywalking.apm.collector.agentstream.worker.service.entry.ServiceEntryRemoteWorker$Factory \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define index d046f4613..b5d27b7b1 100644 --- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define +++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define @@ -26,4 +26,7 @@ org.skywalking.apm.collector.agentstream.worker.segment.cost.define.SegmentCostE org.skywalking.apm.collector.agentstream.worker.segment.cost.define.SegmentCostH2TableDefine org.skywalking.apm.collector.agentstream.worker.global.define.GlobalTraceEsTableDefine -org.skywalking.apm.collector.agentstream.worker.global.define.GlobalTraceH2TableDefine \ No newline at end of file +org.skywalking.apm.collector.agentstream.worker.global.define.GlobalTraceH2TableDefine + +org.skywalking.apm.collector.agentstream.worker.service.entry.define.ServiceEntryEsTableDefine +org.skywalking.apm.collector.agentstream.worker.service.entry.define.ServiceEntryH2TableDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/mock/SegmentPost.java b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/mock/SegmentPost.java index 80228cd16..0a16508ab 100644 --- a/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/mock/SegmentPost.java +++ b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/mock/SegmentPost.java @@ -13,8 +13,7 @@ import org.skywalking.apm.collector.core.CollectorException; */ public class SegmentPost { - // @Test - public void test() throws IOException, InterruptedException, CollectorException { + public static void main(String[] args) throws IOException, InterruptedException, CollectorException { ElasticSearchClient client = new ElasticSearchClient("CollectorDBCluster", true, "127.0.0.1:9300"); client.initialize(); From eb72cb04bab47e65113b53c7d212d786081d3e28 Mon Sep 17 00:00:00 2001 From: pengys5 <8082209@qq.com> Date: Tue, 8 Aug 2017 09:41:25 +0800 Subject: [PATCH 7/8] Change the static json key name to full name in json reader. --- .../jetty/handler/reader/LogJsonReader.java | 8 +-- .../handler/reader/ReferenceJsonReader.java | 44 +++++++------- .../handler/reader/SegmentJsonReader.java | 20 +++---- .../jetty/handler/reader/SpanJsonReader.java | 60 +++++++++---------- .../reader/TraceSegmentJsonReader.java | 8 +-- 5 files changed, 70 insertions(+), 70 deletions(-) diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/LogJsonReader.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/LogJsonReader.java index f8d6526b2..78b8a79ee 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/LogJsonReader.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/LogJsonReader.java @@ -11,17 +11,17 @@ public class LogJsonReader implements StreamJsonReader { private KeyWithStringValueJsonReader keyWithStringValueJsonReader = new KeyWithStringValueJsonReader(); - private static final String TI = "ti"; - private static final String LD = "ld"; + private static final String TIME = "ti"; + private static final String LOG_DATA = "ld"; @Override public LogMessage read(JsonReader reader) throws IOException { LogMessage.Builder builder = LogMessage.newBuilder(); while (reader.hasNext()) { switch (reader.nextName()) { - case TI: + case TIME: builder.setTime(reader.nextLong()); - case LD: + case LOG_DATA: reader.beginArray(); while (reader.hasNext()) { builder.addData(keyWithStringValueJsonReader.read(reader)); diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/ReferenceJsonReader.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/ReferenceJsonReader.java index bd979d6be..113edd199 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/ReferenceJsonReader.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/ReferenceJsonReader.java @@ -11,17 +11,17 @@ public class ReferenceJsonReader implements StreamJsonReader { private ReferenceJsonReader referenceJsonReader = new ReferenceJsonReader(); private SpanJsonReader spanJsonReader = new SpanJsonReader(); - private static final String TS = "ts"; - private static final String AI = "ai"; - private static final String II = "ii"; - private static final String RS = "rs"; - private static final String SS = "ss"; + private static final String TRACE_SEGMENT_ID = "ts"; + private static final String APPLICATION_ID = "ai"; + private static final String APPLICATION_INSTANCE_ID = "ii"; + private static final String TRACE_SEGMENT_REFERENCE = "rs"; + private static final String SPANS = "ss"; @Override public TraceSegmentObject read(JsonReader reader) throws IOException { TraceSegmentObject.Builder builder = TraceSegmentObject.newBuilder(); @@ -29,7 +29,7 @@ public class SegmentJsonReader implements StreamJsonReader { reader.beginObject(); while (reader.hasNext()) { switch (reader.nextName()) { - case TS: + case TRACE_SEGMENT_ID: builder.setTraceSegmentId(uniqueIdJsonReader.read(reader)); if (logger.isDebugEnabled()) { StringBuilder segmentId = new StringBuilder(); @@ -37,20 +37,20 @@ public class SegmentJsonReader implements StreamJsonReader { logger.debug("segment id: {}", segmentId); } break; - case AI: + case APPLICATION_ID: builder.setApplicationId(reader.nextInt()); break; - case II: + case APPLICATION_INSTANCE_ID: builder.setApplicationInstanceId(reader.nextInt()); break; - case RS: + case TRACE_SEGMENT_REFERENCE: reader.beginArray(); while (reader.hasNext()) { builder.addRefs(referenceJsonReader.read(reader)); } reader.endArray(); break; - case SS: + case SPANS: reader.beginArray(); while (reader.hasNext()) { builder.addSpans(spanJsonReader.read(reader)); diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/SpanJsonReader.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/SpanJsonReader.java index 074929525..8de743512 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/SpanJsonReader.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/SpanJsonReader.java @@ -12,21 +12,21 @@ public class SpanJsonReader implements StreamJsonReader { private KeyWithStringValueJsonReader keyWithStringValueJsonReader = new KeyWithStringValueJsonReader(); private LogJsonReader logJsonReader = new LogJsonReader(); - private static final String SI = "si"; - private static final String TV = "tv"; - private static final String LV = "lv"; - private static final String PS = "ps"; - private static final String ST = "st"; - private static final String ET = "et"; - private static final String CI = "ci"; - private static final String CN = "cn"; - private static final String OI = "oi"; - private static final String ON = "on"; - private static final String PI = "pi"; - private static final String PN = "pn"; - private static final String IE = "ie"; - private static final String TO = "to"; - private static final String LO = "lo"; + private static final String SPAN_ID = "si"; + private static final String SPAN_TYPE_VALUE = "tv"; + private static final String SPAN_LAYER_VALUE = "lv"; + private static final String PARENT_SPAN_ID = "ps"; + private static final String START_TIME = "st"; + private static final String END_TIME = "et"; + private static final String COMPONENT_ID = "ci"; + private static final String COMPONENT_NAME = "cn"; + private static final String OPERATION_NAME_ID = "oi"; + private static final String OPERATION_NAME = "on"; + private static final String PEER_ID = "pi"; + private static final String PEER = "pn"; + private static final String IS_ERROR = "ie"; + private static final String TAGS = "to"; + private static final String LOGS = "lo"; @Override public SpanObject read(JsonReader reader) throws IOException { SpanObject.Builder builder = SpanObject.newBuilder(); @@ -34,53 +34,53 @@ public class SpanJsonReader implements StreamJsonReader { reader.beginObject(); while (reader.hasNext()) { switch (reader.nextName()) { - case SI: + case SPAN_ID: builder.setSpanId(reader.nextInt()); break; - case TV: + case SPAN_TYPE_VALUE: builder.setSpanTypeValue(reader.nextInt()); break; - case LV: + case SPAN_LAYER_VALUE: builder.setSpanLayerValue(reader.nextInt()); break; - case PS: + case PARENT_SPAN_ID: builder.setParentSpanId(reader.nextInt()); break; - case ST: + case START_TIME: builder.setStartTime(reader.nextLong()); break; - case ET: + case END_TIME: builder.setEndTime(reader.nextLong()); break; - case CI: + case COMPONENT_ID: builder.setComponentId(reader.nextInt()); break; - case CN: + case COMPONENT_NAME: builder.setComponent(reader.nextString()); break; - case OI: + case OPERATION_NAME_ID: builder.setOperationNameId(reader.nextInt()); break; - case ON: + case OPERATION_NAME: builder.setOperationName(reader.nextString()); break; - case PI: + case PEER_ID: builder.setPeerId(reader.nextInt()); break; - case PN: + case PEER: builder.setPeer(reader.nextString()); break; - case IE: + case IS_ERROR: builder.setIsError(reader.nextBoolean()); break; - case TO: + case TAGS: reader.beginArray(); while (reader.hasNext()) { builder.addTags(keyWithStringValueJsonReader.read(reader)); } reader.endArray(); break; - case LO: + case LOGS: reader.beginArray(); while (reader.hasNext()) { builder.addLogs(logJsonReader.read(reader)); diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegmentJsonReader.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegmentJsonReader.java index 26045e3c9..d00883b53 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegmentJsonReader.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegmentJsonReader.java @@ -15,8 +15,8 @@ public class TraceSegmentJsonReader implements StreamJsonReader { private UniqueIdJsonReader uniqueIdJsonReader = new UniqueIdJsonReader(); private SegmentJsonReader segmentJsonReader = new SegmentJsonReader(); - private static final String GT = "gt"; - private static final String SG = "sg"; + private static final String GLOBAL_TRACE_IDS = "gt"; + private static final String SEGMENT = "sg"; @Override public TraceSegment read(JsonReader reader) throws IOException { TraceSegment traceSegment = new TraceSegment(); @@ -24,7 +24,7 @@ public class TraceSegmentJsonReader implements StreamJsonReader { reader.beginObject(); while (reader.hasNext()) { switch (reader.nextName()) { - case GT: + case GLOBAL_TRACE_IDS: reader.beginArray(); while (reader.hasNext()) { traceSegment.addGlobalTraceId(uniqueIdJsonReader.read(reader)); @@ -39,7 +39,7 @@ public class TraceSegmentJsonReader implements StreamJsonReader { }); } break; - case SG: + case SEGMENT: traceSegment.setTraceSegmentObject(segmentJsonReader.read(reader)); break; default: From a7c11a1baf9a2bd0e25295d12cedac872bd2a0d0 Mon Sep 17 00:00:00 2001 From: pengys5 <8082209@qq.com> Date: Tue, 8 Aug 2017 10:03:34 +0800 Subject: [PATCH 8/8] delete the define id in every data define. #336 #338 --- .../worker/global/define/GlobalTraceDataDefine.java | 4 ---- .../component/define/NodeComponentDataDefine.java | 4 ---- .../node/mapping/define/NodeMappingDataDefine.java | 4 ---- .../noderef/reference/define/NodeRefDataDefine.java | 4 ---- .../summary/define/NodeRefSumDataDefine.java | 6 ------ .../register/application/ApplicationDataDefine.java | 8 +------- .../register/application/ApplicationTable.java | 4 +++- .../register/instance/InstanceDataDefine.java | 10 ++-------- .../register/instance/InstanceEsTableDefine.java | 2 +- .../register/instance/InstanceH2TableDefine.java | 2 +- .../worker/register/instance/InstanceTable.java | 6 ++++-- .../worker/register/instance/dao/InstanceEsDAO.java | 4 ++-- .../register/servicename/ServiceNameDataDefine.java | 8 +------- .../register/servicename/ServiceNameTable.java | 4 +++- .../segment/cost/define/SegmentCostDataDefine.java | 4 ---- .../segment/origin/define/SegmentDataDefine.java | 6 ------ .../entry/define/ServiceEntryDataDefine.java | 4 ---- .../reference/define/ServiceRefDataDefine.java | 6 ------ .../core/cluster/ClusterDefinitionFile.java | 13 ------------- .../apm/collector/stream/worker/impl/data/Data.java | 8 +------- .../stream/worker/impl/data/DataDefine.java | 4 +--- 21 files changed, 20 insertions(+), 95 deletions(-) delete mode 100644 apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterDefinitionFile.java diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/define/GlobalTraceDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/define/GlobalTraceDataDefine.java index 16ba94c1e..9d851d5e5 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/define/GlobalTraceDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/define/GlobalTraceDataDefine.java @@ -14,10 +14,6 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class GlobalTraceDataDefine extends DataDefine { - @Override public int defineId() { - return 403; - } - @Override protected int initialCapacity() { return 4; } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java index 0f08484ce..e617a9679 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java @@ -14,10 +14,6 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class NodeComponentDataDefine extends DataDefine { - @Override public int defineId() { - return 101; - } - @Override protected int initialCapacity() { return 3; } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingDataDefine.java index e227b291e..881a0f163 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingDataDefine.java @@ -14,10 +14,6 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class NodeMappingDataDefine extends DataDefine { - @Override public int defineId() { - return 102; - } - @Override protected int initialCapacity() { return 3; } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefDataDefine.java index 079062861..7ec0fb367 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefDataDefine.java @@ -14,10 +14,6 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class NodeRefDataDefine extends DataDefine { - @Override public int defineId() { - return 201; - } - @Override protected int initialCapacity() { return 3; } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumDataDefine.java index 39146ea63..192c6630e 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumDataDefine.java @@ -14,12 +14,6 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class NodeRefSumDataDefine extends DataDefine { - public static final int DEFINE_ID = 202; - - @Override public int defineId() { - return DEFINE_ID; - } - @Override protected int initialCapacity() { return 9; } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationDataDefine.java index cb964a3a4..3aebfe46a 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationDataDefine.java @@ -12,18 +12,12 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class ApplicationDataDefine extends DataDefine { - public static final int DEFINE_ID = 101; - - @Override public int defineId() { - return DEFINE_ID; - } - @Override protected int initialCapacity() { return 3; } @Override protected void attributeDefine() { - addAttribute(0, new Attribute("id", AttributeType.STRING, new NonOperation())); + addAttribute(0, new Attribute(ApplicationTable.COLUMN_ID, AttributeType.STRING, new NonOperation())); addAttribute(1, new Attribute(ApplicationTable.COLUMN_APPLICATION_CODE, AttributeType.STRING, new CoverOperation())); addAttribute(2, new Attribute(ApplicationTable.COLUMN_APPLICATION_ID, AttributeType.INTEGER, new CoverOperation())); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationTable.java index 797b8d80a..6b067c2b1 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationTable.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationTable.java @@ -1,9 +1,11 @@ package org.skywalking.apm.collector.agentstream.worker.register.application; +import org.skywalking.apm.collector.agentstream.worker.CommonTable; + /** * @author pengys5 */ -public class ApplicationTable { +public class ApplicationTable extends CommonTable { public static final String TABLE = "application"; public static final String COLUMN_APPLICATION_CODE = "application_code"; public static final String COLUMN_APPLICATION_ID = "application_id"; diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java index e14afa7b2..cdb986013 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java @@ -12,20 +12,14 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class InstanceDataDefine extends DataDefine { - public static final int DEFINE_ID = 102; - - @Override public int defineId() { - return DEFINE_ID; - } - @Override protected int initialCapacity() { return 6; } @Override protected void attributeDefine() { - addAttribute(0, new Attribute("id", AttributeType.STRING, new NonOperation())); + addAttribute(0, new Attribute(InstanceTable.COLUMN_ID, AttributeType.STRING, new NonOperation())); addAttribute(1, new Attribute(InstanceTable.COLUMN_APPLICATION_ID, AttributeType.INTEGER, new CoverOperation())); - addAttribute(2, new Attribute(InstanceTable.COLUMN_AGENTUUID, AttributeType.STRING, new CoverOperation())); + addAttribute(2, new Attribute(InstanceTable.COLUMN_AGENT_UUID, AttributeType.STRING, new CoverOperation())); addAttribute(3, new Attribute(InstanceTable.COLUMN_REGISTER_TIME, AttributeType.LONG, new CoverOperation())); addAttribute(4, new Attribute(InstanceTable.COLUMN_INSTANCE_ID, AttributeType.INTEGER, new CoverOperation())); addAttribute(5, new Attribute(InstanceTable.COLUMN_HEARTBEAT_TIME, AttributeType.LONG, new CoverOperation())); diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceEsTableDefine.java index dffcfa9fb..55417fbaf 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceEsTableDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceEsTableDefine.java @@ -26,7 +26,7 @@ public class InstanceEsTableDefine extends ElasticSearchTableDefine { @Override public void initialize() { addColumn(new ElasticSearchColumnDefine(InstanceTable.COLUMN_APPLICATION_ID, ElasticSearchColumnDefine.Type.Integer.name())); - addColumn(new ElasticSearchColumnDefine(InstanceTable.COLUMN_AGENTUUID, ElasticSearchColumnDefine.Type.Keyword.name())); + addColumn(new ElasticSearchColumnDefine(InstanceTable.COLUMN_AGENT_UUID, ElasticSearchColumnDefine.Type.Keyword.name())); addColumn(new ElasticSearchColumnDefine(InstanceTable.COLUMN_REGISTER_TIME, ElasticSearchColumnDefine.Type.Long.name())); addColumn(new ElasticSearchColumnDefine(InstanceTable.COLUMN_INSTANCE_ID, ElasticSearchColumnDefine.Type.Integer.name())); addColumn(new ElasticSearchColumnDefine(InstanceTable.COLUMN_HEARTBEAT_TIME, ElasticSearchColumnDefine.Type.Long.name())); diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceH2TableDefine.java index 58ca3e626..17afcea8e 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceH2TableDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceH2TableDefine.java @@ -14,7 +14,7 @@ public class InstanceH2TableDefine extends H2TableDefine { @Override public void initialize() { addColumn(new H2ColumnDefine(InstanceTable.COLUMN_APPLICATION_ID, H2ColumnDefine.Type.Int.name())); - addColumn(new H2ColumnDefine(InstanceTable.COLUMN_AGENTUUID, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(InstanceTable.COLUMN_AGENT_UUID, H2ColumnDefine.Type.Varchar.name())); addColumn(new H2ColumnDefine(InstanceTable.COLUMN_REGISTER_TIME, H2ColumnDefine.Type.Bigint.name())); addColumn(new H2ColumnDefine(InstanceTable.COLUMN_INSTANCE_ID, H2ColumnDefine.Type.Int.name())); addColumn(new H2ColumnDefine(InstanceTable.COLUMN_HEARTBEAT_TIME, H2ColumnDefine.Type.Bigint.name())); diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceTable.java index 5a26adb6e..fa59f8220 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceTable.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceTable.java @@ -1,12 +1,14 @@ package org.skywalking.apm.collector.agentstream.worker.register.instance; +import org.skywalking.apm.collector.agentstream.worker.CommonTable; + /** * @author pengys5 */ -public class InstanceTable { +public class InstanceTable extends CommonTable { public static final String TABLE = "instance"; public static final String COLUMN_APPLICATION_ID = "application_id"; - public static final String COLUMN_AGENTUUID = "agent_uuid"; + public static final String COLUMN_AGENT_UUID = "agent_uuid"; public static final String COLUMN_REGISTER_TIME = "register_time"; public static final String COLUMN_INSTANCE_ID = "instance_id"; public static final String COLUMN_HEARTBEAT_TIME = "heartbeatTime"; diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceEsDAO.java index 3b56987d9..bf872bca5 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceEsDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceEsDAO.java @@ -34,7 +34,7 @@ public class InstanceEsDAO extends EsDAO implements IInstanceDAO { searchRequestBuilder.setSearchType(SearchType.QUERY_THEN_FETCH); BoolQueryBuilder builder = QueryBuilders.boolQuery(); builder.must().add(QueryBuilders.termQuery(InstanceTable.COLUMN_APPLICATION_ID, applicationId)); - builder.must().add(QueryBuilders.termQuery(InstanceTable.COLUMN_AGENTUUID, agentUUID)); + builder.must().add(QueryBuilders.termQuery(InstanceTable.COLUMN_AGENT_UUID, agentUUID)); searchRequestBuilder.setQuery(builder); searchRequestBuilder.setSize(1); @@ -60,7 +60,7 @@ public class InstanceEsDAO extends EsDAO implements IInstanceDAO { Map source = new HashMap<>(); source.put(InstanceTable.COLUMN_INSTANCE_ID, instance.getInstanceId()); source.put(InstanceTable.COLUMN_APPLICATION_ID, instance.getApplicationId()); - source.put(InstanceTable.COLUMN_AGENTUUID, instance.getAgentUUID()); + source.put(InstanceTable.COLUMN_AGENT_UUID, instance.getAgentUUID()); source.put(InstanceTable.COLUMN_REGISTER_TIME, instance.getRegisterTime()); IndexResponse response = client.prepareIndex(InstanceTable.TABLE, instance.getId()).setSource(source).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameDataDefine.java index d30a15240..42463c27e 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameDataDefine.java @@ -12,18 +12,12 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class ServiceNameDataDefine extends DataDefine { - public static final int DEFINE_ID = 103; - - @Override public int defineId() { - return DEFINE_ID; - } - @Override protected int initialCapacity() { return 4; } @Override protected void attributeDefine() { - addAttribute(0, new Attribute("id", AttributeType.STRING, new NonOperation())); + addAttribute(0, new Attribute(ServiceNameTable.COLUMN_ID, AttributeType.STRING, new NonOperation())); addAttribute(1, new Attribute(ServiceNameTable.COLUMN_SERVICE_NAME, AttributeType.STRING, new CoverOperation())); addAttribute(2, new Attribute(ServiceNameTable.COLUMN_APPLICATION_ID, AttributeType.INTEGER, new CoverOperation())); addAttribute(3, new Attribute(ServiceNameTable.COLUMN_SERVICE_ID, AttributeType.INTEGER, new CoverOperation())); diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameTable.java index f15fd6075..f6c81e491 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameTable.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameTable.java @@ -1,9 +1,11 @@ package org.skywalking.apm.collector.agentstream.worker.register.servicename; +import org.skywalking.apm.collector.agentstream.worker.CommonTable; + /** * @author pengys5 */ -public class ServiceNameTable { +public class ServiceNameTable extends CommonTable { public static final String TABLE = "service_name"; public static final String COLUMN_SERVICE_NAME = "service_name"; public static final String COLUMN_APPLICATION_ID = "application_id"; diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostDataDefine.java index 51f57fd43..fd9f85c03 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostDataDefine.java @@ -14,10 +14,6 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class SegmentCostDataDefine extends DataDefine { - @Override public int defineId() { - return 402; - } - @Override protected int initialCapacity() { return 8; } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/define/SegmentDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/define/SegmentDataDefine.java index 6edf0322f..7e2f75caa 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/define/SegmentDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/define/SegmentDataDefine.java @@ -15,12 +15,6 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class SegmentDataDefine extends DataDefine { - public static final int DEFINE_ID = 401; - - @Override public int defineId() { - return DEFINE_ID; - } - @Override protected int initialCapacity() { return 2; } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryDataDefine.java index ed044eb5d..ea3a37d75 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryDataDefine.java @@ -14,10 +14,6 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class ServiceEntryDataDefine extends DataDefine { - @Override public int defineId() { - return 501; - } - @Override protected int initialCapacity() { return 4; } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefDataDefine.java index 44248a920..55e4b723d 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefDataDefine.java @@ -14,12 +14,6 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class ServiceRefDataDefine extends DataDefine { - public static final int DEFINE_ID = 501; - - @Override public int defineId() { - return DEFINE_ID; - } - @Override protected int initialCapacity() { return 4; } diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterDefinitionFile.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterDefinitionFile.java deleted file mode 100644 index aa1bb7b54..000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterDefinitionFile.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.skywalking.apm.collector.core.cluster; - -import org.skywalking.apm.collector.core.framework.DefinitionFile; - -/** - * @author pengys5 - */ -public class ClusterDefinitionFile extends DefinitionFile { - - @Override protected String fileName() { - return "cluster-configuration.define"; - } -} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Data.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Data.java index 5d5e25af8..5a8832ba3 100644 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Data.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Data.java @@ -8,7 +8,6 @@ import org.skywalking.apm.collector.stream.worker.selector.AbstractHashMessage; * @author pengys5 */ public class Data extends AbstractHashMessage { - private int defineId; private final int stringCapacity; private final int longCapacity; private final int floatCapacity; @@ -22,10 +21,9 @@ public class Data extends AbstractHashMessage { private Boolean[] dataBooleans; private byte[][] dataBytes; - public Data(String id, int defineId, int stringCapacity, int longCapacity, int floatCapacity, int integerCapacity, + public Data(String id, int stringCapacity, int longCapacity, int floatCapacity, int integerCapacity, int booleanCapacity, int byteCapacity) { super(id); - this.defineId = defineId; this.dataStrings = new String[stringCapacity]; this.dataLongs = new Long[longCapacity]; this.dataFloats = new Float[floatCapacity]; @@ -92,10 +90,6 @@ public class Data extends AbstractHashMessage { return dataStrings[0]; } - public int getDefineId() { - return defineId; - } - public RemoteData serialize() { RemoteData.Builder builder = RemoteData.newBuilder(); builder.setIntegerCapacity(integerCapacity); diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefine.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefine.java index 254e742d6..ddbd9c2a3 100644 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefine.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefine.java @@ -42,14 +42,12 @@ public abstract class DataDefine { attributes[position] = attribute; } - public abstract int defineId(); - protected abstract int initialCapacity(); protected abstract void attributeDefine(); public final Data build(String id) { - return new Data(id, defineId(), stringCapacity, longCapacity, floatCapacity, integerCapacity, booleanCapacity, byteCapacity); + return new Data(id, stringCapacity, longCapacity, floatCapacity, integerCapacity, booleanCapacity, byteCapacity); } public void mergeData(Data newData, Data oldData) {