From 63eef704a183163dbfb0567592cf563edbb5368e Mon Sep 17 00:00:00 2001 From: peng-yongsheng <8082209@qq.com> Date: Wed, 1 Nov 2017 22:52:35 +0800 Subject: [PATCH] add queue component --- apm-collector/apm-collector-component/pom.xml | 2 +- .../queue-component/pom.xml | 25 +++ .../collector/queue/DaemonThreadFactory.java | 35 +++++ .../collector/queue/EndOfBatchCommand.java | 25 +++ .../apm/collector/queue/MessageHolder.java | 38 +++++ .../apm/collector/queue/QueueCreator.java | 26 +++ .../collector/queue/QueueEventHandler.java | 26 +++ .../apm/collector/queue/QueueExecutor.java | 26 +++ .../disruptor/DisruptorEventHandler.java | 76 +++++++++ .../disruptor/DisruptorQueueCreator.java | 53 +++++++ .../queue/disruptor/MessageHolderFactory.java | 34 ++++ .../stream/AbstractLocalAsyncWorker.java | 70 +++++++++ .../AbstractLocalAsyncWorkerProvider.java | 53 +++++++ .../stream/AbstractRemoteWorker.java | 66 ++++++++ .../stream/AbstractRemoteWorkerProvider.java | 52 ++++++ .../apm/collector/stream/AbstractWorker.java | 70 +++++++++ .../stream/AbstractWorkerProvider.java | 39 +++++ .../stream/ClusterWorkerContext.java | 39 +++++ .../stream/ClusterWorkerRefCounter.java | 40 +++++ .../apm/collector/stream/Context.java | 35 +++++ .../LocalAsyncWorkerProviderDefineLoader.java | 49 ++++++ .../collector/stream/LocalAsyncWorkerRef.java | 39 +++++ .../LocalWorkerProviderDefinitionFile.java | 30 ++++ .../apm/collector/stream/LookUp.java | 27 ++++ .../apm/collector/stream/Provider.java | 27 ++++ .../stream/ProviderNotFoundException.java | 25 +++ .../RemoteWorkerProviderDefineLoader.java | 49 ++++++ .../RemoteWorkerProviderDefinitionFile.java | 30 ++++ .../apm/collector/stream/RemoteWorkerRef.java | 148 ++++++++++++++++++ .../skywalking/apm/collector/stream/Role.java | 34 ++++ .../stream/UsedRoleNameException.java | 25 +++ .../apm/collector/stream/WorkerContext.java | 91 +++++++++++ .../apm/collector/stream/WorkerException.java | 37 +++++ .../stream/WorkerInvokeException.java | 36 +++++ .../stream/WorkerNotFoundException.java | 25 +++ .../apm/collector/stream/WorkerRef.java | 36 +++++ .../apm/collector/stream/WorkerRefs.java | 57 +++++++ .../stream/selector/ForeverFirstSelector.java | 37 +++++ .../stream/selector/HashCodeSelector.java | 54 +++++++ .../stream/selector/RollingSelector.java | 50 ++++++ .../stream/selector/WorkerSelector.java | 44 ++++++ 41 files changed, 1779 insertions(+), 1 deletion(-) create mode 100644 apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/DaemonThreadFactory.java create mode 100644 apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/EndOfBatchCommand.java create mode 100644 apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/MessageHolder.java create mode 100644 apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/QueueCreator.java create mode 100644 apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/QueueEventHandler.java create mode 100644 apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/QueueExecutor.java create mode 100644 apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/disruptor/DisruptorEventHandler.java create mode 100644 apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/disruptor/DisruptorQueueCreator.java create mode 100644 apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/disruptor/MessageHolderFactory.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalAsyncWorker.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalAsyncWorkerProvider.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractRemoteWorker.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractRemoteWorkerProvider.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractWorker.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractWorkerProvider.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/ClusterWorkerContext.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/ClusterWorkerRefCounter.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/Context.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/LocalAsyncWorkerProviderDefineLoader.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/LocalAsyncWorkerRef.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/LocalWorkerProviderDefinitionFile.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/LookUp.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/Provider.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/ProviderNotFoundException.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/RemoteWorkerProviderDefineLoader.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/RemoteWorkerProviderDefinitionFile.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/RemoteWorkerRef.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/Role.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/UsedRoleNameException.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerContext.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerException.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerInvokeException.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerNotFoundException.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerRef.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerRefs.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/selector/ForeverFirstSelector.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/selector/HashCodeSelector.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/selector/RollingSelector.java create mode 100644 apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/selector/WorkerSelector.java diff --git a/apm-collector/apm-collector-component/pom.xml b/apm-collector/apm-collector-component/pom.xml index c73f011f9..d4ab86353 100644 --- a/apm-collector/apm-collector-component/pom.xml +++ b/apm-collector/apm-collector-component/pom.xml @@ -14,8 +14,8 @@ client-component server-component - queue-component stream-component + queue-component diff --git a/apm-collector/apm-collector-component/queue-component/pom.xml b/apm-collector/apm-collector-component/queue-component/pom.xml index 166d4fcaa..ad0399870 100644 --- a/apm-collector/apm-collector-component/queue-component/pom.xml +++ b/apm-collector/apm-collector-component/queue-component/pom.xml @@ -1,4 +1,22 @@ + + @@ -12,4 +30,11 @@ queue-component jar + + + com.lmax + disruptor + 3.3.6 + + \ No newline at end of file diff --git a/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/DaemonThreadFactory.java b/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/DaemonThreadFactory.java new file mode 100644 index 000000000..ef45c8552 --- /dev/null +++ b/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/DaemonThreadFactory.java @@ -0,0 +1,35 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.queue; + +import java.util.concurrent.ThreadFactory; + +/** + * @author peng-yongsheng + */ +public enum DaemonThreadFactory implements ThreadFactory { + INSTANCE; + + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(r); + t.setDaemon(true); + return t; + } +} diff --git a/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/EndOfBatchCommand.java b/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/EndOfBatchCommand.java new file mode 100644 index 000000000..bfa3a6fb7 --- /dev/null +++ b/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/EndOfBatchCommand.java @@ -0,0 +1,25 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.queue; + +/** + * @author peng-yongsheng + */ +public class EndOfBatchCommand { +} diff --git a/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/MessageHolder.java b/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/MessageHolder.java new file mode 100644 index 000000000..d0c0333a5 --- /dev/null +++ b/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/MessageHolder.java @@ -0,0 +1,38 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.queue; + +/** + * @author peng-yongsheng + */ +public class MessageHolder { + private Object message; + + public Object getMessage() { + return message; + } + + public void setMessage(Object message) { + this.message = message; + } + + public void reset() { + message = null; + } +} diff --git a/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/QueueCreator.java b/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/QueueCreator.java new file mode 100644 index 000000000..6c78d2cb7 --- /dev/null +++ b/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/QueueCreator.java @@ -0,0 +1,26 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.queue; + +/** + * @author peng-yongsheng + */ +public interface QueueCreator { + QueueEventHandler create(int queueSize, QueueExecutor executor); +} diff --git a/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/QueueEventHandler.java b/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/QueueEventHandler.java new file mode 100644 index 000000000..194bb15e6 --- /dev/null +++ b/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/QueueEventHandler.java @@ -0,0 +1,26 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.queue; + +/** + * @author peng-yongsheng + */ +public interface QueueEventHandler { + void tell(Object message); +} diff --git a/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/QueueExecutor.java b/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/QueueExecutor.java new file mode 100644 index 000000000..40db61fdd --- /dev/null +++ b/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/QueueExecutor.java @@ -0,0 +1,26 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.queue; + +/** + * @author peng-yongsheng + */ +public interface QueueExecutor { + void execute(Object message); +} diff --git a/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/disruptor/DisruptorEventHandler.java b/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/disruptor/DisruptorEventHandler.java new file mode 100644 index 000000000..729b3054d --- /dev/null +++ b/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/disruptor/DisruptorEventHandler.java @@ -0,0 +1,76 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.queue.disruptor; + +import com.lmax.disruptor.EventHandler; +import com.lmax.disruptor.RingBuffer; +import org.skywalking.apm.collector.queue.EndOfBatchCommand; +import org.skywalking.apm.collector.queue.MessageHolder; +import org.skywalking.apm.collector.queue.QueueEventHandler; +import org.skywalking.apm.collector.queue.QueueExecutor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author peng-yongsheng + */ +public class DisruptorEventHandler implements EventHandler, QueueEventHandler { + + private final Logger logger = LoggerFactory.getLogger(DisruptorEventHandler.class); + + private RingBuffer ringBuffer; + private QueueExecutor executor; + + DisruptorEventHandler(RingBuffer ringBuffer, QueueExecutor executor) { + this.ringBuffer = ringBuffer; + this.executor = executor; + } + + /** + * Receive the message from disruptor, when message in disruptor is empty, then send the cached data + * to the next workers. + * + * @param event published to the {@link RingBuffer} + * @param sequence of the event being processed + * @param endOfBatch flag to indicate if this is the last event in a batch from the {@link RingBuffer} + */ + public void onEvent(MessageHolder event, long sequence, boolean endOfBatch) { + Object message = event.getMessage(); + event.reset(); + + executor.execute(message); + if (endOfBatch) { + executor.execute(new EndOfBatchCommand()); + } + } + + /** + * Push the message into disruptor ring buffer. + * + * @param message of the data to process. + */ + public void tell(Object message) { + long sequence = ringBuffer.next(); + try { + ringBuffer.get(sequence).setMessage(message); + } finally { + ringBuffer.publish(sequence); + } + } +} diff --git a/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/disruptor/DisruptorQueueCreator.java b/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/disruptor/DisruptorQueueCreator.java new file mode 100644 index 000000000..eb3394de3 --- /dev/null +++ b/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/disruptor/DisruptorQueueCreator.java @@ -0,0 +1,53 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.queue.disruptor; + +import com.lmax.disruptor.RingBuffer; +import com.lmax.disruptor.dsl.Disruptor; +import org.skywalking.apm.collector.queue.DaemonThreadFactory; +import org.skywalking.apm.collector.queue.MessageHolder; +import org.skywalking.apm.collector.queue.QueueCreator; +import org.skywalking.apm.collector.queue.QueueEventHandler; +import org.skywalking.apm.collector.queue.QueueExecutor; + +/** + * @author peng-yongsheng + */ +public class DisruptorQueueCreator implements QueueCreator { + + @Override public QueueEventHandler create(int queueSize, QueueExecutor executor) { + // Specify the size of the ring buffer, must be power of 2. + if (!((((queueSize - 1) & queueSize) == 0) && queueSize != 0)) { + throw new IllegalArgumentException("queue size must be power of 2"); + } + + // Construct the Disruptor + Disruptor disruptor = new Disruptor(MessageHolderFactory.INSTANCE, queueSize, DaemonThreadFactory.INSTANCE); + + RingBuffer ringBuffer = disruptor.getRingBuffer(); + DisruptorEventHandler eventHandler = new DisruptorEventHandler(ringBuffer, executor); + + // Connect the handler + disruptor.handleEventsWith(eventHandler); + + // Start the Disruptor, starts all threads running + disruptor.start(); + return eventHandler; + } +} diff --git a/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/disruptor/MessageHolderFactory.java b/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/disruptor/MessageHolderFactory.java new file mode 100644 index 000000000..04ac2959e --- /dev/null +++ b/apm-collector/apm-collector-component/queue-component/src/main/java/org/skywalking/apm/collector/queue/disruptor/MessageHolderFactory.java @@ -0,0 +1,34 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.queue.disruptor; + +import com.lmax.disruptor.EventFactory; +import org.skywalking.apm.collector.queue.MessageHolder; + +/** + * @author peng-yongsheng + */ +public class MessageHolderFactory implements EventFactory { + + public static MessageHolderFactory INSTANCE = new MessageHolderFactory(); + + public MessageHolder newInstance() { + return new MessageHolder(); + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalAsyncWorker.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalAsyncWorker.java new file mode 100644 index 000000000..66e363199 --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalAsyncWorker.java @@ -0,0 +1,70 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +/** + * The AbstractLocalAsyncWorker implementations represent workers, + * which receive local asynchronous message. + * + * @author peng-yongsheng + * @since v3.0-2017 + */ +public abstract class AbstractLocalAsyncWorker extends AbstractWorker implements QueueExecutor { + + private LocalAsyncWorkerRef workerRef; + + /** + * Construct an AbstractLocalAsyncWorker with the worker role and context. + * + * @param role The responsibility of worker in cluster, more than one workers can have same responsibility which use + * to provide load balancing ability. + * @param clusterContext See {@link ClusterWorkerContext} + */ + public AbstractLocalAsyncWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + /** + * The asynchronous worker always use to persistence data into db, this is the end of the streaming, + * so usually no need to create the next worker instance at the time of this worker instance create. + * + * @throws ProviderNotFoundException When worker provider not found, it will be throw this exception. + */ + @Override + public void preStart() throws ProviderNotFoundException { + } + + @Override protected final LocalAsyncWorkerRef getSelf() { + return workerRef; + } + + @Override protected final void putSelfRef(LocalAsyncWorkerRef workerRef) { + this.workerRef = workerRef; + } + + /** + * Receive message + * + * @param message The persistence data or metric data. + * @throws WorkerException The Exception happen in {@link #onWork(Object)} + */ + final public void allocateJob(Object message) throws WorkerException { + onWork(message); + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalAsyncWorkerProvider.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalAsyncWorkerProvider.java new file mode 100644 index 000000000..995408ffc --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractLocalAsyncWorkerProvider.java @@ -0,0 +1,53 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.core.queue.QueueCreator; +import org.skywalking.apm.collector.core.queue.QueueEventHandler; +import org.skywalking.apm.collector.core.queue.QueueExecutor; +import org.skywalking.apm.collector.queue.QueueModuleContext; +import org.skywalking.apm.collector.queue.QueueModuleGroupDefine; +import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; +import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorkerContainer; + +/** + * @author peng-yongsheng + */ +public abstract class AbstractLocalAsyncWorkerProvider extends AbstractWorkerProvider { + + public abstract int queueSize(); + + @Override final public WorkerRef create() throws ProviderNotFoundException { + T localAsyncWorker = workerInstance(getClusterContext()); + localAsyncWorker.preStart(); + + if (localAsyncWorker instanceof PersistenceWorker) { + PersistenceWorkerContainer.INSTANCE.addWorker((PersistenceWorker)localAsyncWorker); + } + + QueueCreator queueCreator = ((QueueModuleContext)CollectorContextHelper.INSTANCE.getContext(QueueModuleGroupDefine.GROUP_NAME)).getQueueCreator(); + QueueEventHandler queueEventHandler = queueCreator.create(queueSize(), localAsyncWorker); + + LocalAsyncWorkerRef workerRef = new LocalAsyncWorkerRef(role(), queueEventHandler); + getClusterContext().put(workerRef); + localAsyncWorker.putSelfRef(workerRef); + return workerRef; + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractRemoteWorker.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractRemoteWorker.java new file mode 100644 index 000000000..a083aebf1 --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractRemoteWorker.java @@ -0,0 +1,66 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +/** + * The AbstractRemoteWorker implementations represent workers, + * which receive remote messages. + *

+ * Usually, the implementations are doing persistent, or aggregate works. + * + * @author peng-yongsheng + * @since v3.0-2017 + */ +public abstract class AbstractRemoteWorker extends AbstractWorker { + + private RemoteWorkerRef workerRef; + + /** + * Construct an AbstractRemoteWorker with the worker role and context. + * + * @param role If multi-workers are for load balance, they should be more likely called worker instance. Meaning, + * each worker have multi instances. + * @param clusterContext See {@link ClusterWorkerContext} + */ + protected AbstractRemoteWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + /** + * This method use for message producer to call for send message. + * + * @param message The persistence data or metric data. + * @throws Exception The Exception happen in {@link #onWork(Object)} + */ + final public void allocateJob(Object message) throws WorkerInvokeException { + try { + onWork(message); + } catch (WorkerException e) { + throw new WorkerInvokeException(e.getMessage(), e.getCause()); + } + } + + @Override protected final RemoteWorkerRef getSelf() { + return workerRef; + } + + @Override protected final void putSelfRef(RemoteWorkerRef workerRef) { + this.workerRef = workerRef; + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractRemoteWorkerProvider.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractRemoteWorkerProvider.java new file mode 100644 index 000000000..90e139a07 --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractRemoteWorkerProvider.java @@ -0,0 +1,52 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +import org.skywalking.apm.collector.client.grpc.GRPCClient; + +/** + * The AbstractRemoteWorkerProvider implementations represent providers, + * which create instance of cluster workers whose implemented {@link AbstractRemoteWorker}. + *

+ * + * @author peng-yongsheng + * @since v3.0-2017 + */ +public abstract class AbstractRemoteWorkerProvider extends AbstractWorkerProvider { + + /** + * Create the worker instance into akka system, the akka system will control the cluster worker life cycle. + * + * @return The created worker reference. See {@link RemoteWorkerRef} + * @throws ProviderNotFoundException This worker instance attempted to find a provider which use to create another + * worker instance, when the worker provider not find then Throw this Exception. + */ + @Override final public WorkerRef create() { + T clusterWorker = workerInstance(getClusterContext()); + RemoteWorkerRef workerRef = new RemoteWorkerRef(role(), clusterWorker); + getClusterContext().put(workerRef); + return workerRef; + } + + public final RemoteWorkerRef create(GRPCClient client) { + RemoteWorkerRef workerRef = new RemoteWorkerRef(role(), client); + getClusterContext().put(workerRef); + return workerRef; + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractWorker.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractWorker.java new file mode 100644 index 000000000..f60db60c6 --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractWorker.java @@ -0,0 +1,70 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +import org.skywalking.apm.collector.core.framework.Executor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author peng-yongsheng + */ +public abstract class AbstractWorker implements Executor { + + private final Logger logger = LoggerFactory.getLogger(AbstractWorker.class); + + private final Role role; + + private final ClusterWorkerContext clusterContext; + + public AbstractWorker(Role role, ClusterWorkerContext clusterContext) { + this.role = role; + this.clusterContext = clusterContext; + } + + @Override public final void execute(Object message) { + try { + onWork(message); + } catch (WorkerException e) { + logger.error(e.getMessage(), e); + } + } + + /** + * The data process logic in this method. + * + * @param message Cast the message object to a expect subclass. + * @throws WorkerException Don't handle the exception, throw it. + */ + protected abstract void onWork(Object message) throws WorkerException; + + public abstract void preStart() throws ProviderNotFoundException; + + final public ClusterWorkerContext getClusterContext() { + return clusterContext; + } + + final public Role getRole() { + return role; + } + + protected abstract S getSelf(); + + protected abstract void putSelfRef(S workerRef); +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractWorkerProvider.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractWorkerProvider.java new file mode 100644 index 000000000..8ade3430a --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/AbstractWorkerProvider.java @@ -0,0 +1,39 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +/** + * @author peng-yongsheng + */ +public abstract class AbstractWorkerProvider implements Provider { + + private ClusterWorkerContext clusterContext; + + public abstract Role role(); + + public abstract T workerInstance(ClusterWorkerContext clusterContext); + + final public void setClusterContext(ClusterWorkerContext clusterContext) { + this.clusterContext = clusterContext; + } + + final protected ClusterWorkerContext getClusterContext() { + return clusterContext; + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/ClusterWorkerContext.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/ClusterWorkerContext.java new file mode 100644 index 000000000..cc14dcdbc --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/ClusterWorkerContext.java @@ -0,0 +1,39 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author peng-yongsheng + */ +public class ClusterWorkerContext extends WorkerContext { + + private List providers = new ArrayList<>(); + + public List getProviders() { + return providers; + } + + @Override + public void putProvider(AbstractRemoteWorkerProvider provider) { + providers.add(provider); + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/ClusterWorkerRefCounter.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/ClusterWorkerRefCounter.java new file mode 100644 index 000000000..754ad0365 --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/ClusterWorkerRefCounter.java @@ -0,0 +1,40 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author peng-yongsheng + */ +public enum ClusterWorkerRefCounter { + INSTANCE; + + private Map counter = new ConcurrentHashMap<>(); + + public int incrementAndGet(Role role) { + if (!counter.containsKey(role.roleName())) { + AtomicInteger atomic = new AtomicInteger(0); + counter.putIfAbsent(role.roleName(), atomic); + } + return counter.get(role.roleName()).incrementAndGet(); + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/Context.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/Context.java new file mode 100644 index 000000000..314a8d83c --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/Context.java @@ -0,0 +1,35 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +/** + * @author peng-yongsheng + */ +public interface Context extends LookUp { + + void putProvider(AbstractRemoteWorkerProvider provider); + + WorkerRefs lookup(Role role) throws WorkerNotFoundException; + + RemoteWorkerRef lookupInSide(String roleName) throws WorkerNotFoundException; + + void put(WorkerRef workerRef); + + void remove(WorkerRef workerRef); +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/LocalAsyncWorkerProviderDefineLoader.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/LocalAsyncWorkerProviderDefineLoader.java new file mode 100644 index 000000000..19dc7e617 --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/LocalAsyncWorkerProviderDefineLoader.java @@ -0,0 +1,49 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +import java.util.ArrayList; +import java.util.List; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.framework.Loader; +import org.skywalking.apm.collector.core.util.DefinitionLoader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author peng-yongsheng + */ +public class LocalAsyncWorkerProviderDefineLoader implements Loader> { + + private final Logger logger = LoggerFactory.getLogger(LocalAsyncWorkerProviderDefineLoader.class); + + @Override public List load() throws DefineException { + List providers = new ArrayList<>(); + LocalWorkerProviderDefinitionFile definitionFile = new LocalWorkerProviderDefinitionFile(); + logger.info("local async worker provider definition file name: {}", definitionFile.fileName()); + + DefinitionLoader definitionLoader = DefinitionLoader.load(AbstractLocalAsyncWorkerProvider.class, definitionFile); + + for (AbstractLocalAsyncWorkerProvider provider : definitionLoader) { + logger.info("loaded local async worker provider definition class: {}", provider.getClass().getName()); + providers.add(provider); + } + return providers; + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/LocalAsyncWorkerRef.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/LocalAsyncWorkerRef.java new file mode 100644 index 000000000..79834cc6a --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/LocalAsyncWorkerRef.java @@ -0,0 +1,39 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +import org.skywalking.apm.collector.core.queue.QueueEventHandler; + +/** + * @author peng-yongsheng + */ +public class LocalAsyncWorkerRef extends WorkerRef { + + private QueueEventHandler queueEventHandler; + + public LocalAsyncWorkerRef(Role role, QueueEventHandler queueEventHandler) { + super(role); + this.queueEventHandler = queueEventHandler; + } + + @Override + public void tell(Object message) throws WorkerInvokeException { + queueEventHandler.tell(message); + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/LocalWorkerProviderDefinitionFile.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/LocalWorkerProviderDefinitionFile.java new file mode 100644 index 000000000..346223af8 --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/LocalWorkerProviderDefinitionFile.java @@ -0,0 +1,30 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +import org.skywalking.apm.collector.core.framework.DefinitionFile; + +/** + * @author peng-yongsheng + */ +public class LocalWorkerProviderDefinitionFile extends DefinitionFile { + @Override protected String fileName() { + return "local_worker_provider.define"; + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/LookUp.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/LookUp.java new file mode 100644 index 000000000..933dfa7ad --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/LookUp.java @@ -0,0 +1,27 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +/** + * @author peng-yongsheng + */ +public interface LookUp { + + WorkerRefs lookup(Role role) throws WorkerNotFoundException; +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/Provider.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/Provider.java new file mode 100644 index 000000000..4f3c32fd9 --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/Provider.java @@ -0,0 +1,27 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +/** + * @author peng-yongsheng + */ +public interface Provider { + + WorkerRef create() throws ProviderNotFoundException; +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/ProviderNotFoundException.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/ProviderNotFoundException.java new file mode 100644 index 000000000..e3d94270d --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/ProviderNotFoundException.java @@ -0,0 +1,25 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +public class ProviderNotFoundException extends Exception { + public ProviderNotFoundException(String message) { + super(message); + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/RemoteWorkerProviderDefineLoader.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/RemoteWorkerProviderDefineLoader.java new file mode 100644 index 000000000..0e3d57441 --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/RemoteWorkerProviderDefineLoader.java @@ -0,0 +1,49 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +import java.util.ArrayList; +import java.util.List; +import org.skywalking.apm.collector.core.framework.DefineException; +import org.skywalking.apm.collector.core.framework.Loader; +import org.skywalking.apm.collector.core.util.DefinitionLoader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author peng-yongsheng + */ +public class RemoteWorkerProviderDefineLoader implements Loader> { + + private final Logger logger = LoggerFactory.getLogger(RemoteWorkerProviderDefineLoader.class); + + @Override public List load() throws DefineException { + List providers = new ArrayList<>(); + RemoteWorkerProviderDefinitionFile definitionFile = new RemoteWorkerProviderDefinitionFile(); + logger.info("remote worker provider definition file name: {}", definitionFile.fileName()); + + DefinitionLoader definitionLoader = DefinitionLoader.load(AbstractRemoteWorkerProvider.class, definitionFile); + + for (AbstractRemoteWorkerProvider provider : definitionLoader) { + logger.info("loaded remote worker provider definition class: {}", provider.getClass().getName()); + providers.add(provider); + } + return providers; + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/RemoteWorkerProviderDefinitionFile.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/RemoteWorkerProviderDefinitionFile.java new file mode 100644 index 000000000..184da0923 --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/RemoteWorkerProviderDefinitionFile.java @@ -0,0 +1,30 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +import org.skywalking.apm.collector.core.framework.DefinitionFile; + +/** + * @author peng-yongsheng + */ +public class RemoteWorkerProviderDefinitionFile extends DefinitionFile { + @Override protected String fileName() { + return "remote_worker_provider.define"; + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/RemoteWorkerRef.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/RemoteWorkerRef.java new file mode 100644 index 000000000..cd80db730 --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/RemoteWorkerRef.java @@ -0,0 +1,148 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +import io.grpc.stub.StreamObserver; +import org.skywalking.apm.collector.client.grpc.GRPCClient; +import org.skywalking.apm.collector.core.util.Const; +import org.skywalking.apm.collector.remote.grpc.proto.Empty; +import org.skywalking.apm.collector.remote.grpc.proto.RemoteCommonServiceGrpc; +import org.skywalking.apm.collector.remote.grpc.proto.RemoteData; +import org.skywalking.apm.collector.remote.grpc.proto.RemoteMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author peng-yongsheng + */ +public class RemoteWorkerRef extends WorkerRef { + + private final Logger logger = LoggerFactory.getLogger(RemoteWorkerRef.class); + + private final Boolean acrossJVM; + private final RemoteCommonServiceGrpc.RemoteCommonServiceStub stub; + private StreamObserver streamObserver; + private final AbstractRemoteWorker remoteWorker; + private final String address; + + public RemoteWorkerRef(Role role, AbstractRemoteWorker remoteWorker) { + super(role); + this.remoteWorker = remoteWorker; + this.acrossJVM = false; + this.stub = null; + this.address = Const.EMPTY_STRING; + } + + public RemoteWorkerRef(Role role, GRPCClient client) { + super(role); + this.remoteWorker = null; + this.acrossJVM = true; + this.stub = RemoteCommonServiceGrpc.newStub(client.getChannel()); + this.address = client.toString(); + createStreamObserver(); + } + + @Override + public void tell(Object message) throws WorkerInvokeException { + if (acrossJVM) { + try { + RemoteData remoteData = getRole().dataDefine().serialize(message); + RemoteMessage.Builder builder = RemoteMessage.newBuilder(); + builder.setWorkerRole(getRole().roleName()); + builder.setRemoteData(remoteData); + + streamObserver.onNext(builder.build()); + } catch (Throwable e) { + logger.error(e.getMessage(), e); + } + } else { + remoteWorker.allocateJob(message); + } + } + + public Boolean isAcrossJVM() { + return acrossJVM; + } + + private void createStreamObserver() { + StreamStatus status = new StreamStatus(false); + streamObserver = stub.call(new StreamObserver() { + @Override public void onNext(Empty empty) { + } + + @Override public void onError(Throwable throwable) { + logger.error(throwable.getMessage(), throwable); + } + + @Override public void onCompleted() { + status.finished(); + } + }); + } + + class StreamStatus { + private volatile boolean status; + + public StreamStatus(boolean status) { + this.status = status; + } + + public boolean isFinish() { + return status; + } + + public void finished() { + this.status = true; + } + + /** + * @param maxTimeout max wait time, milliseconds. + */ + public void wait4Finish(long maxTimeout) { + long time = 0; + while (!status) { + if (time > maxTimeout) { + break; + } + try2Sleep(5); + time += 5; + } + } + + /** + * Try to sleep, and ignore the {@link InterruptedException} + * + * @param millis the length of time to sleep in milliseconds + */ + private void try2Sleep(long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + + } + } + } + + @Override public String toString() { + StringBuilder toString = new StringBuilder(); + toString.append("acrossJVM: ").append(acrossJVM); + toString.append(", address: ").append(address); + return toString.toString(); + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/Role.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/Role.java new file mode 100644 index 000000000..c23a15f52 --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/Role.java @@ -0,0 +1,34 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +import org.skywalking.apm.collector.storage.define.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author peng-yongsheng + */ +public interface Role { + + String roleName(); + + WorkerSelector workerSelector(); + + DataDefine dataDefine(); +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/UsedRoleNameException.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/UsedRoleNameException.java new file mode 100644 index 000000000..d385b0d07 --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/UsedRoleNameException.java @@ -0,0 +1,25 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +public class UsedRoleNameException extends Exception { + public UsedRoleNameException(String message) { + super(message); + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerContext.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerContext.java new file mode 100644 index 000000000..90e84c772 --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerContext.java @@ -0,0 +1,91 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author peng-yongsheng + */ +public abstract class WorkerContext implements Context { + + private final Logger logger = LoggerFactory.getLogger(WorkerContext.class); + + private Map remoteWorkerRefs; + private Map> roleWorkers; + private Map roles; + + WorkerContext() { + this.roleWorkers = new HashMap<>(); + this.roles = new HashMap<>(); + this.remoteWorkerRefs = new HashMap<>(); + } + + private Map> getRoleWorkers() { + return this.roleWorkers; + } + + @Override final public WorkerRefs lookup(Role role) throws WorkerNotFoundException { + if (getRoleWorkers().containsKey(role.roleName())) { + return new WorkerRefs(getRoleWorkers().get(role.roleName()), role.workerSelector()); + } else { + throw new WorkerNotFoundException("role=" + role.roleName() + ", no available worker."); + } + } + + @Override final public RemoteWorkerRef lookupInSide(String roleName) throws WorkerNotFoundException { + if (remoteWorkerRefs.containsKey(roleName)) { + return remoteWorkerRefs.get(roleName); + } else { + throw new WorkerNotFoundException("role=" + roleName + ", no available worker."); + } + } + + public final void putRole(Role role) { + roles.put(role.roleName(), role); + } + + public final Role getRole(String roleName) { + return roles.get(roleName); + } + + @Override final public void put(WorkerRef workerRef) { + logger.debug("put worker reference into context, role name: {}", workerRef.getRole().roleName()); + if (!getRoleWorkers().containsKey(workerRef.getRole().roleName())) { + getRoleWorkers().putIfAbsent(workerRef.getRole().roleName(), new ArrayList<>()); + } + getRoleWorkers().get(workerRef.getRole().roleName()).add(workerRef); + + if (workerRef instanceof RemoteWorkerRef) { + RemoteWorkerRef remoteWorkerRef = (RemoteWorkerRef)workerRef; + if (!remoteWorkerRef.isAcrossJVM()) { + remoteWorkerRefs.put(workerRef.getRole().roleName(), remoteWorkerRef); + } + } + } + + @Override final public void remove(WorkerRef workerRef) { + getRoleWorkers().remove(workerRef.getRole().roleName()); + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerException.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerException.java new file mode 100644 index 000000000..70360ecc7 --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerException.java @@ -0,0 +1,37 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +/** + * Defines a general exception a worker can throw when it + * encounters difficulty. + * + * @author peng-yongsheng + * @since v3.1-2017 + */ +public class WorkerException extends Exception { + + public WorkerException(String message) { + super(message); + } + + public WorkerException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerInvokeException.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerInvokeException.java new file mode 100644 index 000000000..ce8fdeed6 --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerInvokeException.java @@ -0,0 +1,36 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +/** + * This exception is raised when worker fails to process job during "call" or "ask" + * + * @author peng-yongsheng + * @since v3.1-2017 + */ +public class WorkerInvokeException extends WorkerException { + + public WorkerInvokeException(String message) { + super(message); + } + + public WorkerInvokeException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerNotFoundException.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerNotFoundException.java new file mode 100644 index 000000000..fd49a3384 --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerNotFoundException.java @@ -0,0 +1,25 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +public class WorkerNotFoundException extends WorkerException { + public WorkerNotFoundException(String message) { + super(message); + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerRef.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerRef.java new file mode 100644 index 000000000..9237948bf --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerRef.java @@ -0,0 +1,36 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +/** + * @author peng-yongsheng + */ +public abstract class WorkerRef { + private Role role; + + public WorkerRef(Role role) { + this.role = role; + } + + final public Role getRole() { + return role; + } + + public abstract void tell(Object message) throws WorkerInvokeException; +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerRefs.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerRefs.java new file mode 100644 index 000000000..4a33a2625 --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/WorkerRefs.java @@ -0,0 +1,57 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream; + +import java.util.List; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author peng-yongsheng + */ +public class WorkerRefs { + + private final Logger logger = LoggerFactory.getLogger(WorkerRefs.class); + + private List workerRefs; + private WorkerSelector workerSelector; + private Role role; + + protected WorkerRefs(List workerRefs, WorkerSelector workerSelector) { + this.workerRefs = workerRefs; + this.workerSelector = workerSelector; + } + + protected WorkerRefs(List workerRefs, WorkerSelector workerSelector, Role role) { + this.workerRefs = workerRefs; + this.workerSelector = workerSelector; + this.role = role; + } + + public void tell(Object message) throws WorkerInvokeException { + logger.debug("WorkerSelector instance of {}", workerSelector.getClass()); + workerRefs.forEach(workerRef -> { + if (workerRef instanceof RemoteWorkerRef) { + logger.debug("message hashcode: {}, select workers: {}", message.hashCode(), workerRef.toString()); + } + }); + workerSelector.select(workerRefs, message).tell(message); + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/selector/ForeverFirstSelector.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/selector/ForeverFirstSelector.java new file mode 100644 index 000000000..c404b3b70 --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/selector/ForeverFirstSelector.java @@ -0,0 +1,37 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream.selector; + +import java.util.List; +import org.skywalking.apm.collector.stream.WorkerRef; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author peng-yongsheng + */ +public class ForeverFirstSelector implements WorkerSelector { + + private final Logger logger = LoggerFactory.getLogger(ForeverFirstSelector.class); + + @Override public WorkerRef select(List members, Object message) { + logger.debug("member size: {}", members.size()); + return members.get(0); + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/selector/HashCodeSelector.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/selector/HashCodeSelector.java new file mode 100644 index 000000000..c32101a45 --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/selector/HashCodeSelector.java @@ -0,0 +1,54 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream.selector; + +import java.util.List; +import org.skywalking.apm.collector.core.stream.AbstractHashMessage; +import org.skywalking.apm.collector.stream.worker.AbstractWorker; +import org.skywalking.apm.collector.stream.worker.WorkerRef; + +/** + * The HashCodeSelector is a simple implementation of {@link WorkerSelector}. It choose {@link WorkerRef} + * by message {@link AbstractHashMessage} key's hashcode, so it can use to send the same hashcode message to same {@link + * WorkerRef}. Usually, use to database operate which avoid dirty data. + * + * @author peng-yongsheng + * @since v3.0-2017 + */ +public class HashCodeSelector implements WorkerSelector { + + /** + * Use message hashcode to select {@link WorkerRef}. + * + * @param members given {@link WorkerRef} list, which size is greater than 0; + * @param message the {@link AbstractWorker} is going to send. + * @return the selected {@link WorkerRef} + */ + @Override + public WorkerRef select(List members, Object message) { + if (message instanceof AbstractHashMessage) { + AbstractHashMessage hashMessage = (AbstractHashMessage)message; + int size = members.size(); + int selectIndex = Math.abs(hashMessage.getHashCode()) % size; + return members.get(selectIndex); + } else { + throw new IllegalArgumentException("the message send into HashCodeSelector must implementation of AbstractHashMessage, the message object class is: " + message.getClass().getName()); + } + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/selector/RollingSelector.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/selector/RollingSelector.java new file mode 100644 index 000000000..85cdee3ca --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/selector/RollingSelector.java @@ -0,0 +1,50 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream.selector; + +import java.util.List; +import org.skywalking.apm.collector.stream.worker.AbstractWorker; +import org.skywalking.apm.collector.stream.worker.WorkerRef; + +/** + * The RollingSelector is a simple implementation of {@link WorkerSelector}. + * It choose {@link WorkerRef} nearly random, by round-robin. + * + * @author peng-yongsheng + * @since v3.0-2017 + */ +public class RollingSelector implements WorkerSelector { + + private int index = 0; + + /** + * Use round-robin to select {@link WorkerRef}. + * + * @param members given {@link WorkerRef} list, which size is greater than 0; + * @param message message the {@link AbstractWorker} is going to send. + * @return the selected {@link WorkerRef} + */ + @Override + public WorkerRef select(List members, Object message) { + int size = members.size(); + index++; + int selectIndex = Math.abs(index) % size; + return members.get(selectIndex); + } +} diff --git a/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/selector/WorkerSelector.java b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/selector/WorkerSelector.java new file mode 100644 index 000000000..a9b3463ae --- /dev/null +++ b/apm-collector/apm-collector-component/stream-component/src/main/java/org/skywalking/apm/collector/stream/selector/WorkerSelector.java @@ -0,0 +1,44 @@ +/* + * Copyright 2017, OpenSkywalking Organization All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Project repository: https://github.com/OpenSkywalking/skywalking + */ + +package org.skywalking.apm.collector.stream.selector; + +import java.util.List; +import org.skywalking.apm.collector.stream.worker.AbstractWorker; +import org.skywalking.apm.collector.stream.worker.WorkerRef; + +/** + * The WorkerSelector should be implemented by any class whose instances + * are intended to provide select a {@link WorkerRef} from a {@link WorkerRef} list. + *

+ * Actually, the WorkerRef is designed to provide a routing ability in the collector cluster + * + * @author peng-yongsheng + * @since v3.0-2017 + */ +public interface WorkerSelector { + + /** + * select a {@link WorkerRef} from a {@link WorkerRef} list. + * + * @param members given {@link WorkerRef} list, which size is greater than 0; + * @param message the {@link AbstractWorker} is going to send. + * @return the selected {@link WorkerRef} + */ + T select(List members, Object message); +}