diff --git a/Makefile b/Makefile index f18e728066..3caefdf20f 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,7 @@ DOCKER_BUILD_TOP:=${CONTEXT}/docker_build HUB ?= skywalking OAP_NAME ?= oap UI_NAME ?= ui +DATA_GENERATOR_NAME ?= data-generator TAG ?= latest .PHONY: docker docker.all @@ -57,6 +58,7 @@ BUILD_ARGS := $(BUILD_ARGS) --build-arg DIST=$(DIST) --build-arg SKYWALKING_CLI_ %.ui: NAME = $(UI_NAME) %.oap: NAME = $(OAP_NAME) +%.data-generator: NAME = $(DATA_GENERATOR_NAME) docker.%: PLATFORMS = docker.%: LOAD_OR_PUSH = --load @@ -86,4 +88,4 @@ define DOCKER_RULE -t $(HUB)/$(NAME):latest \ $(DOCKER_BUILD_TOP)/$(NAME) docker buildx rm skywalking_main || true -endef \ No newline at end of file +endef diff --git a/apm-dist/src/main/assembly/binary.xml b/apm-dist/src/main/assembly/binary.xml index 9fd3e92a3e..d067970f74 100644 --- a/apm-dist/src/main/assembly/binary.xml +++ b/apm-dist/src/main/assembly/binary.xml @@ -82,6 +82,31 @@ oap-libs + + + ${project.basedir}/../oap-server/server-tools/data-generator/target/ + + data-generator-${project.version}.jar + + oap-libs + + + ${project.basedir}/../oap-server/server-tools/data-generator/src/main/assembly/bin + tools/data-generator/bin + + *.sh + + 0755 + + + ${project.basedir}/../oap-server/server-tools/data-generator/src/main/resources + tools/data-generator/config + + application.yml + + + + ${project.basedir}/../tools/profile-exporter diff --git a/docker/data-generator/Dockerfile b/docker/data-generator/Dockerfile new file mode 100644 index 0000000000..9f9a58152b --- /dev/null +++ b/docker/data-generator/Dockerfile @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG BASE_IMAGE='eclipse-temurin:11-jre' + +ARG SKYWALKING_CLI_VERSION + +FROM apache/skywalking-cli:$SKYWALKING_CLI_VERSION as cli + +FROM $BASE_IMAGE + +WORKDIR /skywalking + +ENV JAVA_OPTS=" -Xms2G " + +ARG DIST + +COPY "$DIST" . + +RUN set -ex; \ + tar -xzf "$DIST" --strip 1; \ + rm -rf "$DIST"; \ + rm -rf "config/log4j2.xml"; \ + rm -rf "bin"; \ + rm -rf "webapp"; \ + rm -rf "agent"; \ + mkdir "bin"; + +COPY --from=cli /swctl ./bin + +COPY log4j2.xml config/ +COPY docker-entrypoint.sh . + +RUN mkdir ext-config; \ + mkdir ext-libs; + +EXPOSE 12800 11800 1234 + +ENTRYPOINT ["bash", "docker-entrypoint.sh"] diff --git a/docker/data-generator/docker-entrypoint.sh b/docker/data-generator/docker-entrypoint.sh new file mode 100755 index 0000000000..a6c986d303 --- /dev/null +++ b/docker/data-generator/docker-entrypoint.sh @@ -0,0 +1,46 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +echo "[Entrypoint] Apache SkyWalking Data Generator Docker Image" + +EXT_LIB_DIR=/skywalking/ext-libs +EXT_CONFIG_DIR=/skywalking/ext-config + +# Override default application.yml with application.yml in the data generator module + +cp -vfRL tools/data-generator/config/application.yml config/ + +# Override configuration files +if [ "$(ls -A $EXT_CONFIG_DIR)" ]; then + cp -vfRL ${EXT_CONFIG_DIR}/* config/ +fi + +CLASSPATH="config:$CLASSPATH" +for i in oap-libs/*.jar +do + CLASSPATH="$i:$CLASSPATH" +done +for i in "${EXT_LIB_DIR}"/*.jar +do + CLASSPATH="$i:$CLASSPATH" +done + +set -ex + +exec java ${JAVA_OPTS} -classpath ${CLASSPATH} org.apache.skywalking.starter.DataGeneratorStartUp "$@" diff --git a/docker/data-generator/log4j2.xml b/docker/data-generator/log4j2.xml new file mode 100644 index 0000000000..6152fc75c2 --- /dev/null +++ b/docker/data-generator/log4j2.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + diff --git a/docs/en/changes/changes-9.0.0.md b/docs/en/changes/changes-9.0.0.md index 933e2cde50..d3b59d325c 100644 --- a/docs/en/changes/changes-9.0.0.md +++ b/docs/en/changes/changes-9.0.0.md @@ -125,7 +125,7 @@ * Add `Column.shardingKeyIdx` for column definition for BanyanDB. ``` -Sharding key is used to group time series data per metric of one entity in one place (same sharding and/or same +Sharding key is used to group time series data per metric of one entity in one place (same sharding and/or same row for column-oriented database). For example, ServiceA's traffic gauge, service call per minute, includes following timestamp values, then it should be sharded by service ID diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 5d22d10b95..08684ac452 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -27,6 +27,7 @@ priority**](https://github.com/apache/skywalking/pull/8664) to avoid consuming issues. * Fix the problem that some configurations (such as group.id) did not take effect due to the override order when using the kafkaConsumerConfig property to extend the configuration in Kafka Fetcher. * Remove build time from the OAP version. +* Add data-generator module to run OAP in testing mode, generating mock data for testing. #### UI diff --git a/docs/en/setup/backend/backend-data-generator.md b/docs/en/setup/backend/backend-data-generator.md new file mode 100644 index 0000000000..a2084bd27b --- /dev/null +++ b/docs/en/setup/backend/backend-data-generator.md @@ -0,0 +1,207 @@ +# Mock data generator for testing + +In 9.1.0, SkyWalking adds a module to generate mock data for testing. You can use this module to generate +mock data that will be sent to the storage. + +To start the data generator, execute the script `tools/data-generator/bin/start.sh`. + +Note that SkyWalking doesn't release a Docker image for this module, but you can still build it yourselves +by running the commands: + +```shell +# build a Docker image for local use +make docker.data-generator + +# or push to your registry +export HUB= +make push.docker.data-generator +``` + +Currently the module can generate two kinds of SkyWalking data, segments and logs. For each type, +there are some generators that can be used to fill the fields. + +## Generate mock data + +To generate mock data, `POST` a request to URL path `/mock-data/segments/tasks` (segments) or +`/mock-data/logs/tasks` (logs) with a generator template: + +```shell +curl -XPOST 'http://localhost:12800/mock-data/segments/tasks?size=20' -H'Content-Type: application/json' -d "@segment-template.json" + +curl -XPOST 'http://localhost:12800/mock-data/logs/tasks?size=20' -H'Content-Type: application/json' -d "@logs-template.json" +``` + +There are two possible types of task to generate mock data, `size` and `qps`: + +- `size` (`/mock-data/segments/tasks?size=20`): the task will generate total number of `size` segments/logs and then finish. +- `qps` (`/mock-data/segments/tasks?qps=20`): the task will generate `qps` segments/logs per second continuously, until the task is [cancelled](#cancel-a-task). + +Refer to [the segment template](segment-template.json), [the log template](logs-template.json) and the [Generators](#generators) for more details +about how to compose a template. + +## Cancel a task + +When the task is acknowledged by the server it will return a task id that can be used to cancelled +the task by sending a `DELETE` request to URL path `/mock-data/logs/tasks` with a parameter `requestId` (i.e. +`/mock-data/logs/tasks?requestId={request id returned in previous request}`): + +```shell +curl -XDELETE 'http://localhost:12800/mock-data/segments/task?requestId=70d8a39e-b51e-49de-a6fc-43abf80482c1' +curl -XDELETE 'http://localhost:12800/mock-data/logs/task?requestId=70d8a39e-b51e-49de-a6fc-43abf80482c1' +``` + +## Cancel all tasks + +When needed, you can also send a `DELETE` request to path `/mock-data/segments/tasks` to cancel all segment tasks. + +```shell +curl -XDELETE 'http://localhost:12800/mock-data/segments/tasks +curl -XDELETE 'http://localhost:12800/mock-data/logs/tasks +``` + +## Generators + +### `uuid` + +`uuid` generator leverages `java.util.UUID` to generate a string. You can use `uuid` generator to fill the +`traceId` field of segments. + +`changingFrequency` property can be used when you want to reuse a `uuid` for multiple times, for example, +if you want a `traceId` to be reused by 5 segments, then setting `changingFrequency` to `5` would do the trick. +By setting `changingFrequency` to `5`, `uuid` generates 1 string, and uses it for 5 times, then re-generates +a new uuid string and uses it for another 5 times. + +```json +"traceId": { + "type": "uuid", + "changingFrequency": "5" +} +``` + +### `randomString` (`String`) + +#### `length` (`int`) + +`length` specifies the length of the random string to be generated, +i.e. `generatedString.length() == length` is always `true`. + +#### `prefix` (`String`) + +`prefix` is always added to the random strings **after** they are generated, that means: + +- `generatedString.startsWith(prefix)` is always `true`, and, +- `generatedString.length() == length + prefix.length()` is always true. + +#### `letters` (`boolean`) + +Specifies whether the random string contains letters (i.e. `a-zA-Z`). + +#### `numbers` (`boolean`) + +Specifies whether the random string contains numbers (i.e. `0-9`). + +#### `domainSize` (`int`) + +When generating random strings, you might just want some random strings and use them over and over again randomly, +by setting `domainSize`, the generator generates `domainSize` random strings, and pick them randomly every time +you need a string. + +### `randomBool` (`boolean`) + +This generator generates a `Boolean` value, `true` or `false` with a default possibility of 50%, while you can change the `possibility` below. + +#### `possibility` (`double`, `[0, 1]`) + +`possibility` is a `double` value `>= 0` and `<= 1`, it's `0.5` by default, meaning **about** half of the generated values are `true`. + +To always return a fixed boolean value `true`, you can just set the `possibility` to `1`, to always return a fixed boolean value `false`, you can set the `possibility` to `0` + +```json +"error": { + "type": "randomBool", + "possibility": "0.9" +} +``` + +> 90 percent of the generated values are `true`. + +### `randomInt` (`long`) + +#### `min` (`long`) + +The minimum value of the random integers, meaning all generated values satisfy `generatedInt >= min`. + +#### `max` (`long`) + +The maximum value of the random integers, meaning all generated values satisfy `generatedInt < min`. + +#### `domainSize` (`int`) + +This is similar to [`randomString`'s `domainSize`](#domainsize-int). + +### `randomList` (`list` / `array`) + +#### `size` (`int`) + +The list size of the generated list, i.e. `generatedList.size() == size`. + +#### `item` (`object`) + +`item` is a template that will be use as a prototype to generate the list items, for example when generating a +list of `Tag`, the `item` should be the prototype of `Tag`, which can be composed by the generators again. + +```json +"tags": { + "type": "randomList", + "size": 5, + "item": { + "key": { + "type": "randomString", + "length": "10", + "prefix": "test_tag_", + "letters": true, + "numbers": true, + "domainSize": 10 + }, + "value": { + "type": "randomString", + "length": "10", + "prefix": "test_value_", + "letters": true, + "numbers": true + } + } +} +``` + +### `fixedString` (`string`) + +This generator always returns a fixed `value` of string. + +### `sequence` (`long`) + +`sequence` generator generates a sequence of monotonically increasing integers, with a configurable `fluctuation`. + +#### `min` (`long`) + +The minimum value of the sequence. + +#### `max` (`long`) + +The maximum value of the sequence. + +#### `step` (`long`) + +The increasing step of this sequence, i.e. `the next generated value == the previous value + step`. + +#### `domainSize` (`int`) + +This is similar to [`randomString`'s `domainSize`](#domainsize-int). + +#### `fluctuation` (`int`) + +By default, sequence is strictly increasing numbers, but in some cases you might want the numbers to fluctuate +slightly while they are increasing. Adding property `fluctuation` to the generator will add a random number +`>= -fluctuation, <= fluctuation` to the sequence elements. + +For example, `min = 10, max = 15, step = 1` generates a sequence `[10, 11, 12, 13, 14, 15]`, but adding `fluctuation = 2` **might** generate a sequence `[10, 12, 11, 14, 13, 15]`. diff --git a/docs/en/setup/backend/logs-template.json b/docs/en/setup/backend/logs-template.json new file mode 100644 index 0000000000..28308b32db --- /dev/null +++ b/docs/en/setup/backend/logs-template.json @@ -0,0 +1,83 @@ +{ + "timestamp": { + "type": "sequence", + "min": "1649643929000", + "max": "1649653929000" + }, + "serviceName": { + "type": "randomString", + "length": "20", + "prefix": "test_svc_name_", + "letters": true, + "numbers": true + }, + "serviceInstanceName": { + "type": "randomString", + "length": "20", + "prefix": "test_svc_inst_name_", + "letters": true, + "numbers": true + }, + "endpointName": { + "type": "randomString", + "length": "20", + "prefix": "test_endpoint_", + "letters": true, + "numbers": true + }, + "traceId": { + "type": "randomString", + "length": "20", + "prefix": "test_trace_id_", + "letters": true, + "numbers": true + }, + "traceSegmentId": { + "type": "randomString", + "length": "20", + "prefix": "test", + "letters": true, + "numbers": true + }, + "spanId": { + "type": "randomInt", + "min": "0", + "max": "5" + }, + "contentType": { + "type": "randomInt", + "min": 1, + "max": 1 + }, + "content": { + "type": "randomString", + "length": "10", + "prefix": "test", + "letters": true, + "numbers": true + }, + "error": { + "type": "randomBool" + }, + "tags": { + "type": "randomList", + "size": 5, + "item": { + "key": { + "type": "randomString", + "length": "10", + "prefix": "test", + "letters": true, + "numbers": true, + "domainSize": 10 + }, + "value": { + "type": "randomString", + "length": "10", + "prefix": "test", + "letters": true, + "numbers": true + } + } + } +} diff --git a/docs/en/setup/backend/segment-template.json b/docs/en/setup/backend/segment-template.json new file mode 100644 index 0000000000..88b1e5cfc6 --- /dev/null +++ b/docs/en/setup/backend/segment-template.json @@ -0,0 +1,145 @@ +{ + "traceId": { + "type": "uuid", + "changingFrequency": "5" + }, + "serviceInstanceName": { + "type": "randomString", + "length": "10", + "letters": true, + "numbers": true + }, + "segmentId": { + "type": "randomString", + "length": "20", + "letters": true, + "numbers": true + }, + "serviceName": { + "type": "randomString", + "length": "10", + "letters": true, + "numbers": true + }, + "endpointName": { + "type": "randomString", + "length": "10", + "prefix": "test_", + "letters": true, + "numbers": true, + "domainSize": 10 + }, + "startTime": { + "type": "sequence", + "min": "1650510807000", + "max": "1650512807000" + }, + "latency": { + "type": "randomInt", + "min": 100, + "max": 1000 + }, + "error": { + "type": "randomInt", + "min": 1, + "max": 1 + }, + "tags": { + "type": "randomList", + "size": 5, + "item": { + "key": { + "type": "randomString", + "length": "10", + "prefix": "test_tag_", + "letters": true, + "numbers": true, + "domainSize": 10 + }, + "value": { + "type": "randomString", + "length": "10", + "prefix": "test_value_", + "letters": true, + "numbers": true + } + } + }, + "spans": { + "type": "randomList", + "size": 5, + "item": { + "spanId": { + "type": "sequence", + "min": "0", + "max": "5" + }, + "parentSpanId": { + "type": "sequence", + "min": "-1", + "max": "4" + }, + "startTime": { + "type": "randomInt", + "min": "1650510807000", + "max": "1650512807000", + "domainSize": 10 + }, + "endTime": { + "type": "randomInt", + "min": "1650510807000", + "max": "1650512807000", + "domainSize": 10 + }, + "operationName": { + "type": "randomString", + "length": "10", + "prefix": "test_endpoint_", + "letters": true, + "numbers": true + }, + "peer": { + "type": "randomString", + "length": "10", + "prefix": "test_peer_", + "letters": true, + "numbers": true + }, + "spanLayer": { + "type": "randomInt", + "min": "0", + "max": "4" + }, + "componentId": { + "type": "randomInt", + "min": "0", + "max": "4" + }, + "error": { + "type": "randomBool", + "possibility": "0.9" + }, + "tags": { + "type": "randomList", + "size": 5, + "item": { + "key": { + "type": "randomString", + "length": "10", + "prefix": "test_tag_key_", + "letters": true, + "numbers": true, + "domainSize": 10 + }, + "value": { + "type": "randomString", + "length": "10", + "prefix": "test_tag_val_", + "letters": true, + "numbers": true + } + } + } + } + } +} diff --git a/docs/menu.yml b/docs/menu.yml index 7d3d19d5df..4854a72525 100644 --- a/docs/menu.yml +++ b/docs/menu.yml @@ -179,6 +179,8 @@ catalog: path: "/en/setup/backend/backend-health-check" - name: "CLI Setup" path: "https://github.com/apache/skywalking-cli" + - name: "Mock Data Generator Setup" + path: "/en/setup/backend/backend-data-generator" - name: "Contributing Guides" catalog: - name: "Contribute" @@ -237,7 +239,3 @@ catalog: path: "/en/changes/changes-6.x" - name: "5.x" path: "/en/changes/changes-5.x" - - - - diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Segment.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Segment.java index 6249be8464..fc2bd8a0e7 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Segment.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Segment.java @@ -22,10 +22,12 @@ import java.util.ArrayList; import java.util.List; import lombok.Getter; import lombok.Setter; +import lombok.ToString; import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SEGMENT; +@ToString @ScopeDeclaration(id = SEGMENT, name = "Segment") public class Segment extends Source { diff --git a/oap-server/server-library/library-server/src/main/java/org/apache/skywalking/oap/server/library/server/http/HTTPServer.java b/oap-server/server-library/library-server/src/main/java/org/apache/skywalking/oap/server/library/server/http/HTTPServer.java index 7f59096c57..f6749e100f 100644 --- a/oap-server/server-library/library-server/src/main/java/org/apache/skywalking/oap/server/library/server/http/HTTPServer.java +++ b/oap-server/server-library/library-server/src/main/java/org/apache/skywalking/oap/server/library/server/http/HTTPServer.java @@ -60,7 +60,7 @@ public class HTTPServer implements Server { .http1MaxHeaderSize(config.getMaxRequestHeaderSize()) .idleTimeout(Duration.ofMillis(config.getIdleTimeOut())) .decorator(Route.ofCatchAll(), (delegate, ctx, req) -> { - if (!this.allowedMethods.contains(ctx.method())) { + if (!allowedMethods.contains(ctx.method())) { return HttpResponse.of(HttpStatus.METHOD_NOT_ALLOWED); } return delegate.serve(ctx, req); diff --git a/oap-server/server-tools/data-generator/pom.xml b/oap-server/server-tools/data-generator/pom.xml new file mode 100644 index 0000000000..ce9fced371 --- /dev/null +++ b/oap-server/server-tools/data-generator/pom.xml @@ -0,0 +1,55 @@ + + + + + 4.0.0 + + server-tools + org.apache.skywalking + 9.1.0-SNAPSHOT + + data-generator + data-generator + + Module to generate data for testing. + + + + + org.apache.skywalking + server-starter + ${project.version} + + + + + + + maven-jar-plugin + + + application.yml + + + + + + diff --git a/oap-server/server-tools/data-generator/src/main/assembly/bin/start.sh b/oap-server/server-tools/data-generator/src/main/assembly/bin/start.sh new file mode 100644 index 0000000000..da1a78d295 --- /dev/null +++ b/oap-server/server-tools/data-generator/src/main/assembly/bin/start.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +PRG="$0" +PRGDIR=$(dirname "$PRG") +[ -z "$OAP_HOME" ] && OAP_HOME=$(cd "$PRGDIR/../../.." > /dev/null || exit 1; pwd) + +OAP_LOG_DIR="${OAP_LOG_DIR:-${OAP_HOME}/logs}" +JAVA_OPTS="${JAVA_OPTS:- -Xms256M -Xmx512M}" + +if [ ! -d "${OAP_LOG_DIR}" ]; then + mkdir -p "${OAP_LOG_DIR}" +fi + +_RUNJAVA=${JAVA_HOME}/bin/java +[ -z "$JAVA_HOME" ] && _RUNJAVA=java + +CLASSPATH="$OAP_HOME/tools/data-generator/config:$OAP_HOME/config:$CLASSPATH" +for i in "$OAP_HOME"/oap-libs/*.jar +do + CLASSPATH="$i:$CLASSPATH" +done + +OAP_OPTIONS=" -Doap.logDir=${OAP_LOG_DIR}" + +eval exec "\"$_RUNJAVA\" ${JAVA_OPTS} ${OAP_OPTIONS} -classpath $CLASSPATH org.apache.skywalking.starter.DataGeneratorStartUp \ + 2>${OAP_LOG_DIR}/oap.log 1> /dev/null &" + +if [ $? -eq 0 ]; then + sleep 1 + echo "SkyWalking OAP started successfully!" +else + echo "SkyWalking OAP started failure!" + exit 1 +fi diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/BoolGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/BoolGenerator.java new file mode 100644 index 0000000000..7bda23aa7a --- /dev/null +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/BoolGenerator.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.generator; + +import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import lombok.Data; + +@JsonDeserialize(builder = BoolGenerator.Builder.class) +public final class BoolGenerator implements Generator { + private final Random random = ThreadLocalRandom.current(); + private final double possibility; + + public BoolGenerator(Builder builder) { + possibility = builder.possibility; + } + + @Override + public Boolean next() { + return random.nextDouble() < possibility; + } + + @Override + public String toString() { + return String.valueOf(next()); + } + + @Data + public static class Builder { + private double possibility = .5; + + public BoolGenerator build() { + return new BoolGenerator(this); + } + } +} diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/FixedStringGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/FixedStringGenerator.java new file mode 100644 index 0000000000..edf11a54ce --- /dev/null +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/FixedStringGenerator.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.generator; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import lombok.Data; + +@JsonDeserialize(builder = FixedStringGenerator.Builder.class) +public final class FixedStringGenerator implements Generator { + private final String value; + + public FixedStringGenerator(Builder builder) { + value = builder.value; + } + + @Override + public String next() { + return value; + } + + @Data + public static class Builder { + private String value; + + public FixedStringGenerator build() { + return new FixedStringGenerator(this); + } + } + + @Override + public String toString() { + return String.valueOf(next()); + } +} diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/Generator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/Generator.java new file mode 100644 index 0000000000..511f18873c --- /dev/null +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/Generator.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.generator; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type") +@JsonSubTypes({ + @Type(value = StringGenerator.class, name = "randomString"), + @Type(value = BoolGenerator.class, name = "randomBool"), + @Type(value = IntGenerator.class, name = "randomInt"), + @Type(value = ListGenerator.class, name = "randomList"), + @Type(value = FixedStringGenerator.class, name = "fixedString"), + @Type(value = SequenceGenerator.class, name = "sequence"), + @Type(value = UUIDGenerator.class, name = "uuid"), +}) +public interface Generator { + public T next(); + + default public void reset() { + } +} diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/IntGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/IntGenerator.java new file mode 100644 index 0000000000..bd2b5a6e5e --- /dev/null +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/IntGenerator.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.generator; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ThreadLocalRandom; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.google.common.base.Preconditions; +import lombok.Data; + +@JsonDeserialize(builder = IntGenerator.Builder.class) +public final class IntGenerator implements Generator { + private final boolean limitedDomain; + private final Long min; + private final Long max; + private final Integer domainSize; + private final ThreadLocalRandom random = ThreadLocalRandom.current(); + private final Set domain = new HashSet<>(); + + public IntGenerator(Builder builder) { + min = builder.min; + max = builder.max; + domainSize = builder.domainSize; + limitedDomain = builder.domainSize != null && builder.domainSize > 0; + + reset(); + } + + @Override + public Long next() { + if (!limitedDomain) { + return next0(); + } + return domain + .stream() + .skip(random.nextInt(domain.size())) + .findFirst() + .orElseThrow(() -> new RuntimeException("Should never happen")); + } + + @Override + public void reset() { + if (limitedDomain) { + domain.clear(); + + while (domain.size() < domainSize) { + domain.add(next0()); + } + } + } + + private long next0() { + if (min != null && max != null) { + return random.nextLong(max - min + 1) + min; // Might overflow, but it's not worthy to check + } + if (min != null) { + return Math.abs(random.nextLong()) + min; + } + return random.nextLong(max); + } + + @Override + public String toString() { + return String.valueOf(next()); + } + + @Data + public static class Builder { + private Long min; + private Long max; + private Integer domainSize; + + public IntGenerator build() { + if (min != null && max != null) { + Preconditions.checkArgument(min <= max, "min must be <= max"); + if (domainSize != null) { + Preconditions.checkArgument(domainSize <= max - min, + "domain size must be <= max - min"); + } + } + return new IntGenerator(this); + } + } +} diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/ListGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/ListGenerator.java new file mode 100644 index 0000000000..07974cd3c7 --- /dev/null +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/ListGenerator.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.generator; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import lombok.Data; + +@JsonDeserialize(builder = ListGenerator.Builder.class) +public final class ListGenerator implements Generator> { + private final T item; + private final int size; + + public ListGenerator(Builder builder) { + item = builder.item; + size = builder.size; + } + + @Override + public List next() { + return IntStream + .range(0, size) + .mapToObj($ -> item) + .collect(Collectors.toList()); + } + + @Override + public void reset() { + ((Generator) item).reset(); + } + + @Data + public static class Builder { + private int size; + private T item; + + public ListGenerator build() { + return new ListGenerator<>(this); + } + } +} diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/SequenceGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/SequenceGenerator.java new file mode 100644 index 0000000000..3df3f9246a --- /dev/null +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/SequenceGenerator.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.generator; + +import java.util.HashSet; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.ThreadLocalRandom; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.google.common.base.Preconditions; +import lombok.Data; +import lombok.experimental.Accessors; + +@JsonDeserialize(builder = SequenceGenerator.Builder.class) +public final class SequenceGenerator implements Generator { + private final boolean limitedDomain; + private final long min; + private final long max; + private final long step; + private final Integer fluctuation; + private final Integer domainSize; + private final Random random = ThreadLocalRandom.current(); + private final Set domain = new HashSet<>(); + private volatile Long last; + + public SequenceGenerator(Builder builder) { + min = builder.min; + max = builder.max; + step = builder.step; + fluctuation = builder.fluctuation; + domainSize = builder.domainSize; + limitedDomain = builder.domainSize != null && builder.domainSize > 0; + + reset(); + } + + @Override + public Long next() { + if (!limitedDomain) { + return next0(); + } + return domain + .stream() + .skip(random.nextInt(domain.size())) + .findFirst() + .orElseThrow(() -> new RuntimeException("Should never happen")); + } + + private synchronized long next0() { + long next = last == null ? min : last + step; + + if (fluctuation != null) { + int j = random.nextInt(fluctuation); + next += random.nextBoolean() ? j : -j; + } + + if (next > max) { + return max; + } + if (next < min) { + return min; + } + + return last = next; + } + + @Override + public void reset() { + last = null; + if (limitedDomain) { + domain.clear(); + while (domain.size() < domainSize) { + domain.add(next0()); + } + } + } + + @Override + public String toString() { + return String.valueOf(next()); + } + + @Data + @Accessors(chain = true) + public static class Builder { + private long min = Long.MIN_VALUE; + private long max = Long.MAX_VALUE; + private long step = 1; + private Integer domainSize; + private Integer fluctuation; + + public SequenceGenerator build() { + if (domainSize != null) { + Preconditions.checkArgument(domainSize > 0, "domainSize must be > 0"); + Preconditions.checkArgument(domainSize + min <= max, + "domain size must be <= max - min"); + } + return new SequenceGenerator(this); + } + } +} diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/StringGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/StringGenerator.java new file mode 100644 index 0000000000..06a6bfb770 --- /dev/null +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/StringGenerator.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.generator; + +import java.util.HashSet; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.ThreadLocalRandom; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.google.common.base.Strings; +import org.apache.commons.lang3.RandomStringUtils; +import lombok.Data; + +@JsonDeserialize(builder = StringGenerator.Builder.class) +public final class StringGenerator implements Generator { + private final int length; + private final String prefix; + private final boolean letters; + private final boolean numbers; + private final boolean limitedDomain; + private final Random random = ThreadLocalRandom.current(); + private final Set domain = new HashSet<>(); + + public StringGenerator(Builder builder) { + length = builder.length; + prefix = builder.prefix; + letters = builder.letters; + numbers = builder.numbers; + limitedDomain = builder.domainSize > 0; + + if (limitedDomain) { + while (domain.size() < builder.domainSize) { + final String r = RandomStringUtils.random(length, letters, numbers); + if (!Strings.isNullOrEmpty(builder.prefix)) { + domain.add(builder.prefix + r); + } else { + domain.add(r); + } + } + } + } + + @Override + public String next() { + if (!limitedDomain) { + return Strings.nullToEmpty(prefix) + + RandomStringUtils.random(length, letters, numbers); + } + return domain + .stream() + .skip(random.nextInt(domain.size())) + .findFirst() + .orElseThrow(() -> new RuntimeException("Should never happen")); + } + + @Override + public String toString() { + return String.valueOf(next()); + } + + @Data + public static class Builder { + private int length; + private String prefix; + private boolean letters; + private boolean numbers; + private int domainSize; + + public StringGenerator build() { + return new StringGenerator(this); + } + } +} diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/UUIDGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/UUIDGenerator.java new file mode 100644 index 0000000000..22a75efe8b --- /dev/null +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/UUIDGenerator.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.generator; + +import static com.google.common.base.Preconditions.checkArgument; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import lombok.Data; + +@JsonDeserialize(builder = UUIDGenerator.Builder.class) +public final class UUIDGenerator implements Generator { + private final int changingFrequency; + private final AtomicInteger counter; + private final AtomicReference last = + new AtomicReference<>(UUID.randomUUID().toString()); + + public UUIDGenerator(Builder builder) { + checkArgument(builder.changingFrequency > 0, "changingFrequency must be greater than 0"); + changingFrequency = builder.changingFrequency; + counter = new AtomicInteger(); + reset(); + } + + @Override + public String next() { + if (counter.incrementAndGet() < changingFrequency) { + return last.get(); + } + reset(); + return last.getAndSet(UUID.randomUUID().toString()); + } + + @Override + public String toString() { + return String.valueOf(next()); + } + + @Override + public void reset() { + counter.set(0); + } + + @Data + public static class Builder { + private int changingFrequency = 1; + + public UUIDGenerator build() { + return new UUIDGenerator(this); + } + } +} diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/module/DataGeneratorModule.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/module/DataGeneratorModule.java new file mode 100644 index 0000000000..044f91e5df --- /dev/null +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/module/DataGeneratorModule.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.module; + +import org.apache.skywalking.oap.server.library.module.ModuleDefine; + +public class DataGeneratorModule extends ModuleDefine { + public static final String NAME = "data-generator"; + + public DataGeneratorModule() { + super(NAME); + } + + @Override + public Class[] services() { + return new Class[0]; + } +} diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/module/DataGeneratorModuleProvider.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/module/DataGeneratorModuleProvider.java new file mode 100644 index 0000000000..23ce2a6fa3 --- /dev/null +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/module/DataGeneratorModuleProvider.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.module; + +import java.util.Arrays; +import java.util.List; +import com.linecorp.armeria.common.HttpMethod; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.server.HTTPHandlerRegister; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; +import org.apache.skywalking.oap.server.library.module.ModuleDefine; +import org.apache.skywalking.oap.server.library.module.ModuleProvider; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; +import org.apache.skywalking.restapi.LogGeneratorHandler; +import org.apache.skywalking.restapi.SegmentGeneratorHandler; + +public class DataGeneratorModuleProvider extends ModuleProvider { + + @Override + public String name() { + return "default"; + } + + @Override + public Class module() { + return DataGeneratorModule.class; + } + + @Override + public ModuleConfig createConfigBeanIfAbsent() { + return null; + } + + @Override + public void prepare() throws ServiceNotProvidedException, ModuleStartException { + } + + @Override + public void start() throws ServiceNotProvidedException, ModuleStartException { + final List methods = Arrays.asList( + HttpMethod.GET, + HttpMethod.POST, + HttpMethod.DELETE + ); + final HTTPHandlerRegister register = + getManager() + .find(CoreModule.NAME) + .provider() + .getService(HTTPHandlerRegister.class); + register.addHandler(new SegmentGeneratorHandler(getManager()), methods); + register.addHandler(new LogGeneratorHandler(getManager()), methods); + } + + @Override + public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException { + } + + @Override + public String[] requiredModules() { + return new String[0]; + } +} diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/LogGeneratorHandler.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/LogGeneratorHandler.java new file mode 100644 index 0000000000..303bed64f2 --- /dev/null +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/LogGeneratorHandler.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.restapi; + +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.function.IntConsumer; +import java.util.stream.IntStream; +import com.linecorp.armeria.common.HttpResponse; +import com.linecorp.armeria.common.HttpStatus; +import com.linecorp.armeria.common.MediaType; +import com.linecorp.armeria.common.util.EventLoopGroups; +import com.linecorp.armeria.server.annotation.Default; +import com.linecorp.armeria.server.annotation.Delete; +import com.linecorp.armeria.server.annotation.Get; +import com.linecorp.armeria.server.annotation.Param; +import com.linecorp.armeria.server.annotation.Post; +import com.linecorp.armeria.server.annotation.ProducesJson; +import com.linecorp.armeria.server.annotation.RequestObject; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.source.Log; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import io.netty.channel.EventLoopGroup; +import io.netty.util.concurrent.Future; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class LogGeneratorHandler { + private final SourceReceiver sourceReceiver; + private final Map> futures = new ConcurrentHashMap<>(); + private final EventLoopGroup eventLoopGroup = EventLoopGroups.newEventLoopGroup(10); + + public LogGeneratorHandler(ModuleManager manager) { + sourceReceiver = manager.find(CoreModule.NAME).provider().getService(SourceReceiver.class); + } + + @Post("/mock-data/logs/tasks") + public HttpResponse generateMockLogs( + @Default("0") @Param("size") int size, + @Default("0") @Param("qps") int qps, + @RequestObject LogRequest request) { + + if (size > 0 && qps > 0) { + return HttpResponse.of( + HttpStatus.BAD_REQUEST, + MediaType.PLAIN_TEXT, + "size and qps can't be both set"); + } + log.info("Generate {} mock logs, qps: {}, template: {}", size, qps, request); + + final IntConsumer generator = unused -> { + final Log l = request.next(); + log.debug("Generating log: {}", l); + sourceReceiver.receive(l); + }; + final String requestId = UUID.randomUUID().toString(); + final Future future; + if (size > 0) { + future = eventLoopGroup.submit(() -> IntStream + .range(0, size) + .forEach(generator)); + } else { + future = eventLoopGroup.scheduleAtFixedRate(() -> IntStream + .range(0, qps) + .forEach(generator), 0, 1, TimeUnit.SECONDS); + } + + futures.put(requestId, future); + + future.addListener(f -> { + if (f.isDone()) { + futures.remove(requestId); + log.info("Generate mock logs finished: {}, requestId: {}", f.isSuccess(), + requestId); + } + if (f.cause() != null) { + log.error(f.cause().getMessage(), f.cause()); + } + }); + + return HttpResponse.of(MediaType.PLAIN_TEXT, requestId); + } + + @Delete("/mock-data/logs/task") + public HttpResponse cancelRequest(@Param("requestId") String requestId) { + final Future future = futures.get(requestId); + if (future == null) { + return HttpResponse.of( + HttpStatus.NOT_FOUND, + MediaType.PLAIN_TEXT_UTF_8, + "No such request: %s", requestId); + } + log.info("Cancelling request: {}", requestId); + future.cancel(true); + return HttpResponse.of(HttpStatus.OK); + } + + @Delete("/mock-data/logs/tasks") + public HttpResponse cancelAllRequests() { + futures.forEach((t, u) -> { + log.info("Cancelling request: {}", t); + u.cancel(true); + }); + return HttpResponse.of(HttpStatus.OK); + } + + @ProducesJson + @Get("/mock-data/logs/tasks") + public Set listRequest() { + return futures.keySet(); + } +} diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/LogRequest.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/LogRequest.java new file mode 100644 index 0000000000..bf41e2bf49 --- /dev/null +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/LogRequest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.restapi; + +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import org.apache.skywalking.apm.network.logging.v3.LogTags; +import org.apache.skywalking.generator.Generator; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; +import org.apache.skywalking.oap.server.core.query.type.ContentType; +import org.apache.skywalking.oap.server.core.source.Log; +import lombok.Data; +import lombok.SneakyThrows; + +@Data +@JsonTypeInfo(use = JsonTypeInfo.Id.NONE) +public final class LogRequest implements Generator { + @JsonIgnore + private final ObjectMapper om = new ObjectMapper(); + + private Generator timestamp; + private Generator serviceName; + private Generator serviceInstanceName; + private Generator endpointName; + private Generator traceId; + private Generator traceSegmentId; + private Generator spanId; + private Generator contentType; + private Generator content; + private Generator> tags; + private Generator error; + + @SneakyThrows + @Override + public Log next() { + final Log log = new Log(); + log.setTimestamp(getTimestamp().next()); + log.setServiceId( + IDManager.ServiceID.buildId( + getServiceName().next(), + true)); + log.setServiceInstanceId( + IDManager.ServiceInstanceID.buildId( + log.getServiceId(), + getServiceInstanceName().next())); + log.setEndpointId( + IDManager.EndpointID.buildId( + log.getServiceId(), + getEndpointName().next())); + log.setTraceId(getTraceId().next()); + log.setTraceSegmentId(getTraceSegmentId().next()); + log.setSpanId(getSpanId().next().intValue()); + log.setContentType(ContentType.instanceOf(getContentType().next().intValue())); + log.setContent(getContent().next()); + log.setError(getError().next()); + log.setTimeBucket(TimeBucket.getRecordTimeBucket(log.getTimestamp())); + log.setTags( + getTags() + .next() + .stream() + .map(TagGenerator::next) + .collect(Collectors.toList())); + log.setTagsRawData( + LogTags + .newBuilder() + .addAllData( + log + .getTags() + .stream() + .map(it -> KeyStringValuePair + .newBuilder() + .setKey(it.getKey()) + .setValue(it.getValue()) + .build()) + .collect(Collectors.toList())) + .build() + .toByteArray()); + log.setUniqueId(UUID.randomUUID().toString()); + return log; + } +} diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGeneratorHandler.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGeneratorHandler.java new file mode 100644 index 0000000000..fb5a0db01e --- /dev/null +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGeneratorHandler.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.restapi; + +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.function.IntConsumer; +import java.util.stream.IntStream; +import com.linecorp.armeria.common.HttpResponse; +import com.linecorp.armeria.common.HttpStatus; +import com.linecorp.armeria.common.MediaType; +import com.linecorp.armeria.common.util.EventLoopGroups; +import com.linecorp.armeria.server.annotation.Default; +import com.linecorp.armeria.server.annotation.Delete; +import com.linecorp.armeria.server.annotation.Get; +import com.linecorp.armeria.server.annotation.Param; +import com.linecorp.armeria.server.annotation.Post; +import com.linecorp.armeria.server.annotation.ProducesJson; +import com.linecorp.armeria.server.annotation.RequestObject; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.source.Segment; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import io.netty.channel.EventLoopGroup; +import io.netty.util.concurrent.Future; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class SegmentGeneratorHandler { + private final SourceReceiver sourceReceiver; + private final Map> futures = new ConcurrentHashMap<>(); + private final EventLoopGroup eventLoopGroup = EventLoopGroups.newEventLoopGroup(10); + + public SegmentGeneratorHandler(ModuleManager manager) { + sourceReceiver = manager.find(CoreModule.NAME).provider().getService(SourceReceiver.class); + } + + @Post("/mock-data/segments/tasks") + public HttpResponse generateMockSegments( + @Default("0") @Param("size") int size, + @Default("0") @Param("qps") int qps, + @RequestObject SegmentRequest request) { + + if (size > 0 && qps > 0) { + return HttpResponse.of( + HttpStatus.BAD_REQUEST, + MediaType.PLAIN_TEXT, + "size and qps can't be both set"); + } + log.info("Generate {} mock segments, qps: {}, template: {}", size, qps, request); + + final IntConsumer generator = unused -> { + final Segment segment = request.next(); + log.debug("Generating segment: {}", segment); + sourceReceiver.receive(segment); + }; + final String requestId = UUID.randomUUID().toString(); + final Future future; + if (size > 0) { + future = eventLoopGroup.submit(() -> IntStream + .range(0, size) + .forEach(generator)); + } else { + future = eventLoopGroup.scheduleAtFixedRate(() -> IntStream + .range(0, qps) + .forEach(generator), 0, 1, TimeUnit.SECONDS); + } + + futures.put(requestId, future); + + future.addListener(f -> { + if (f.isDone()) { + futures.remove(requestId); + log.info("Generate mock segments finished: {}, requestId: {}", f.isSuccess(), + requestId); + } + if (f.cause() != null && !(f.cause() instanceof CancellationException)) { + log.error("Exception in future: ", f.cause()); + } + }); + + return HttpResponse.of(MediaType.PLAIN_TEXT, requestId); + } + + @Delete("/mock-data/segments/task") + public HttpResponse cancelRequest(@Param("requestId") String requestId) { + final Future future = futures.get(requestId); + if (future == null) { + return HttpResponse.of( + HttpStatus.NOT_FOUND, + MediaType.PLAIN_TEXT_UTF_8, + "No such request: %s", requestId); + } + log.info("Cancelling request: {}", requestId); + future.cancel(true); + return HttpResponse.of(HttpStatus.OK); + } + + @Delete("/mock-data/segments/tasks") + public HttpResponse cancelAllRequests() { + futures.forEach((t, u) -> { + log.info("Cancelling request: {}", t); + u.cancel(true); + }); + return HttpResponse.of(HttpStatus.OK); + } + + @ProducesJson + @Get("/mock-data/segments/tasks") + public Set listRequest() { + return futures.keySet(); + } +} diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentRequest.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentRequest.java new file mode 100644 index 0000000000..70d4d63167 --- /dev/null +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentRequest.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.restapi; + +import java.util.List; +import java.util.stream.Collectors; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; +import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; +import org.apache.skywalking.generator.Generator; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; +import org.apache.skywalking.oap.server.core.source.Segment; +import lombok.Data; + +@Data +@JsonTypeInfo(use = JsonTypeInfo.Id.NONE) +public final class SegmentRequest implements Generator { + private Generator segmentId; + private Generator traceId; + private Generator serviceName; + private Generator serviceInstanceName; + private Generator endpointName; + private Generator startTime; + private Generator latency; + private Generator error; + private Generator> tags; + private Generator> spans; + + @Override + public Segment next() { + final String traceId = getTraceId().next(); + final String serviceName = getServiceName().next(); + final String serviceInstanceName = getServiceInstanceName().next(); + final String endpointName = getEndpointName().next(); + final String segmentId = getSegmentId().next(); + + final SegmentObject segmentObj = SegmentObject + .newBuilder() + .setTraceId(traceId) + .setTraceSegmentId(segmentId) + .addAllSpans( + getSpans() + .next() + .stream() + .map(SpanGenerator::next) + .collect(Collectors.toList())) + .setService(serviceName) + .setServiceInstance(serviceInstanceName) + .build(); + + // Reset the span generator to generate the span id from 0 + getSpans().reset(); + + final Segment segment = new Segment(); + segment.setSegmentId(segmentId); + segment.setTraceId(traceId); + segment.setServiceId( + IDManager.ServiceID.buildId(serviceName, true)); + segment.setServiceInstanceId( + IDManager.ServiceInstanceID.buildId( + segment.getServiceId(), + serviceInstanceName)); + segment.setEndpointId( + IDManager.EndpointID.buildId( + segment.getServiceId(), + endpointName)); + segment.setStartTime(getStartTime().next()); + segment.setLatency(getLatency().next().intValue()); + segment.setIsError(getError().next().intValue()); + segment.setTimeBucket(TimeBucket.getRecordTimeBucket(segment.getStartTime())); + segment.setTags( + getTags() + .next() + .stream() + .map(TagGenerator::next) + .collect(Collectors.toList())); + + segment.setDataBinary(segmentObj.toByteArray()); + return segment; + } +} diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SpanGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SpanGenerator.java new file mode 100644 index 0000000000..b8af9aea77 --- /dev/null +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SpanGenerator.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.restapi; + +import java.util.List; +import java.util.stream.Collectors; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import org.apache.skywalking.apm.network.language.agent.v3.SpanLayer; +import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; +import org.apache.skywalking.generator.Generator; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +@JsonTypeInfo(use = JsonTypeInfo.Id.NONE) +public final class SpanGenerator implements Generator { + private Generator spanId; + private Generator parentSpanId; + private Generator startTime; + private Generator endTime; + private Generator operationName; + private Generator peer; + private Generator spanLayer; + private Generator componentId; + private Generator error; + private Generator> tags; + + @Override + public SpanObject next() { + return SpanObject + .newBuilder() + .setSpanId(getSpanId().next().intValue()) + .setParentSpanId(getParentSpanId().next().intValue()) + .setStartTime(getStartTime().next()) + .setEndTime(getEndTime().next()) + .setOperationName(getOperationName().next()) + .setPeer(getPeer().next()) + .setSpanLayer(SpanLayer.forNumber(getSpanLayer().next().intValue())) + .setComponentId(getComponentId().next().intValue()) + .setIsError(getError().next()) + .addAllTags( + getTags() + .next() + .stream() + .map(TagGenerator::next) + .map(it -> KeyStringValuePair + .newBuilder().setKey(it.getKey()) + .setValue(it.getValue()).build()) + .collect(Collectors.toList())) + .build(); + } + + @Override + public void reset() { + spanId.reset(); + parentSpanId.reset(); + startTime.reset(); + endTime.reset(); + operationName.reset(); + peer.reset(); + spanLayer.reset(); + componentId.reset(); + error.reset(); + tags.reset(); + } +} diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/TagGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/TagGenerator.java new file mode 100644 index 0000000000..e55d79ba2d --- /dev/null +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/TagGenerator.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.restapi; + +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.apache.skywalking.generator.Generator; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +@JsonTypeInfo(use = JsonTypeInfo.Id.NONE) +public final class TagGenerator implements Generator { + private Generator key; + private Generator value; + + @Override + public Tag next() { + return new Tag(key.next(), value.next()); + } + + @Override + public void reset() { + key.reset(); + value.reset(); + } +} diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/starter/DataGeneratorStartUp.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/starter/DataGeneratorStartUp.java new file mode 100644 index 0000000000..4f3581137e --- /dev/null +++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/starter/DataGeneratorStartUp.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.starter; + +import org.apache.skywalking.oap.server.starter.OAPServerBootstrap; + +public class DataGeneratorStartUp { + public static void main(String[] args) { + OAPServerBootstrap.start(); + } +} diff --git a/oap-server/server-tools/data-generator/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine b/oap-server/server-tools/data-generator/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine new file mode 100644 index 0000000000..02226bb7fb --- /dev/null +++ b/oap-server/server-tools/data-generator/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.module.DataGeneratorModule diff --git a/oap-server/server-tools/data-generator/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider b/oap-server/server-tools/data-generator/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider new file mode 100644 index 0000000000..01a75a2a5a --- /dev/null +++ b/oap-server/server-tools/data-generator/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.module.DataGeneratorModuleProvider diff --git a/oap-server/server-tools/data-generator/src/main/resources/application.yml b/oap-server/server-tools/data-generator/src/main/resources/application.yml new file mode 100755 index 0000000000..60057f1e17 --- /dev/null +++ b/oap-server/server-tools/data-generator/src/main/resources/application.yml @@ -0,0 +1,179 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cluster: + selector: ${SW_CLUSTER:standalone} + standalone: +core: + selector: ${SW_CORE:default} + default: + # Mixed: Receive agent data, Level 1 aggregate, Level 2 aggregate + # Receiver: Receive agent data, Level 1 aggregate + # Aggregator: Level 2 aggregate + role: ${SW_CORE_ROLE:Mixed} # Mixed/Receiver/Aggregator + restHost: ${SW_CORE_REST_HOST:0.0.0.0} + restPort: ${SW_CORE_REST_PORT:12800} + restContextPath: ${SW_CORE_REST_CONTEXT_PATH:/} + restMinThreads: ${SW_CORE_REST_JETTY_MIN_THREADS:1} + restMaxThreads: ${SW_CORE_REST_JETTY_MAX_THREADS:200} + restIdleTimeOut: ${SW_CORE_REST_JETTY_IDLE_TIMEOUT:30000} + restAcceptorPriorityDelta: ${SW_CORE_REST_JETTY_DELTA:0} + restAcceptQueueSize: ${SW_CORE_REST_JETTY_QUEUE_SIZE:0} + httpMaxRequestHeaderSize: ${SW_CORE_HTTP_MAX_REQUEST_HEADER_SIZE:8192} + gRPCHost: ${SW_CORE_GRPC_HOST:0.0.0.0} + gRPCPort: ${SW_CORE_GRPC_PORT:11800} + maxConcurrentCallsPerConnection: ${SW_CORE_GRPC_MAX_CONCURRENT_CALL:0} + maxMessageSize: ${SW_CORE_GRPC_MAX_MESSAGE_SIZE:0} + gRPCThreadPoolQueueSize: ${SW_CORE_GRPC_POOL_QUEUE_SIZE:-1} + gRPCThreadPoolSize: ${SW_CORE_GRPC_THREAD_POOL_SIZE:-1} + gRPCSslEnabled: ${SW_CORE_GRPC_SSL_ENABLED:false} + gRPCSslKeyPath: ${SW_CORE_GRPC_SSL_KEY_PATH:""} + gRPCSslCertChainPath: ${SW_CORE_GRPC_SSL_CERT_CHAIN_PATH:""} + gRPCSslTrustedCAPath: ${SW_CORE_GRPC_SSL_TRUSTED_CA_PATH:""} + downsampling: + - Hour + - Day + # Set a timeout on metrics data. After the timeout has expired, the metrics data will automatically be deleted. + enableDataKeeperExecutor: ${SW_CORE_ENABLE_DATA_KEEPER_EXECUTOR:true} # Turn it off then automatically metrics data delete will be close. + dataKeeperExecutePeriod: ${SW_CORE_DATA_KEEPER_EXECUTE_PERIOD:5} # How often the data keeper executor runs periodically, unit is minute + recordDataTTL: ${SW_CORE_RECORD_DATA_TTL:3} # Unit is day + metricsDataTTL: ${SW_CORE_METRICS_DATA_TTL:7} # Unit is day + # The period of L1 aggregation flush to L2 aggregation. Unit is ms. + l1FlushPeriod: ${SW_CORE_L1_AGGREGATION_FLUSH_PERIOD:500} + # The threshold of session time. Unit is ms. Default value is 70s. + storageSessionTimeout: ${SW_CORE_STORAGE_SESSION_TIMEOUT:70000} + # The period of doing data persistence. Unit is second.Default value is 25s + persistentPeriod: ${SW_CORE_PERSISTENT_PERIOD:25} + # Cache metrics data for 1 minute to reduce database queries, and if the OAP cluster changes within that minute, + # the metrics may not be accurate within that minute. + enableDatabaseSession: ${SW_CORE_ENABLE_DATABASE_SESSION:true} + topNReportPeriod: ${SW_CORE_TOPN_REPORT_PERIOD:10} # top_n record worker report cycle, unit is minute + # Extra model column are the column defined by in the codes, These columns of model are not required logically in aggregation or further query, + # and it will cause more load for memory, network of OAP and storage. + # But, being activated, user could see the name in the storage entities, which make users easier to use 3rd party tool, such as Kibana->ES, to query the data by themselves. + activeExtraModelColumns: ${SW_CORE_ACTIVE_EXTRA_MODEL_COLUMNS:false} + # The max length of service + instance names should be less than 200 + serviceNameMaxLength: ${SW_SERVICE_NAME_MAX_LENGTH:70} + instanceNameMaxLength: ${SW_INSTANCE_NAME_MAX_LENGTH:70} + # The max length of service + endpoint names should be less than 240 + endpointNameMaxLength: ${SW_ENDPOINT_NAME_MAX_LENGTH:150} + # Define the set of span tag keys, which should be searchable through the GraphQL. + searchableTracesTags: ${SW_SEARCHABLE_TAG_KEYS:http.method,status_code,db.type,db.instance,mq.queue,mq.topic,mq.broker} + # Define the set of log tag keys, which should be searchable through the GraphQL. + searchableLogsTags: ${SW_SEARCHABLE_LOGS_TAG_KEYS:level} + # Define the set of alarm tag keys, which should be searchable through the GraphQL. + searchableAlarmTags: ${SW_SEARCHABLE_ALARM_TAG_KEYS:level} + # The number of threads used to prepare metrics data to the storage. + prepareThreads: ${SW_CORE_PREPARE_THREADS:2} + # Turn it on then automatically grouping endpoint by the given OpenAPI definitions. + enableEndpointNameGroupingByOpenapi: ${SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPAENAPI:true} +storage: + selector: ${SW_STORAGE:h2} + elasticsearch: + namespace: ${SW_NAMESPACE:""} + clusterNodes: ${SW_STORAGE_ES_CLUSTER_NODES:localhost:9200} + protocol: ${SW_STORAGE_ES_HTTP_PROTOCOL:"http"} + connectTimeout: ${SW_STORAGE_ES_CONNECT_TIMEOUT:3000} + socketTimeout: ${SW_STORAGE_ES_SOCKET_TIMEOUT:30000} + responseTimeout: ${SW_STORAGE_ES_RESPONSE_TIMEOUT:15000} + numHttpClientThread: ${SW_STORAGE_ES_NUM_HTTP_CLIENT_THREAD:0} + user: ${SW_ES_USER:""} + password: ${SW_ES_PASSWORD:""} + trustStorePath: ${SW_STORAGE_ES_SSL_JKS_PATH:""} + trustStorePass: ${SW_STORAGE_ES_SSL_JKS_PASS:""} + secretsManagementFile: ${SW_ES_SECRETS_MANAGEMENT_FILE:""} # Secrets management file in the properties format includes the username, password, which are managed by 3rd party tool. + dayStep: ${SW_STORAGE_DAY_STEP:1} # Represent the number of days in the one minute/hour/day index. + indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:1} # Shard number of new indexes + indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:1} # Replicas number of new indexes + # Super data set has been defined in the codes, such as trace segments.The following 3 config would be improve es performance when storage super size data in es. + superDatasetDayStep: ${SW_SUPERDATASET_STORAGE_DAY_STEP:-1} # Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value is less than 0 + superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5} # This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces. + superDatasetIndexReplicasNumber: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_REPLICAS_NUMBER:0} # Represent the replicas number in the super size dataset record index, the default value is 0. + indexTemplateOrder: ${SW_STORAGE_ES_INDEX_TEMPLATE_ORDER:0} # the order of index template + bulkActions: ${SW_STORAGE_ES_BULK_ACTIONS:5000} # Execute the async bulk record data every ${SW_STORAGE_ES_BULK_ACTIONS} requests + # flush the bulk every 10 seconds whatever the number of requests + # INT(flushInterval * 2/3) would be used for index refresh period. + flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:15} + concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests + resultWindowMaxSize: ${SW_STORAGE_ES_QUERY_MAX_WINDOW_SIZE:10000} + metadataQueryMaxSize: ${SW_STORAGE_ES_QUERY_MAX_SIZE:5000} + segmentQueryMaxSize: ${SW_STORAGE_ES_QUERY_SEGMENT_SIZE:200} + profileTaskQueryMaxSize: ${SW_STORAGE_ES_QUERY_PROFILE_TASK_SIZE:200} + oapAnalyzer: ${SW_STORAGE_ES_OAP_ANALYZER:"{\"analyzer\":{\"oap_analyzer\":{\"type\":\"stop\"}}}"} # the oap analyzer. + oapLogAnalyzer: ${SW_STORAGE_ES_OAP_LOG_ANALYZER:"{\"analyzer\":{\"oap_log_analyzer\":{\"type\":\"standard\"}}}"} # the oap log analyzer. It could be customized by the ES analyzer configuration to support more language log formats, such as Chinese log, Japanese log and etc. + advanced: ${SW_STORAGE_ES_ADVANCED:""} + h2: + driver: ${SW_STORAGE_H2_DRIVER:org.h2.jdbcx.JdbcDataSource} + url: ${SW_STORAGE_H2_URL:jdbc:h2:mem:skywalking-oap-db;DB_CLOSE_DELAY=-1} + user: ${SW_STORAGE_H2_USER:sa} + metadataQueryMaxSize: ${SW_STORAGE_H2_QUERY_MAX_SIZE:5000} + maxSizeOfArrayColumn: ${SW_STORAGE_MAX_SIZE_OF_ARRAY_COLUMN:20} + numOfSearchableValuesPerTag: ${SW_STORAGE_NUM_OF_SEARCHABLE_VALUES_PER_TAG:2} + maxSizeOfBatchSql: ${SW_STORAGE_MAX_SIZE_OF_BATCH_SQL:100} + asyncBatchPersistentPoolSize: ${SW_STORAGE_ASYNC_BATCH_PERSISTENT_POOL_SIZE:1} + +agent-analyzer: + selector: ${SW_AGENT_ANALYZER:default} + default: + # The default sampling rate and the default trace latency time configured by the 'traceSamplingPolicySettingsFile' file. + traceSamplingPolicySettingsFile: ${SW_TRACE_SAMPLING_POLICY_SETTINGS_FILE:trace-sampling-policy-settings.yml} + slowDBAccessThreshold: ${SW_SLOW_DB_THRESHOLD:default:200,mongodb:100} # The slow database access thresholds. Unit ms. + forceSampleErrorSegment: ${SW_FORCE_SAMPLE_ERROR_SEGMENT:true} # When sampling mechanism active, this config can open(true) force save some error segment. true is default. + segmentStatusAnalysisStrategy: ${SW_SEGMENT_STATUS_ANALYSIS_STRATEGY:FROM_SPAN_STATUS} # Determine the final segment status from the status of spans. Available values are `FROM_SPAN_STATUS` , `FROM_ENTRY_SPAN` and `FROM_FIRST_SPAN`. `FROM_SPAN_STATUS` represents the segment status would be error if any span is in error status. `FROM_ENTRY_SPAN` means the segment status would be determined by the status of entry spans only. `FROM_FIRST_SPAN` means the segment status would be determined by the status of the first span only. + # Nginx and Envoy agents can't get the real remote address. + # Exit spans with the component in the list would not generate the client-side instance relation metrics. + noUpstreamRealAddressAgents: ${SW_NO_UPSTREAM_REAL_ADDRESS:6000,9000} + meterAnalyzerActiveFiles: ${SW_METER_ANALYZER_ACTIVE_FILES:datasource,threadpool} # Which files could be meter analyzed, files split by "," + +log-analyzer: + selector: ${SW_LOG_ANALYZER:default} + default: + lalFiles: ${SW_LOG_LAL_FILES:default} + malFiles: ${SW_LOG_MAL_FILES:""} + +event-analyzer: + selector: ${SW_EVENT_ANALYZER:default} + default: + +query: + selector: ${SW_QUERY:graphql} + graphql: + path: ${SW_QUERY_GRAPHQL_PATH:/graphql} + # Enable the log testing API to test the LAL. + # NOTE: This API evaluates untrusted code on the OAP server. + # A malicious script can do significant damage (steal keys and secrets, remove files and directories, install malware, etc). + # As such, please enable this API only when you completely trust your users. + enableLogTestTool: ${SW_QUERY_GRAPHQL_ENABLE_LOG_TEST_TOOL:false} + +alarm: + selector: ${SW_ALARM:default} + default: + +telemetry: + selector: ${SW_TELEMETRY:none} + none: + +configuration: + selector: ${SW_CONFIGURATION:none} + none: + +health-checker: + selector: ${SW_HEALTH_CHECKER:-} + default: + checkIntervalSeconds: ${SW_HEALTH_CHECKER_INTERVAL_SECONDS:5} + +data-generator: + selector: ${SW_DATA_GENERATOR:default} + default: diff --git a/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/IntGeneratorTest.java b/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/IntGeneratorTest.java new file mode 100644 index 0000000000..9799f16496 --- /dev/null +++ b/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/IntGeneratorTest.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.skywalking.generator; + +import static org.junit.Assert.assertEquals; +import org.junit.Test; + +public class IntGeneratorTest { + @Test + public void testFixedInt() { + final IntGenerator.Builder builder = new IntGenerator.Builder(); + builder.setMin(1L); + builder.setMax(1L); + final IntGenerator generator = builder.build(); + for (int i = 0; i < 100; i++) { + assertEquals(1, generator.next().intValue()); + } + } +} diff --git a/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/SequenceGeneratorTest.java b/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/SequenceGeneratorTest.java new file mode 100644 index 0000000000..af624cbe14 --- /dev/null +++ b/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/SequenceGeneratorTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.generator; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import org.junit.Test; + +public final class SequenceGeneratorTest { + @Test + public void testSequence() { + SequenceGenerator generator = + new SequenceGenerator.Builder() + .setMin(1L) + .setMax(100L) + .build(); + + for (int i = 0; i < 10; i++) { + assertEquals(i + 1, generator.next().intValue()); + } + } + + @Test + public void testFluctuation() { + SequenceGenerator generator = + new SequenceGenerator.Builder() + .setMin(1L) + .setMax(100L) + .setFluctuation(1) + .build(); + + for (int i = 1; i < 10; i++) { + Long next = generator.next(); + assertTrue(i <= next.intValue()); + assertTrue(i * 2 >= next.intValue()); + } + } +} diff --git a/oap-server/server-tools/pom.xml b/oap-server/server-tools/pom.xml index 4371350c79..0ed3da6d8b 100644 --- a/oap-server/server-tools/pom.xml +++ b/oap-server/server-tools/pom.xml @@ -30,5 +30,6 @@ profile-exporter + data-generator