From 1af2e46e3328a1d6d2b79d34372b42a9b9a15205 Mon Sep 17 00:00:00 2001 From: kezhenxu94 Date: Wed, 18 Aug 2021 16:40:25 +0800 Subject: [PATCH] Add a new API to test log analysis language (#7475) --- CHANGES.md | 1 + .../setup/backend/configuration-vocabulary.md | 1 + .../oap/log/analyzer/dsl/Binding.java | 45 ++++++ .../dsl/spec/extractor/ExtractorSpec.java | 18 ++- .../analyzer/dsl/spec/filter/FilterSpec.java | 21 ++- .../log/listener/RecordAnalysisListener.java | 2 + .../src/main/resources/application.yml | 5 + .../query-graphql-plugin/pom.xml | 7 +- .../oap/query/graphql/GraphQLQueryConfig.java | 4 +- .../query/graphql/GraphQLQueryProvider.java | 6 +- .../query/graphql/resolver/LogTestQuery.java | 128 +++++++++++++++++ .../query/graphql/type/LogTestRequest.java | 30 ++++ .../query/graphql/type/LogTestResponse.java | 33 +++++ .../oap/query/graphql/type/Metrics.java | 36 +++++ .../src/main/resources/query-protocol | 2 +- .../graphql/resolver/LogTestQueryTest.java | 129 ++++++++++++++++++ 16 files changed, 454 insertions(+), 14 deletions(-) create mode 100644 oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/LogTestQuery.java create mode 100644 oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/LogTestRequest.java create mode 100644 oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/LogTestResponse.java create mode 100644 oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/Metrics.java create mode 100644 oap-server/server-query-plugin/query-graphql-plugin/src/test/java/org/apache/skywalking/oap/query/graphql/resolver/LogTestQueryTest.java diff --git a/CHANGES.md b/CHANGES.md index cd56ec7ec..8da38e05e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -41,6 +41,7 @@ Release Notes. * Add component id for Python falcon plugin. * Add `rpcStatusCode` for `rpc.status_code` tag. The `responseCode` field is marked as deprecated and replaced by `httpResponseStatusCode` field. * Remove the duplicated tags to reduce the storage payload. +* Add a new API to test log analysis language. #### UI diff --git a/docs/en/setup/backend/configuration-vocabulary.md b/docs/en/setup/backend/configuration-vocabulary.md index 45922501e..f6db25e75 100644 --- a/docs/en/setup/backend/configuration-vocabulary.md +++ b/docs/en/setup/backend/configuration-vocabulary.md @@ -246,6 +246,7 @@ core|default|role|Option values: `Mixed/Receiver/Aggregator`. **Receiver** mode | - | - | sampleRate | Sampling rate for receiving trace. Precise to 1/10000. 10000 means sampling rate of 100% by default. | SW_RECEIVER_BROWSER_SAMPLE_RATE | 10000 | | query | graphql | - | GraphQL query implementation. | - | | - | - | path | Root path of GraphQL query and mutation. | SW_QUERY_GRAPHQL_PATH | /graphql| +| - | - | enableLogTestTool | Enable the log testing API to test the LAL. **NOTE**: This API evaluates untrusted code on the OAP server. A malicious script can do significant damage (steal keys and secrets, remove files and directories, install malware, etc). As such, please enable this API only when you completely trust your users. | SW_QUERY_GRAPHQL_ENABLE_LOG_TEST_TOOL | false | | alarm | default | - | Read [alarm doc](backend-alarm.md) for more details. | - | | telemetry | - | - | Read [telemetry doc](backend-telemetry.md) for more details. | - | | - | none| - | No op implementation. | - | diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/Binding.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/Binding.java index 55582acde..00b67c538 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/Binding.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/Binding.java @@ -22,10 +22,15 @@ import com.google.protobuf.Message; import groovy.lang.Closure; import groovy.lang.GroovyObjectSupport; import groovy.lang.MissingPropertyException; +import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Matcher; import lombok.Getter; import org.apache.skywalking.apm.network.logging.v3.LogData; +import org.apache.skywalking.oap.meter.analyzer.dsl.SampleFamily; +import org.apache.skywalking.oap.server.core.source.Log; /** * The binding bridge between OAP and the DSL, which provides some convenient methods to ease the use of the raw {@link groovy.lang.Binding#setProperty(java.lang.String, java.lang.Object)} and {@link @@ -40,6 +45,10 @@ public class Binding extends groovy.lang.Binding { public static final String KEY_ABORT = "abort"; + public static final String KEY_METRICS_CONTAINER = "metrics_container"; + + public static final String KEY_LOG_CONTAINER = "log_container"; + public Binding() { setProperty(KEY_PARSED, new Parsed()); } @@ -48,6 +57,8 @@ public class Binding extends groovy.lang.Binding { setProperty(KEY_LOG, log); setProperty(KEY_SAVE, true); setProperty(KEY_ABORT, false); + setProperty(KEY_METRICS_CONTAINER, null); + setProperty(KEY_LOG_CONTAINER, null); parsed().log = log; return this; } @@ -106,6 +117,40 @@ public class Binding extends groovy.lang.Binding { return (boolean) getProperty(KEY_ABORT); } + /** + * Set the metrics container to store all metrics generated from the pipeline, + * if no container is set, all generated metrics will be sent to MAL engine for further processing, + * if metrics container is set, all metrics are only stored in the container, and won't be sent to MAL. + * + * @param container the metrics container + */ + public Binding metricsContainer(List container) { + setProperty(KEY_METRICS_CONTAINER, container); + return this; + } + + public Optional> metricsContainer() { + // noinspection unchecked + return Optional.ofNullable((List) getProperty(KEY_METRICS_CONTAINER)); + } + + /** + * Set the log container to store the final log if it should be persisted in storage, + * if no container is set, the final log will be sent to source receiver, + * if log container is set, the log is only stored in the container, and won't be sent to source receiver. + * + * @param container the log container + */ + public Binding logContainer(AtomicReference container) { + setProperty(KEY_LOG_CONTAINER, container); + return this; + } + + public Optional> logContainer() { + // noinspection unchecked + return Optional.ofNullable((AtomicReference) getProperty(KEY_LOG_CONTAINER)); + } + public static class Parsed extends GroovyObjectSupport { @Getter private Matcher matcher; diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java index c51f445ff..86dc4c83e 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java @@ -26,6 +26,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.stream.Collectors; import lombok.experimental.Delegate; import org.apache.commons.lang3.StringUtils; @@ -188,12 +189,19 @@ public class ExtractorSpec extends AbstractSpec { cl.call(); final Sample sample = builder.build(); + final SampleFamily sampleFamily = SampleFamilyBuilder.newBuilder(sample).build(); - metricConverts.forEach(it -> it.toMeter( - ImmutableMap.builder() - .put(sample.getName(), SampleFamilyBuilder.newBuilder(sample).build()) - .build() - )); + final Optional> possibleMetricsContainer = BINDING.get().metricsContainer(); + + if (possibleMetricsContainer.isPresent()) { + possibleMetricsContainer.get().add(sampleFamily); + } else { + metricConverts.forEach(it -> it.toMeter( + ImmutableMap.builder() + .put(sample.getName(), sampleFamily) + .build() + )); + } } public static class SampleBuilder { diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java index 1f448e040..4bb11265f 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java @@ -26,6 +26,8 @@ import groovy.lang.DelegatesTo; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; import org.apache.skywalking.apm.network.logging.v3.LogData; import org.apache.skywalking.oap.log.analyzer.dsl.Binding; import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec; @@ -38,6 +40,7 @@ import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig; import org.apache.skywalking.oap.log.analyzer.provider.log.listener.LogAnalysisListenerFactory; import org.apache.skywalking.oap.log.analyzer.provider.log.listener.RecordAnalysisListener; import org.apache.skywalking.oap.log.analyzer.provider.log.listener.TrafficAnalysisListener; +import org.apache.skywalking.oap.server.core.source.Log; import org.apache.skywalking.oap.server.library.module.ModuleManager; import org.apache.skywalking.oap.server.library.module.ModuleStartException; import org.slf4j.Logger; @@ -163,9 +166,21 @@ public class FilterSpec extends AbstractSpec { return; } - factories.stream() - .map(LogAnalysisListenerFactory::create) - .forEach(it -> it.parse(logData, extraLog).build()); + final Optional> container = BINDING.get().logContainer(); + if (container.isPresent()) { + factories.stream() + .map(LogAnalysisListenerFactory::create) + .filter(it -> it instanceof RecordAnalysisListener) + .map(it -> it.parse(logData, extraLog)) + .map(it -> (RecordAnalysisListener) it) + .map(RecordAnalysisListener::getLog) + .findFirst() + .ifPresent(log -> container.get().set(log)); + } else { + factories.stream() + .map(LogAnalysisListenerFactory::create) + .forEach(it -> it.parse(logData, extraLog).build()); + } } @SuppressWarnings("unused") diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/provider/log/listener/RecordAnalysisListener.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/provider/log/listener/RecordAnalysisListener.java index 21d2d563a..0aed40c77 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/provider/log/listener/RecordAnalysisListener.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/provider/log/listener/RecordAnalysisListener.java @@ -23,6 +23,7 @@ import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.UUID; +import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import org.apache.skywalking.apm.network.logging.v3.LogData; @@ -53,6 +54,7 @@ public class RecordAnalysisListener implements LogAnalysisListener { private final SourceReceiver sourceReceiver; private final NamingControl namingControl; private final List searchableTagKeys; + @Getter private final Log log = new Log(); @Override diff --git a/oap-server/server-bootstrap/src/main/resources/application.yml b/oap-server/server-bootstrap/src/main/resources/application.yml index 3cb87b326..28af6ef57 100755 --- a/oap-server/server-bootstrap/src/main/resources/application.yml +++ b/oap-server/server-bootstrap/src/main/resources/application.yml @@ -419,6 +419,11 @@ query: selector: ${SW_QUERY:graphql} graphql: path: ${SW_QUERY_GRAPHQL_PATH:/graphql} + # Enable the log testing API to test the LAL. + # NOTE: This API evaluates untrusted code on the OAP server. + # A malicious script can do significant damage (steal keys and secrets, remove files and directories, install malware, etc). + # As such, please enable this API only when you completely trust your users. + enableLogTestTool: ${SW_QUERY_GRAPHQL_ENABLE_LOG_TEST_TOOL:false} alarm: selector: ${SW_ALARM:default} diff --git a/oap-server/server-query-plugin/query-graphql-plugin/pom.xml b/oap-server/server-query-plugin/query-graphql-plugin/pom.xml index 53121016f..f79846f02 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/pom.xml +++ b/oap-server/server-query-plugin/query-graphql-plugin/pom.xml @@ -38,6 +38,11 @@ server-health-checker ${project.version} + + org.apache.skywalking + log-analyzer + ${project.version} + com.graphql-java graphql-java @@ -55,4 +60,4 @@ jackson-module-afterburner - \ No newline at end of file + diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryConfig.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryConfig.java index 8f89e6b0e..0882017ea 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryConfig.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryConfig.java @@ -18,7 +18,6 @@ package org.apache.skywalking.oap.query.graphql; -import lombok.AccessLevel; import lombok.Getter; import lombok.Setter; import org.apache.skywalking.oap.server.library.module.ModuleConfig; @@ -26,8 +25,9 @@ import org.apache.skywalking.oap.server.library.module.ModuleConfig; /** * The config of {@code query.graphql}. */ -@Getter(AccessLevel.PACKAGE) +@Getter @Setter public class GraphQLQueryConfig extends ModuleConfig { private String path; + private boolean enableLogTestTool; } diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java index f912198e6..241448a5d 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java @@ -27,6 +27,7 @@ import org.apache.skywalking.oap.query.graphql.resolver.BrowserLogQuery; import org.apache.skywalking.oap.query.graphql.resolver.EventQuery; import org.apache.skywalking.oap.query.graphql.resolver.HealthQuery; import org.apache.skywalking.oap.query.graphql.resolver.LogQuery; +import org.apache.skywalking.oap.query.graphql.resolver.LogTestQuery; import org.apache.skywalking.oap.query.graphql.resolver.MetadataQuery; import org.apache.skywalking.oap.query.graphql.resolver.MetricQuery; import org.apache.skywalking.oap.query.graphql.resolver.MetricsQuery; @@ -80,7 +81,7 @@ public class GraphQLQueryProvider extends ModuleProvider { .resolvers(new MetadataQuery(getManager())) .file("query-protocol/topology.graphqls") .resolvers(new TopologyQuery(getManager())) - /** + /* * Metrics v2 query protocol is an alternative metrics query(s) of original v1, * defined in the metric.graphql, top-n-records.graphqls, and aggregation.graphqls. */ @@ -101,7 +102,8 @@ public class GraphQLQueryProvider extends ModuleProvider { .file("query-protocol/alarm.graphqls") .resolvers(new AlarmQuery(getManager())) .file("query-protocol/log.graphqls") - .resolvers(new LogQuery(getManager())) + .resolvers(new LogQuery(getManager()), + new LogTestQuery(getManager(), config)) .file("query-protocol/profile.graphqls") .resolvers(new ProfileQuery(getManager()), new ProfileMutation(getManager())) .file("query-protocol/ui-configuration.graphqls") diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/LogTestQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/LogTestQuery.java new file mode 100644 index 000000000..e71916e81 --- /dev/null +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/LogTestQuery.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.query.graphql.resolver; + +import com.coxautodev.graphql.tools.GraphQLQueryResolver; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import org.apache.skywalking.apm.network.logging.v3.LogData; +import org.apache.skywalking.oap.log.analyzer.dsl.Binding; +import org.apache.skywalking.oap.log.analyzer.dsl.DSL; +import org.apache.skywalking.oap.log.analyzer.module.LogAnalyzerModule; +import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig; +import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleProvider; +import org.apache.skywalking.oap.query.graphql.GraphQLQueryConfig; +import org.apache.skywalking.oap.query.graphql.type.LogTestRequest; +import org.apache.skywalking.oap.query.graphql.type.LogTestResponse; +import org.apache.skywalking.oap.query.graphql.type.Metrics; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.query.type.KeyValue; +import org.apache.skywalking.oap.server.core.query.type.Log; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.util.ProtoBufJsonUtils; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; +import static org.apache.skywalking.apm.util.StringUtil.isNotBlank; + +@RequiredArgsConstructor +public class LogTestQuery implements GraphQLQueryResolver { + private final ModuleManager moduleManager; + + private final GraphQLQueryConfig config; + + public LogTestResponse test(LogTestRequest request) throws Exception { + if (!config.isEnableLogTestTool()) { + throw new IllegalAccessException("LAL debug tool is not enabled"); + } + + requireNonNull(request, "request"); + checkArgument(isNotBlank(request.getLog()), "request.log cannot be blank"); + checkArgument(isNotBlank(request.getDsl()), "request.dsl cannot be blank"); + + final LogAnalyzerModuleProvider provider = + (LogAnalyzerModuleProvider) moduleManager.find(LogAnalyzerModule.NAME) + .provider(); + final LogAnalyzerModuleConfig config = + (LogAnalyzerModuleConfig) provider.createConfigBeanIfAbsent(); + final DSL dsl = DSL.of(moduleManager, config, request.getDsl()); + final Binding binding = new Binding(); + + final LogData.Builder log = LogData.newBuilder(); + ProtoBufJsonUtils.fromJSON(request.getLog(), log); + binding.log(log); + + binding.logContainer(new AtomicReference<>()); + binding.metricsContainer(new ArrayList<>()); + + dsl.bind(binding); + dsl.evaluate(); + + final LogTestResponse.LogTestResponseBuilder builder = LogTestResponse.builder(); + binding.logContainer().map(AtomicReference::get).ifPresent(it -> { + final Log l = new Log(); + + if (isNotBlank(it.getServiceId())) { + l.setServiceName(IDManager.ServiceID.analysisId(it.getServiceId()).getName()); + } + l.setServiceId(it.getServiceId()); + if (isNotBlank(it.getServiceInstanceId())) { + String name = IDManager.ServiceInstanceID.analysisId(it.getServiceId()).getName(); + l.setServiceInstanceName(name); + } + l.setServiceInstanceId(it.getServiceInstanceId()); + l.setEndpointId(it.getEndpointId()); + if (isNotBlank(it.getEndpointId())) { + String name = IDManager.EndpointID.analysisId(it.getEndpointId()).getEndpointName(); + l.setEndpointName(name); + } + l.setTraceId(it.getTraceId()); + l.setTimestamp(it.getTimestamp()); + l.setContentType(it.getContentType()); + l.setContent(it.getContent()); + final List tags = it.getTags() + .stream() + .map(tag -> new KeyValue(tag.getKey(), tag.getValue())) + .collect(Collectors.toList()); + l.getTags().addAll(tags); + + builder.log(l); + }); + binding.metricsContainer().ifPresent(it -> { + final List samples = + it.stream() + .flatMap(s -> Arrays.stream(s.samples)) + .map(s -> new Metrics( + s.getName(), + s.getLabels().entrySet() + .stream().map(kv -> new KeyValue(kv.getKey(), kv.getValue())) + .collect(Collectors.toList()), + (long) s.getValue(), + s.getTimestamp() + )) + .collect(Collectors.toList()); + builder.metrics(samples); + }); + return builder.build(); + } +} diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/LogTestRequest.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/LogTestRequest.java new file mode 100644 index 000000000..5a2070143 --- /dev/null +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/LogTestRequest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.query.graphql.type; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class LogTestRequest { + private String log; + + private String dsl; +} diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/LogTestResponse.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/LogTestResponse.java new file mode 100644 index 000000000..0354dfabc --- /dev/null +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/LogTestResponse.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.query.graphql.type; + +import java.util.List; +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.query.type.Log; + +@Setter +@Getter +@Builder +public class LogTestResponse { + private final Log log; + private final List metrics; +} diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/Metrics.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/Metrics.java new file mode 100644 index 000000000..9a8271fdb --- /dev/null +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/Metrics.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.query.graphql.type; + +import java.util.List; +import lombok.Data; +import lombok.RequiredArgsConstructor; +import org.apache.skywalking.oap.server.core.query.type.KeyValue; + +@Data +@RequiredArgsConstructor +public class Metrics { + private final String name; + + private final List tags; + + private final long value; + + private final long timestamp; +} diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol index e9ecb5153..47202fc1e 160000 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol @@ -1 +1 @@ -Subproject commit e9ecb5153fbab9ab21d805c898d05fbe45e7c4d6 +Subproject commit 47202fc1eaa1864c587a78f423a0685ffbe294ad diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/test/java/org/apache/skywalking/oap/query/graphql/resolver/LogTestQueryTest.java b/oap-server/server-query-plugin/query-graphql-plugin/src/test/java/org/apache/skywalking/oap/query/graphql/resolver/LogTestQueryTest.java new file mode 100644 index 000000000..38f87867b --- /dev/null +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/test/java/org/apache/skywalking/oap/query/graphql/resolver/LogTestQueryTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.query.graphql.resolver; + +import junit.framework.TestCase; +import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig; +import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleProvider; +import org.apache.skywalking.oap.query.graphql.GraphQLQueryConfig; +import org.apache.skywalking.oap.query.graphql.type.LogTestRequest; +import org.apache.skywalking.oap.query.graphql.type.LogTestResponse; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.config.ConfigService; +import org.apache.skywalking.oap.server.core.config.NamingControl; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder; +import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class LogTestQueryTest extends TestCase { + @Mock + private ModuleManager moduleManager; + + @Mock + private GraphQLQueryConfig config; + + @Mock + private ModuleProviderHolder providerHolder; + + @Mock + private LogAnalyzerModuleProvider serviceHolder; + + @Mock + private LogAnalyzerModuleConfig lalConfig; + + @Before + public void setup() { + when(moduleManager.find(anyString())) + .thenReturn(providerHolder); + when(providerHolder.provider()) + .thenReturn(serviceHolder); + when(serviceHolder.createConfigBeanIfAbsent()) + .thenReturn(lalConfig); + + final ModuleProviderHolder m = mock(ModuleProviderHolder.class); + when(moduleManager.find(CoreModule.NAME)).thenReturn(m); + final ModuleServiceHolder s = mock(ModuleServiceHolder.class); + when(m.provider()).thenReturn(s); + final ConfigService c = mock(ConfigService.class); + when(s.getService(ConfigService.class)).thenReturn(c); + final NamingControl namingControl = mock(NamingControl.class); + when(s.getService(NamingControl.class)).thenReturn(namingControl); + when(namingControl.formatServiceName(anyString())).thenCallRealMethod(); + when(c.getSearchableLogsTags()).thenReturn(""); + } + + @Test + public void shouldThrowWhenDisabled() { + final LogTestQuery query = new LogTestQuery(moduleManager, config); + try { + query.test(new LogTestRequest()); + fail(); + } catch (Exception e) { + assertTrue(e instanceof IllegalAccessException); + assertEquals("LAL debug tool is not enabled", e.getMessage()); + } + } + + @Test + public void test() throws Exception { + when(config.isEnableLogTestTool()).thenReturn(true); + final LogTestQuery query = new LogTestQuery(moduleManager, config); + final LogTestRequest request = new LogTestRequest(); + request.setLog("" + + "{" + + " body: {" + + " text: {" + + " text: 'Save user test'" + + " }" + + " }," + + " type: TEXT," + + " timestamp: 12312313," + + " service: 'test'" + + "}"); + request.setDsl("" + + "filter {\n" + + " extractor {\n" + + " metrics {\n" + + " timestamp log.timestamp as Long\n" + + " labels level: parsed.level, service: log.service, instance: log.serviceInstance\n" + + " name 'log_count'\n" + + " value 1\n" + + " }\n" + + " }\n" + + " sink {\n" + + " }\n" + + "}"); + final LogTestResponse response = query.test(request); + assertEquals("Save user test", response.getLog().getContent()); + assertFalse(response.getMetrics().isEmpty()); + assertEquals("log_count", response.getMetrics().iterator().next().getName()); + assertEquals(1, response.getMetrics().iterator().next().getValue()); + assertEquals(12312313, response.getMetrics().iterator().next().getTimestamp()); + } +}