diff --git a/CHANGES.md b/CHANGES.md index 368d722cb..fa4b5facf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -77,6 +77,7 @@ Release Notes. * Upgrade etcd cluster coordinator and dynamic configuration to v3.x. * Configuration: Allow to configure server maximum request header size. * Add thread state metric and class loaded info metric to JVMMetric. +* Performance: compile LAL DSL statically and run with type checked. #### UI diff --git a/Makefile b/Makefile index 4ec2a1e61..acbab49c0 100644 --- a/Makefile +++ b/Makefile @@ -96,7 +96,7 @@ docker.agent: $(SW_ROOT)/docker/agent/Dockerfile.agent # 4. This rule runs $(BUILD_PRE) prior to any docker build and only if specified as a dependency variable # 5. This rule finally runs docker build passing $(BUILD_ARGS) to docker if they are specified as a dependency variable -DOCKER_RULE=time (mkdir -p $(DOCKER_BUILD_TOP)/$@ && cp -r $^ $(DOCKER_BUILD_TOP)/$@ && cd $(DOCKER_BUILD_TOP)/$@ && $(BUILD_PRE) docker build $(BUILD_ARGS) -t $(HUB)/$(subst docker.,,$@):$(TAG) -f Dockerfile$(suffix $@) .) +DOCKER_RULE=time (mkdir -p $(DOCKER_BUILD_TOP)/$@ && cp -r $^ $(DOCKER_BUILD_TOP)/$@ && cd $(DOCKER_BUILD_TOP)/$@ && $(BUILD_PRE) docker build --no-cache $(BUILD_ARGS) -t $(HUB)/$(subst docker.,,$@):$(TAG) -f Dockerfile$(suffix $@) .) # for each docker.XXX target create a push.docker.XXX target that pushes # the local docker image to another hub diff --git a/dist-material/release-docs/LICENSE b/dist-material/release-docs/LICENSE index ecb5a6f46..e049a27f0 100755 --- a/dist-material/release-docs/LICENSE +++ b/dist-material/release-docs/LICENSE @@ -455,7 +455,7 @@ Apache 2.0 licenses ======================================== echarts 5.0.2: https://github.com/apache/echarts Apache-2.0 Material Icons 3.0.1 https://github.com/google/material-design-icons Apache-2.0 -groovy 3.0.3 https://github.com/apache/groovy Apache-2.0 +groovy 3.0.8 https://github.com/apache/groovy Apache-2.0 ======================================== BSD licenses diff --git a/oap-server-bom/pom.xml b/oap-server-bom/pom.xml index 6fad449b8..993d301b6 100644 --- a/oap-server-bom/pom.xml +++ b/oap-server-bom/pom.xml @@ -68,7 +68,7 @@ 2.3.28 3.25.0-GA 0.10.3 - 3.0.3 + 3.0.8 2.4.8.Final 1.9.4 1.12.0 diff --git a/oap-server/analyzer/log-analyzer/pom.xml b/oap-server/analyzer/log-analyzer/pom.xml index f1dd346da..fbe0e6ca9 100644 --- a/oap-server/analyzer/log-analyzer/pom.xml +++ b/oap-server/analyzer/log-analyzer/pom.xml @@ -42,6 +42,10 @@ org.codehaus.groovy groovy + + com.fasterxml.jackson.core + jackson-databind + diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/DSL.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/DSL.java index c5fad4e21..c7db4053b 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/DSL.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/DSL.java @@ -19,14 +19,20 @@ package org.apache.skywalking.oap.log.analyzer.dsl; import groovy.lang.GroovyShell; +import groovy.transform.CompileStatic; import groovy.util.DelegatingScript; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; +import org.apache.skywalking.oap.log.analyzer.dsl.spec.LALDelegatingScript; import org.apache.skywalking.oap.log.analyzer.dsl.spec.filter.FilterSpec; import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig; import org.apache.skywalking.oap.server.library.module.ModuleManager; import org.apache.skywalking.oap.server.library.module.ModuleStartException; import org.codehaus.groovy.control.CompilerConfiguration; +import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer; + +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; @RequiredArgsConstructor(access = AccessLevel.PRIVATE) public class DSL { @@ -38,7 +44,16 @@ public class DSL { final LogAnalyzerModuleConfig config, final String dsl) throws ModuleStartException { final CompilerConfiguration cc = new CompilerConfiguration(); - cc.setScriptBaseClass(DelegatingScript.class.getName()); + final ASTTransformationCustomizer customizer = + new ASTTransformationCustomizer( + singletonMap( + "extensions", + singletonList(LALPrecompiledExtension.class.getName()) + ), + CompileStatic.class + ); + cc.addCompilationCustomizers(customizer); + cc.setScriptBaseClass(LALDelegatingScript.class.getName()); final GroovyShell sh = new GroovyShell(cc); final DelegatingScript script = (DelegatingScript) sh.parse(dsl); diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/LALPrecompiledExtension.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/LALPrecompiledExtension.java new file mode 100644 index 000000000..56888a980 --- /dev/null +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/LALPrecompiledExtension.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.log.analyzer.dsl; + +import org.apache.skywalking.apm.network.logging.v3.LogData; +import org.apache.skywalking.apm.network.logging.v3.LogDataBody; +import org.apache.skywalking.apm.network.logging.v3.LogTags; +import org.apache.skywalking.apm.network.logging.v3.TraceContext; +import org.codehaus.groovy.ast.expr.ConstantExpression; +import org.codehaus.groovy.ast.expr.Expression; +import org.codehaus.groovy.ast.expr.PropertyExpression; +import org.codehaus.groovy.ast.expr.VariableExpression; +import org.codehaus.groovy.transform.stc.AbstractTypeCheckingExtension; +import org.codehaus.groovy.transform.stc.StaticTypeCheckingVisitor; + +import static org.codehaus.groovy.ast.ClassHelper.makeCached; + +public class LALPrecompiledExtension extends AbstractTypeCheckingExtension { + + public LALPrecompiledExtension(final StaticTypeCheckingVisitor typeCheckingVisitor) { + super(typeCheckingVisitor); + } + + @Override + public boolean handleUnresolvedProperty(final PropertyExpression pexp) { + final Expression exp = pexp.getObjectExpression(); + + if (exp.getText().startsWith("parsed")) { + makeDynamic(pexp); + setHandled(true); + return true; + } + + if (exp.getText().startsWith("log")) { + if (handleLogVariable(pexp)) { + return true; + } + } + + return super.handleUnresolvedProperty(pexp); + } + + private boolean handleLogVariable(final PropertyExpression pexp) { + final Expression exp = pexp.getObjectExpression(); + final Expression p = pexp.getProperty(); + + if (exp instanceof VariableExpression) { + final VariableExpression v = (VariableExpression) exp; + if (v.getName().equals("log")) { + storeType(v, makeCached(LogData.Builder.class)); + } + if (p instanceof ConstantExpression) { + final ConstantExpression c = (ConstantExpression) p; + switch (c.getText()) { + case "body": + storeType(pexp, makeCached(LogDataBody.class)); + break; + case "traceContext": + storeType(pexp, makeCached(TraceContext.class)); + break; + case "tags": + storeType(pexp, makeCached(LogTags.class)); + break; + } + } + setHandled(true); + return true; + } + + return false; + } +} + diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/LALDelegatingScript.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/LALDelegatingScript.java new file mode 100644 index 000000000..a057373ef --- /dev/null +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/LALDelegatingScript.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.log.analyzer.dsl.spec; + +import groovy.lang.Closure; +import groovy.lang.DelegatesTo; +import groovy.util.DelegatingScript; +import org.apache.skywalking.oap.log.analyzer.dsl.spec.filter.FilterSpec; + +public class LALDelegatingScript extends DelegatingScript { + @Override + public Object run() { + return null; + } + + public void filter(@DelegatesTo(value = FilterSpec.class, strategy = Closure.DELEGATE_ONLY) Closure closure) { + closure.setDelegate(getDelegate()); + closure.call(); + } + + public void json(@DelegatesTo(value = FilterSpec.class) Closure closure) { + closure.setDelegate(getDelegate()); + closure.call(); + } + + public void text(@DelegatesTo(value = FilterSpec.class) Closure closure) { + closure.setDelegate(getDelegate()); + closure.call(); + } + + public void extractor(@DelegatesTo(value = FilterSpec.class) Closure closure) { + closure.setDelegate(getDelegate()); + closure.call(); + } + + public void sink(@DelegatesTo(value = FilterSpec.class) Closure closure) { + closure.setDelegate(getDelegate()); + closure.call(); + } +} diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java index 49ea7d607..c51f445ff 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java @@ -21,6 +21,7 @@ package org.apache.skywalking.oap.log.analyzer.dsl.spec.extractor; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; import groovy.lang.Closure; +import groovy.lang.DelegatesTo; import java.util.Collection; import java.util.List; import java.util.Map; @@ -54,7 +55,8 @@ public class ExtractorSpec extends AbstractSpec { final LogAnalyzerModuleConfig moduleConfig) throws ModuleStartException { super(moduleManager, moduleConfig); - final MeterSystem meterSystem = moduleManager.find(CoreModule.NAME).provider().getService(MeterSystem.class); + final MeterSystem meterSystem = + moduleManager.find(CoreModule.NAME).provider().getService(MeterSystem.class); metricConverts = moduleConfig.malConfigs() .stream() @@ -93,7 +95,7 @@ public class ExtractorSpec extends AbstractSpec { } @SuppressWarnings("unused") - public void tag(final Map kv) { + public void tag(final Map kv) { if (BINDING.get().shouldAbort()) { return; } @@ -108,7 +110,8 @@ public class ExtractorSpec extends AbstractSpec { kv.entrySet() .stream() .filter(it -> isNotBlank(it.getKey())) - .filter(it -> nonNull(it.getValue()) && isNotBlank(Objects.toString(it.getValue()))) + .filter(it -> nonNull(it.getValue()) && + isNotBlank(Objects.toString(it.getValue()))) .map(it -> { final Object val = it.getValue(); String valStr = Objects.toString(val); @@ -176,7 +179,7 @@ public class ExtractorSpec extends AbstractSpec { } @SuppressWarnings("unused") - public void metrics(final Closure cl) { + public void metrics(@DelegatesTo(SampleBuilder.class) final Closure cl) { if (BINDING.get().shouldAbort()) { return; } @@ -188,8 +191,8 @@ public class ExtractorSpec extends AbstractSpec { metricConverts.forEach(it -> it.toMeter( ImmutableMap.builder() - .put(sample.getName(), SampleFamilyBuilder.newBuilder(sample).build()) - .build() + .put(sample.getName(), SampleFamilyBuilder.newBuilder(sample).build()) + .build() )); } @@ -198,11 +201,15 @@ public class ExtractorSpec extends AbstractSpec { private final Sample.SampleBuilder sampleBuilder = Sample.builder(); @SuppressWarnings("unused") - public Sample.SampleBuilder labels(final Map labels) { - final Map filtered = labels.entrySet() - .stream() - .filter(it -> isNotBlank(it.getKey()) && isNotBlank(it.getValue())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + public Sample.SampleBuilder labels(final Map labels) { + final Map filtered = + labels.entrySet() + .stream() + .filter(it -> isNotBlank(it.getKey()) && nonNull(it.getValue())) + .collect( + Collectors.toMap(Map.Entry::getKey, + it -> Objects.toString(it.getValue())) + ); return sampleBuilder.labels(ImmutableMap.copyOf(filtered)); } } diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java index 7683bb63e..43f86a360 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java @@ -18,10 +18,10 @@ package org.apache.skywalking.oap.log.analyzer.dsl.spec.filter; -import com.google.gson.reflect.TypeToken; +import com.fasterxml.jackson.core.type.TypeReference; import com.google.protobuf.TextFormat; import groovy.lang.Closure; -import java.lang.reflect.Type; +import groovy.lang.DelegatesTo; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -57,14 +57,14 @@ public class FilterSpec extends AbstractSpec { private final SinkSpec sink; - private final Type parsedType; + private final TypeReference> parsedType; public FilterSpec(final ModuleManager moduleManager, final LogAnalyzerModuleConfig moduleConfig) throws ModuleStartException { super(moduleManager, moduleConfig); - parsedType = new TypeToken>() { - }.getType(); + parsedType = new TypeReference>() { + }; factories = Arrays.asList( new RecordAnalysisListener.Factory(moduleManager(), moduleConfig()), @@ -81,7 +81,7 @@ public class FilterSpec extends AbstractSpec { } @SuppressWarnings("unused") - public void text(final Closure cl) { + public void text(@DelegatesTo(TextParserSpec.class) final Closure cl) { if (BINDING.get().shouldAbort()) { return; } @@ -90,7 +90,7 @@ public class FilterSpec extends AbstractSpec { } @SuppressWarnings("unused") - public void json(final Closure cl) { + public void json(@DelegatesTo(JsonParserSpec.class) final Closure cl) { if (BINDING.get().shouldAbort()) { return; } @@ -99,7 +99,8 @@ public class FilterSpec extends AbstractSpec { final LogData.Builder logData = BINDING.get().log(); try { - final Map parsed = jsonParser.create().fromJson( + + final Map parsed = jsonParser.create().readValue( logData.getBody().getJson().getJson(), parsedType ); @@ -111,8 +112,8 @@ public class FilterSpec extends AbstractSpec { } } - @SuppressWarnings({"unused", "unchecked"}) - public void yaml(final Closure cl) { + @SuppressWarnings({"unused"}) + public void yaml(@DelegatesTo(YamlParserSpec.class) final Closure cl) { if (BINDING.get().shouldAbort()) { return; } @@ -121,7 +122,7 @@ public class FilterSpec extends AbstractSpec { final LogData.Builder logData = BINDING.get().log(); try { - final Map parsed = (Map) yamlParser.create().load( + final Map parsed = yamlParser.create().load( logData.getBody().getYaml().getYaml() ); @@ -134,7 +135,7 @@ public class FilterSpec extends AbstractSpec { } @SuppressWarnings("unused") - public void extractor(final Closure cl) { + public void extractor(@DelegatesTo(ExtractorSpec.class) final Closure cl) { if (BINDING.get().shouldAbort()) { return; } @@ -143,7 +144,7 @@ public class FilterSpec extends AbstractSpec { } @SuppressWarnings("unused") - public void sink(final Closure cl) { + public void sink(@DelegatesTo(SinkSpec.class) final Closure cl) { if (BINDING.get().shouldAbort()) { return; } @@ -166,7 +167,7 @@ public class FilterSpec extends AbstractSpec { } @SuppressWarnings("unused") - public void filter(final Closure cl) { + public void filter(final Closure cl) { cl.call(); } } diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/AbstractParserSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/AbstractParserSpec.java index e410cbe8b..a7db11902 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/AbstractParserSpec.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/AbstractParserSpec.java @@ -18,26 +18,32 @@ package org.apache.skywalking.oap.log.analyzer.dsl.spec.parser; -import lombok.Getter; -import lombok.Setter; import lombok.experimental.Accessors; import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec; import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig; import org.apache.skywalking.oap.server.library.module.ModuleManager; -@Accessors(fluent = true) +@Accessors public class AbstractParserSpec extends AbstractSpec { /** * Whether the filter chain should abort when parsing the logs failed. * - * Failing to parse the logs means either parsing throws exceptions or the logs not matching the desired patterns. + * Failing to parse the logs means either parsing throws exceptions or the logs not matching the + * desired patterns. */ - @Getter - @Setter private boolean abortOnFailure = true; public AbstractParserSpec(final ModuleManager moduleManager, final LogAnalyzerModuleConfig moduleConfig) { super(moduleManager, moduleConfig); } + + @SuppressWarnings("unused") // used in user LAL scripts + public void abortOnFailure(final boolean abortOnFailure) { + this.abortOnFailure = abortOnFailure; + } + + public boolean abortOnFailure() { + return this.abortOnFailure; + } } diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java index 850daab1e..1fabdbcf1 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java @@ -18,28 +18,23 @@ package org.apache.skywalking.oap.log.analyzer.dsl.spec.parser; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig; import org.apache.skywalking.oap.server.library.module.ModuleManager; public class JsonParserSpec extends AbstractParserSpec { - private final GsonBuilder gsonBuilder; - - private final Gson gson; + private final ObjectMapper mapper; public JsonParserSpec(final ModuleManager moduleManager, final LogAnalyzerModuleConfig moduleConfig) { super(moduleManager, moduleConfig); - gsonBuilder = new GsonBuilder(); - - // We just create a gson instance in advance for now (for the sake of performance), + // We just create a mapper instance in advance for now (for the sake of performance), // when we want to provide some extra options, we'll move this into method "create" then. - gson = gsonBuilder.create(); + mapper = new ObjectMapper(); } - public Gson create() { - return gson; + public ObjectMapper create() { + return mapper; } } diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/TextParserSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/TextParserSpec.java index c77f63911..f20a1e9d8 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/TextParserSpec.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/TextParserSpec.java @@ -31,13 +31,13 @@ public class TextParserSpec extends AbstractParserSpec { } @SuppressWarnings("unused") - public boolean regexp(final String regexp) { - return regexp(Pattern.compile(regexp)); + public void regexp(final String regexp) { + regexp(Pattern.compile(regexp)); } - public boolean regexp(final Pattern pattern) { + public void regexp(final Pattern pattern) { if (BINDING.get().shouldAbort()) { - return false; + return; } final LogData.Builder log = BINDING.get().log(); final Matcher matcher = pattern.matcher(log.getBody().getText().getText()); @@ -47,7 +47,6 @@ public class TextParserSpec extends AbstractParserSpec { } else if (abortOnFailure()) { BINDING.get().abort(); } - return matched; } public boolean grok(final String grok) { diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SamplerSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SamplerSpec.java index e32f79238..77df93541 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SamplerSpec.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SamplerSpec.java @@ -19,6 +19,8 @@ package org.apache.skywalking.oap.log.analyzer.dsl.spec.sink; import groovy.lang.Closure; +import groovy.lang.DelegatesTo; +import groovy.lang.GString; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec; @@ -28,7 +30,7 @@ import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig; import org.apache.skywalking.oap.server.library.module.ModuleManager; public class SamplerSpec extends AbstractSpec { - private final Map samplers; + private final Map samplers; private final RateLimitingSampler.ResetHandler rlsResetHandler; public SamplerSpec(final ModuleManager moduleManager, @@ -40,7 +42,7 @@ public class SamplerSpec extends AbstractSpec { } @SuppressWarnings("unused") - public void rateLimit(final String id, final Closure cl) { + public void rateLimit(final GString id, @DelegatesTo(RateLimitingSampler.class) final Closure cl) { if (BINDING.get().shouldAbort()) { return; } diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SinkSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SinkSpec.java index ee78be982..82566f9b2 100644 --- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SinkSpec.java +++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SinkSpec.java @@ -19,6 +19,7 @@ package org.apache.skywalking.oap.log.analyzer.dsl.spec.sink; import groovy.lang.Closure; +import groovy.lang.DelegatesTo; import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec; import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig; import org.apache.skywalking.oap.server.library.module.ModuleManager; @@ -35,7 +36,7 @@ public class SinkSpec extends AbstractSpec { } @SuppressWarnings("unused") - public void sampler(final Closure cl) { + public void sampler(@DelegatesTo(SamplerSpec.class) final Closure cl) { if (BINDING.get().shouldAbort()) { return; } @@ -44,7 +45,7 @@ public class SinkSpec extends AbstractSpec { } @SuppressWarnings("unused") - public void enforcer(final Closure cl) { + public void enforcer(final Closure cl) { if (BINDING.get().shouldAbort()) { return; } @@ -52,7 +53,7 @@ public class SinkSpec extends AbstractSpec { } @SuppressWarnings("unused") - public void dropper(final Closure cl) { + public void dropper(final Closure cl) { if (BINDING.get().shouldAbort()) { return; } diff --git a/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/dsl/DSLTest.java b/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/dsl/DSLTest.java new file mode 100644 index 000000000..36f5451cd --- /dev/null +++ b/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/dsl/DSLTest.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.log.analyzer.dsl; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import org.apache.skywalking.apm.network.logging.v3.LogData; +import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.config.ConfigService; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder; +import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.powermock.reflect.Whitebox; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(Parameterized.class) +public class DSLTest { + @Parameterized.Parameters(name = "{index}: {0}") + public static Collection data() { + return Arrays.asList( + new String[] { + "parser", + "filter {\n" + + " json {\n" + + " abortOnFailure false // for test purpose, we want to persist all logs\n" + + " }\n" + + " text {\n" + + " abortOnFailure false // for test purpose, we want to persist all logs\n" + + " regexp $/(?s)(?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.\\d{3}) \\[TID:(?.+?)] \\[(?.+?)] (?\\w{4,}) (?.{1,36}) (?.+)/$" + + " }\n" + + " yaml {\n" + + " abortOnFailure false // for test purpose, we want to persist all logs\n" + + " }" + + "}", + }, + new String[] { + "extractor", + "filter {\n" + + " extractor {\n" + + " service \"test\"\n" + + " instance \"test\"\n" + + " endpoint \"test\"\n" + + " traceId \"123\"\n" + + " segmentId \"123\"\n" + + " spanId \"123\"\n" + + " timestamp \"123\"\n" + + " metrics {\n" + + " name \"metricsName\"\n" + + " value 123\n" + + " timestamp \"123\"\n" + + " labels \"k1\": \"v1\"\n" + + " }\n" + + " }\n" + + "}", + }, + new String[] { + "sink", + "filter {\n" + + " sink {\n" + + " enforcer {\n" + + " }\n" + + " dropper {\n" + + " }\n" + + " sampler {\n" + + " if (parsed?.commonProperties?.responseFlags) {\n" + + " // use service:errorCode as sampler id so that each service:errorCode has its own sampler,\n" + + " // e.g. checkoutservice:[upstreamConnectionFailure], checkoutservice:[upstreamRetryLimitExceeded]\n" + + " rateLimit(\"${log.service}:${log.body.json.json}:${log.tags.getData(0).key}:${parsed?.commonProperties?.responseFlags}\") {\n" + + " qps 100\n" + + " }\n" + + " } else {\n" + + " // use service:responseCode as sampler id so that each service:responseCode has its own sampler,\n" + + " // e.g. checkoutservice:500, checkoutservice:404.\n" + + " rateLimit(\"${log.service}:${log.body?.type}:${log.traceContext?.traceId}:${parsed?.response?.responseCode}\") {\n" + + " qps 100\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", + }, + new String[] { + "e2e", + "filter {\n" + + " text {\n" + + " abortOnFailure false // for test purpose, we want to persist all logs\n" + + " regexp $/(?s)(?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.\\d{3}) \\[TID:(?.+?)] \\[(?.+?)] (?\\w{4,}) (?.{1,36}) (?.+)/$\n" + + " }\n" + + " extractor {\n" + + " metrics {\n" + + " timestamp \"${log.timestamp}\"\n" + + " labels level: parsed.level, service: log.service, instance: log.serviceInstance\n" + + " name \"log_count\"\n" + + " value 1\n" + + " }\n" + + " }\n" + + " sink {\n" + + " }\n" + + "}\n", + }, + new String[] { + "e2e", + "filter {\n" + + " json {\n" + + " }\n" + + " // only collect abnormal logs (http status code >= 300, or commonProperties?.responseFlags is not empty)\n" + + " if (parsed?.response?.responseCode as Integer < 400 && !parsed?.commonProperties?.responseFlags) {\n" + + " abort {}\n" + + " }\n" + + " extractor {\n" + + " if (parsed?.response?.responseCode) {\n" + + " tag 'status.code': parsed?.response?.responseCode as int\n" + + " }\n" + + " tag 'response.flag': (parsed?.commonProperties?.responseFlags as Map)?.keySet()\n" + + " }\n" + + " sink {\n" + + " sampler {\n" + + " if (parsed?.commonProperties?.responseFlags) {\n" + + " // use service:errorCode as sampler id so that each service:errorCode has its own sampler,\n" + + " // e.g. checkoutservice:[upstreamConnectionFailure], checkoutservice:[upstreamRetryLimitExceeded]\n" + + " rateLimit(\"${log.service}:${(parsed?.commonProperties?.responseFlags as Map)?.keySet()}\") {\n" + + " qps 100\n" + + " }\n" + + " } else {\n" + + " // use service:responseCode as sampler id so that each service:responseCode has its own sampler,\n" + + " // e.g. checkoutservice:500, checkoutservice:404.\n" + + " rateLimit(\"${log.service}:${parsed?.response?.responseCode}\") {\n" + + " qps 100\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n", + } + ); + } + + @Parameterized.Parameter() + public String name; + + @Parameterized.Parameter(1) + public String script; + + final ModuleManager manager = mock(ModuleManager.class); + + @Before + public void setup() { + Whitebox.setInternalState(manager, "isInPrepareStage", false); + when(manager.find(anyString())).thenReturn(mock(ModuleProviderHolder.class)); + when(manager.find(CoreModule.NAME).provider()).thenReturn(mock(ModuleServiceHolder.class)); + when(manager.find(CoreModule.NAME).provider().getService(SourceReceiver.class)) + .thenReturn(mock(SourceReceiver.class)); + when(manager.find(CoreModule.NAME).provider().getService(ConfigService.class)) + .thenReturn(mock(ConfigService.class)); + when(manager.find(CoreModule.NAME) + .provider() + .getService(ConfigService.class) + .getSearchableLogsTags()) + .thenReturn(""); + } + + @Test + public void testDslStaticCompile() throws ModuleStartException { + final DSL dsl = DSL.of(manager, new LogAnalyzerModuleConfig(), script); + Whitebox.setInternalState( + Whitebox.getInternalState(dsl, "filterSpec"), "factories", Collections.emptyList() + ); + + dsl.bind(new Binding().log(LogData.newBuilder().build())); + dsl.evaluate(); + } +} diff --git a/oap-server/analyzer/log-analyzer/src/test/resources/log-mal-rules/placeholder.yaml b/oap-server/analyzer/log-analyzer/src/test/resources/log-mal-rules/placeholder.yaml new file mode 100644 index 000000000..034440ce5 --- /dev/null +++ b/oap-server/analyzer/log-analyzer/src/test/resources/log-mal-rules/placeholder.yaml @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Refer to examples in config-examples/log-mal.yaml diff --git a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 index 25654dacb..b881a98e1 100644 --- a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 +++ b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 @@ -39,7 +39,7 @@ SRC_SERVICE_INSTANCE_JVM_MEMORY: 'ServiceInstanceJVMMemory'; SRC_SERVICE_INSTANCE_JVM_MEMORY_POOL: 'ServiceInstanceJVMMemoryPool'; SRC_SERVICE_INSTANCE_JVM_GC: 'ServiceInstanceJVMGC'; SRC_SERVICE_INSTANCE_JVM_THREAD: 'ServiceInstanceJVMThread'; -SRC_SERVICE_INSTANCE_JVM_CLASS:'ServiceInstanceJVMClass'; +SRC_SERVICE_INSTANCE_JVM_CLASS: 'ServiceInstanceJVMClass'; SRC_DATABASE_ACCESS: 'DatabaseAccess'; SRC_SERVICE_INSTANCE_CLR_CPU: 'ServiceInstanceCLRCPU'; SRC_SERVICE_INSTANCE_CLR_GC: 'ServiceInstanceCLRGC'; diff --git a/oap-server/server-bootstrap/src/main/resources/lal/envoy-als.yaml b/oap-server/server-bootstrap/src/main/resources/lal/envoy-als.yaml index 96a7c1a28..aad0a6159 100644 --- a/oap-server/server-bootstrap/src/main/resources/lal/envoy-als.yaml +++ b/oap-server/server-bootstrap/src/main/resources/lal/envoy-als.yaml @@ -20,21 +20,21 @@ rules: json { } // only collect abnormal logs (http status code >= 300, or commonProperties?.responseFlags is not empty) - if (parsed?.response?.responseCode < 400 && !parsed?.commonProperties?.responseFlags) { + if (parsed?.response?.responseCode as Integer < 400 && !parsed?.commonProperties?.responseFlags) { abort {} } extractor { if (parsed?.response?.responseCode) { tag 'status.code': parsed?.response?.responseCode as int } - tag 'response.flag': parsed?.commonProperties?.responseFlags?.keySet() + tag 'response.flag': (parsed?.commonProperties?.responseFlags as Map)?.keySet() } sink { sampler { if (parsed?.commonProperties?.responseFlags) { // use service:errorCode as sampler id so that each service:errorCode has its own sampler, // e.g. checkoutservice:[upstreamConnectionFailure], checkoutservice:[upstreamRetryLimitExceeded] - rateLimit("${log.service}:${parsed?.commonProperties?.responseFlags?.keySet()}") { + rateLimit("${log.service}:${(parsed?.commonProperties?.responseFlags as Map)?.keySet()}") { qps 100 } } else { diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/mock/MockModuleManager.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/mock/MockModuleManager.java index 8d5ee8aa3..63c95bb34 100644 --- a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/mock/MockModuleManager.java +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/mock/MockModuleManager.java @@ -37,10 +37,12 @@ public abstract class MockModuleManager extends ModuleManager { moduleProviderHolderMap.put(name, provider); } + @Override public boolean has(String moduleName) { return moduleProviderHolderMap.containsKey(moduleName); } + @Override public ModuleProviderHolder find(String moduleName) throws ModuleNotFoundRuntimeException { if (!moduleProviderHolderMap.containsKey(moduleName)) { throw new ModuleNotFoundRuntimeException("ModuleProviderHolder[" + moduleName + "] cannot found in MOCK."); diff --git a/test/e2e/e2e-test/docker/log/lal.yaml b/test/e2e/e2e-test/docker/log/lal.yaml index 2b949885f..bfd7a6105 100644 --- a/test/e2e/e2e-test/docker/log/lal.yaml +++ b/test/e2e/e2e-test/docker/log/lal.yaml @@ -23,7 +23,7 @@ rules: } extractor { metrics { - timestamp log.timestamp + timestamp log.timestamp as Long labels level: parsed.level, service: log.service, instance: log.serviceInstance name "log_count" value 1 diff --git a/tools/dependencies/known-oap-backend-dependencies-es7.txt b/tools/dependencies/known-oap-backend-dependencies-es7.txt index 8e4b67795..8f2c7b686 100755 --- a/tools/dependencies/known-oap-backend-dependencies-es7.txt +++ b/tools/dependencies/known-oap-backend-dependencies-es7.txt @@ -47,7 +47,7 @@ flatbuffers-java-1.12.0.jar freemarker-2.3.28.jar graphql-java-8.0.jar graphql-java-tools-5.2.3.jar -groovy-3.0.3.jar +groovy-3.0.8.jar grpc-api-1.32.1.jar grpc-context-1.32.1.jar grpc-core-1.32.1.jar diff --git a/tools/dependencies/known-oap-backend-dependencies.txt b/tools/dependencies/known-oap-backend-dependencies.txt index 16d38cdf4..9dcd63e5f 100755 --- a/tools/dependencies/known-oap-backend-dependencies.txt +++ b/tools/dependencies/known-oap-backend-dependencies.txt @@ -45,7 +45,7 @@ flatbuffers-java-1.12.0.jar freemarker-2.3.28.jar graphql-java-8.0.jar graphql-java-tools-5.2.3.jar -groovy-3.0.3.jar +groovy-3.0.8.jar grpc-api-1.32.1.jar grpc-context-1.32.1.jar grpc-core-1.32.1.jar