kazusa
/
skywalking-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

perf: optimize Groovy-based LAL DSL with static compilation (#7263) 

Groovy naturally supports many dynamic features that we don't benefit for now but cost performance loss, in this patch we compile our Groovy-based DSL scripts statically to optimize performance.
This commit is contained in:
kezhenxu94 2021-07-09 18:00:25 +08:00 committed by GitHub
parent 58dce444fc
commit fd92f4678a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 456 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.md
View File

@ -77,6 +77,7 @@ Release Notes.
* Upgrade etcd cluster coordinator and dynamic configuration to v3.x.
* Configuration: Allow to configure server maximum request header size.
* Add thread state metric and class loaded info metric to JVMMetric.
* Performance: compile LAL DSL statically and run with type checked.
#### UI

2
Makefile
View File

@ -96,7 +96,7 @@ docker.agent: $(SW_ROOT)/docker/agent/Dockerfile.agent
# 4. This rule runs $(BUILD_PRE) prior to any docker build and only if specified as a dependency variable
# 5. This rule finally runs docker build passing $(BUILD_ARGS) to docker if they are specified as a dependency variable
DOCKER_RULE=time (mkdir -p $(DOCKER_BUILD_TOP)/$@ && cp -r $^ $(DOCKER_BUILD_TOP)/$@ && cd $(DOCKER_BUILD_TOP)/$@ && $(BUILD_PRE) docker build $(BUILD_ARGS) -t $(HUB)/$(subst docker.,,$@):$(TAG) -f Dockerfile$(suffix $@) .)
DOCKER_RULE=time (mkdir -p $(DOCKER_BUILD_TOP)/$@ && cp -r $^ $(DOCKER_BUILD_TOP)/$@ && cd $(DOCKER_BUILD_TOP)/$@ && $(BUILD_PRE) docker build --no-cache $(BUILD_ARGS) -t $(HUB)/$(subst docker.,,$@):$(TAG) -f Dockerfile$(suffix $@) .)
# for each docker.XXX target create a push.docker.XXX target that pushes
# the local docker image to another hub

2
dist-material/release-docs/LICENSE
View File

@ -455,7 +455,7 @@ Apache 2.0 licenses
========================================
echarts 5.0.2: https://github.com/apache/echarts Apache-2.0
Material Icons 3.0.1 https://github.com/google/material-design-icons Apache-2.0
groovy 3.0.3 https://github.com/apache/groovy Apache-2.0
groovy 3.0.8 https://github.com/apache/groovy Apache-2.0
========================================
BSD licenses

2
oap-server-bom/pom.xml
View File

@ -68,7 +68,7 @@
<freemarker.version>2.3.28</freemarker.version>
<javaassist.version>3.25.0-GA</javaassist.version>
<vavr.version>0.10.3</vavr.version>
<groovy.version>3.0.3</groovy.version>
<groovy.version>3.0.8</groovy.version>
<mvel.version>2.4.8.Final</mvel.version>
<commons-beanutils.version>1.9.4</commons-beanutils.version>
<flatbuffers-java.version>1.12.0</flatbuffers-java.version>

4
oap-server/analyzer/log-analyzer/pom.xml
View File

@ -42,6 +42,10 @@
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
</dependencies>
</project>

17
oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/DSL.java
View File

@ -19,14 +19,20 @@
package org.apache.skywalking.oap.log.analyzer.dsl;
import groovy.lang.GroovyShell;
import groovy.transform.CompileStatic;
import groovy.util.DelegatingScript;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import org.apache.skywalking.oap.log.analyzer.dsl.spec.LALDelegatingScript;
import org.apache.skywalking.oap.log.analyzer.dsl.spec.filter.FilterSpec;
import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
import org.apache.skywalking.oap.server.library.module.ModuleStartException;
import org.codehaus.groovy.control.CompilerConfiguration;
import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer;
import static java.util.Collections.singletonList;
import static java.util.Collections.singletonMap;
@RequiredArgsConstructor(access = AccessLevel.PRIVATE)
public class DSL {
@ -38,7 +44,16 @@ public class DSL {
final LogAnalyzerModuleConfig config,
final String dsl) throws ModuleStartException {
final CompilerConfiguration cc = new CompilerConfiguration();
cc.setScriptBaseClass(DelegatingScript.class.getName());
final ASTTransformationCustomizer customizer =
new ASTTransformationCustomizer(
singletonMap(
"extensions",
singletonList(LALPrecompiledExtension.class.getName())
),
CompileStatic.class
);
cc.addCompilationCustomizers(customizer);
cc.setScriptBaseClass(LALDelegatingScript.class.getName());
final GroovyShell sh = new GroovyShell(cc);
final DelegatingScript script = (DelegatingScript) sh.parse(dsl);

89
oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/LALPrecompiledExtension.java Normal file
View File

@ -0,0 +1,89 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.log.analyzer.dsl;
import org.apache.skywalking.apm.network.logging.v3.LogData;
import org.apache.skywalking.apm.network.logging.v3.LogDataBody;
import org.apache.skywalking.apm.network.logging.v3.LogTags;
import org.apache.skywalking.apm.network.logging.v3.TraceContext;
import org.codehaus.groovy.ast.expr.ConstantExpression;
import org.codehaus.groovy.ast.expr.Expression;
import org.codehaus.groovy.ast.expr.PropertyExpression;
import org.codehaus.groovy.ast.expr.VariableExpression;
import org.codehaus.groovy.transform.stc.AbstractTypeCheckingExtension;
import org.codehaus.groovy.transform.stc.StaticTypeCheckingVisitor;
import static org.codehaus.groovy.ast.ClassHelper.makeCached;
public class LALPrecompiledExtension extends AbstractTypeCheckingExtension {
public LALPrecompiledExtension(final StaticTypeCheckingVisitor typeCheckingVisitor) {
super(typeCheckingVisitor);
}
@Override
public boolean handleUnresolvedProperty(final PropertyExpression pexp) {
final Expression exp = pexp.getObjectExpression();
if (exp.getText().startsWith("parsed")) {
makeDynamic(pexp);
setHandled(true);
return true;
}
if (exp.getText().startsWith("log")) {
if (handleLogVariable(pexp)) {
return true;
}
}
return super.handleUnresolvedProperty(pexp);
}
private boolean handleLogVariable(final PropertyExpression pexp) {
final Expression exp = pexp.getObjectExpression();
final Expression p = pexp.getProperty();
if (exp instanceof VariableExpression) {
final VariableExpression v = (VariableExpression) exp;
if (v.getName().equals("log")) {
storeType(v, makeCached(LogData.Builder.class));
}
if (p instanceof ConstantExpression) {
final ConstantExpression c = (ConstantExpression) p;
switch (c.getText()) {
case "body":
storeType(pexp, makeCached(LogDataBody.class));
break;
case "traceContext":
storeType(pexp, makeCached(TraceContext.class));
break;
case "tags":
storeType(pexp, makeCached(LogTags.class));
break;
}
}
setHandled(true);
return true;
}
return false;
}
}

56
oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/LALDelegatingScript.java Normal file
View File

@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.log.analyzer.dsl.spec;
import groovy.lang.Closure;
import groovy.lang.DelegatesTo;
import groovy.util.DelegatingScript;
import org.apache.skywalking.oap.log.analyzer.dsl.spec.filter.FilterSpec;
public class LALDelegatingScript extends DelegatingScript {
@Override
public Object run() {
return null;
}
public void filter(@DelegatesTo(value = FilterSpec.class, strategy = Closure.DELEGATE_ONLY) Closure<?> closure) {
closure.setDelegate(getDelegate());
closure.call();
}
public void json(@DelegatesTo(value = FilterSpec.class) Closure<?> closure) {
closure.setDelegate(getDelegate());
closure.call();
}
public void text(@DelegatesTo(value = FilterSpec.class) Closure<?> closure) {
closure.setDelegate(getDelegate());
closure.call();
}
public void extractor(@DelegatesTo(value = FilterSpec.class) Closure<?> closure) {
closure.setDelegate(getDelegate());
closure.call();
}
public void sink(@DelegatesTo(value = FilterSpec.class) Closure<?> closure) {
closure.setDelegate(getDelegate());
closure.call();
}
}

29
oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java
View File

@ -21,6 +21,7 @@ package org.apache.skywalking.oap.log.analyzer.dsl.spec.extractor;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableMap;
import groovy.lang.Closure;
import groovy.lang.DelegatesTo;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@ -54,7 +55,8 @@ public class ExtractorSpec extends AbstractSpec {
final LogAnalyzerModuleConfig moduleConfig) throws ModuleStartException {
super(moduleManager, moduleConfig);
final MeterSystem meterSystem = moduleManager.find(CoreModule.NAME).provider().getService(MeterSystem.class);
final MeterSystem meterSystem =
moduleManager.find(CoreModule.NAME).provider().getService(MeterSystem.class);
metricConverts = moduleConfig.malConfigs()
.stream()
@ -93,7 +95,7 @@ public class ExtractorSpec extends AbstractSpec {
}
@SuppressWarnings("unused")
public void tag(final Map<String, Object> kv) {
public void tag(final Map<String, ?> kv) {
if (BINDING.get().shouldAbort()) {
return;
}
@ -108,7 +110,8 @@ public class ExtractorSpec extends AbstractSpec {
kv.entrySet()
.stream()
.filter(it -> isNotBlank(it.getKey()))
.filter(it -> nonNull(it.getValue()) && isNotBlank(Objects.toString(it.getValue())))
.filter(it -> nonNull(it.getValue()) &&
isNotBlank(Objects.toString(it.getValue())))
.map(it -> {
final Object val = it.getValue();
String valStr = Objects.toString(val);
@ -176,7 +179,7 @@ public class ExtractorSpec extends AbstractSpec {
}
@SuppressWarnings("unused")
public void metrics(final Closure<Void> cl) {
public void metrics(@DelegatesTo(SampleBuilder.class) final Closure<?> cl) {
if (BINDING.get().shouldAbort()) {
return;
}
@ -188,8 +191,8 @@ public class ExtractorSpec extends AbstractSpec {
metricConverts.forEach(it -> it.toMeter(
ImmutableMap.<String, SampleFamily>builder()
.put(sample.getName(), SampleFamilyBuilder.newBuilder(sample).build())
.build()
.put(sample.getName(), SampleFamilyBuilder.newBuilder(sample).build())
.build()
));
}
@ -198,11 +201,15 @@ public class ExtractorSpec extends AbstractSpec {
private final Sample.SampleBuilder sampleBuilder = Sample.builder();
@SuppressWarnings("unused")
public Sample.SampleBuilder labels(final Map<String, String> labels) {
final Map<String, String> filtered = labels.entrySet()
.stream()
.filter(it -> isNotBlank(it.getKey()) && isNotBlank(it.getValue()))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
public Sample.SampleBuilder labels(final Map<String, ?> labels) {
final Map<String, String> filtered =
labels.entrySet()
.stream()
.filter(it -> isNotBlank(it.getKey()) && nonNull(it.getValue()))
.collect(
Collectors.toMap(Map.Entry::getKey,
it -> Objects.toString(it.getValue()))
);
return sampleBuilder.labels(ImmutableMap.copyOf(filtered));
}
}

29
oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java
View File

@ -18,10 +18,10 @@
package org.apache.skywalking.oap.log.analyzer.dsl.spec.filter;
import com.google.gson.reflect.TypeToken;
import com.fasterxml.jackson.core.type.TypeReference;
import com.google.protobuf.TextFormat;
import groovy.lang.Closure;
import java.lang.reflect.Type;
import groovy.lang.DelegatesTo;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
@ -57,14 +57,14 @@ public class FilterSpec extends AbstractSpec {
private final SinkSpec sink;
private final Type parsedType;
private final TypeReference<Map<String, Object>> parsedType;
public FilterSpec(final ModuleManager moduleManager,
final LogAnalyzerModuleConfig moduleConfig) throws ModuleStartException {
super(moduleManager, moduleConfig);
parsedType = new TypeToken<Map<String, Object>>() {
}.getType();
parsedType = new TypeReference<Map<String, Object>>() {
};
factories = Arrays.asList(
new RecordAnalysisListener.Factory(moduleManager(), moduleConfig()),
@ -81,7 +81,7 @@ public class FilterSpec extends AbstractSpec {
}
@SuppressWarnings("unused")
public void text(final Closure<Void> cl) {
public void text(@DelegatesTo(TextParserSpec.class) final Closure<?> cl) {
if (BINDING.get().shouldAbort()) {
return;
}
@ -90,7 +90,7 @@ public class FilterSpec extends AbstractSpec {
}
@SuppressWarnings("unused")
public void json(final Closure<Void> cl) {
public void json(@DelegatesTo(JsonParserSpec.class) final Closure<?> cl) {
if (BINDING.get().shouldAbort()) {
return;
}
@ -99,7 +99,8 @@ public class FilterSpec extends AbstractSpec {
final LogData.Builder logData = BINDING.get().log();
try {
final Map<String, Object> parsed = jsonParser.create().fromJson(
final Map<String, Object> parsed = jsonParser.create().readValue(
logData.getBody().getJson().getJson(), parsedType
);
@ -111,8 +112,8 @@ public class FilterSpec extends AbstractSpec {
}
}
@SuppressWarnings({"unused", "unchecked"})
public void yaml(final Closure<Void> cl) {
@SuppressWarnings({"unused"})
public void yaml(@DelegatesTo(YamlParserSpec.class) final Closure<?> cl) {
if (BINDING.get().shouldAbort()) {
return;
}
@ -121,7 +122,7 @@ public class FilterSpec extends AbstractSpec {
final LogData.Builder logData = BINDING.get().log();
try {
final Map<String, Object> parsed = (Map<String, Object>) yamlParser.create().load(
final Map<String, Object> parsed = yamlParser.create().load(
logData.getBody().getYaml().getYaml()
);
@ -134,7 +135,7 @@ public class FilterSpec extends AbstractSpec {
}
@SuppressWarnings("unused")
public void extractor(final Closure<Void> cl) {
public void extractor(@DelegatesTo(ExtractorSpec.class) final Closure<?> cl) {
if (BINDING.get().shouldAbort()) {
return;
}
@ -143,7 +144,7 @@ public class FilterSpec extends AbstractSpec {
}
@SuppressWarnings("unused")
public void sink(final Closure<Void> cl) {
public void sink(@DelegatesTo(SinkSpec.class) final Closure<?> cl) {
if (BINDING.get().shouldAbort()) {
return;
}
@ -166,7 +167,7 @@ public class FilterSpec extends AbstractSpec {
}
@SuppressWarnings("unused")
public void filter(final Closure<Void> cl) {
public void filter(final Closure<?> cl) {
cl.call();
}
}

18
oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/AbstractParserSpec.java
View File

@ -18,26 +18,32 @@
package org.apache.skywalking.oap.log.analyzer.dsl.spec.parser;
import lombok.Getter;
import lombok.Setter;
import lombok.experimental.Accessors;
import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec;
import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
@Accessors(fluent = true)
@Accessors
public class AbstractParserSpec extends AbstractSpec {
/**
* Whether the filter chain should abort when parsing the logs failed.
*
* Failing to parse the logs means either parsing throws exceptions or the logs not matching the desired patterns.
* Failing to parse the logs means either parsing throws exceptions or the logs not matching the
* desired patterns.
*/
@Getter
@Setter
private boolean abortOnFailure = true;
public AbstractParserSpec(final ModuleManager moduleManager,
final LogAnalyzerModuleConfig moduleConfig) {
super(moduleManager, moduleConfig);
}
@SuppressWarnings("unused") // used in user LAL scripts
public void abortOnFailure(final boolean abortOnFailure) {
this.abortOnFailure = abortOnFailure;
}
public boolean abortOnFailure() {
return this.abortOnFailure;
}
}

17
oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java
View File

@ -18,28 +18,23 @@
package org.apache.skywalking.oap.log.analyzer.dsl.spec.parser;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
public class JsonParserSpec extends AbstractParserSpec {
private final GsonBuilder gsonBuilder;
private final Gson gson;
private final ObjectMapper mapper;
public JsonParserSpec(final ModuleManager moduleManager,
final LogAnalyzerModuleConfig moduleConfig) {
super(moduleManager, moduleConfig);
gsonBuilder = new GsonBuilder();
// We just create a gson instance in advance for now (for the sake of performance),
// We just create a mapper instance in advance for now (for the sake of performance),
// when we want to provide some extra options, we'll move this into method "create" then.
gson = gsonBuilder.create();
mapper = new ObjectMapper();
}
public Gson create() {
return gson;
public ObjectMapper create() {
return mapper;
}
}

9
oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/TextParserSpec.java
View File

@ -31,13 +31,13 @@ public class TextParserSpec extends AbstractParserSpec {
}
@SuppressWarnings("unused")
public boolean regexp(final String regexp) {
return regexp(Pattern.compile(regexp));
public void regexp(final String regexp) {
regexp(Pattern.compile(regexp));
}
public boolean regexp(final Pattern pattern) {
public void regexp(final Pattern pattern) {
if (BINDING.get().shouldAbort()) {
return false;
return;
}
final LogData.Builder log = BINDING.get().log();
final Matcher matcher = pattern.matcher(log.getBody().getText().getText());
@ -47,7 +47,6 @@ public class TextParserSpec extends AbstractParserSpec {
} else if (abortOnFailure()) {
BINDING.get().abort();
}
return matched;
}
public boolean grok(final String grok) {

6
oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SamplerSpec.java
View File

@ -19,6 +19,8 @@
package org.apache.skywalking.oap.log.analyzer.dsl.spec.sink;
import groovy.lang.Closure;
import groovy.lang.DelegatesTo;
import groovy.lang.GString;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec;
@ -28,7 +30,7 @@ import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
public class SamplerSpec extends AbstractSpec {
private final Map<String, Sampler> samplers;
private final Map<GString, Sampler> samplers;
private final RateLimitingSampler.ResetHandler rlsResetHandler;
public SamplerSpec(final ModuleManager moduleManager,
@ -40,7 +42,7 @@ public class SamplerSpec extends AbstractSpec {
}
@SuppressWarnings("unused")
public void rateLimit(final String id, final Closure<Void> cl) {
public void rateLimit(final GString id, @DelegatesTo(RateLimitingSampler.class) final Closure<?> cl) {
if (BINDING.get().shouldAbort()) {
return;
}

7
oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SinkSpec.java
View File

@ -19,6 +19,7 @@
package org.apache.skywalking.oap.log.analyzer.dsl.spec.sink;
import groovy.lang.Closure;
import groovy.lang.DelegatesTo;
import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec;
import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
@ -35,7 +36,7 @@ public class SinkSpec extends AbstractSpec {
}
@SuppressWarnings("unused")
public void sampler(final Closure<Void> cl) {
public void sampler(@DelegatesTo(SamplerSpec.class) final Closure<?> cl) {
if (BINDING.get().shouldAbort()) {
return;
}
@ -44,7 +45,7 @@ public class SinkSpec extends AbstractSpec {
}
@SuppressWarnings("unused")
public void enforcer(final Closure<Void> cl) {
public void enforcer(final Closure<?> cl) {
if (BINDING.get().shouldAbort()) {
return;
}
@ -52,7 +53,7 @@ public class SinkSpec extends AbstractSpec {
}
@SuppressWarnings("unused")
public void dropper(final Closure<Void> cl) {
public void dropper(final Closure<?> cl) {
if (BINDING.get().shouldAbort()) {
return;
}

199
oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/dsl/DSLTest.java Normal file
View File

@ -0,0 +1,199 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.log.analyzer.dsl;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import org.apache.skywalking.apm.network.logging.v3.LogData;
import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
import org.apache.skywalking.oap.server.core.CoreModule;
import org.apache.skywalking.oap.server.core.config.ConfigService;
import org.apache.skywalking.oap.server.core.source.SourceReceiver;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder;
import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder;
import org.apache.skywalking.oap.server.library.module.ModuleStartException;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.powermock.reflect.Whitebox;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@RunWith(Parameterized.class)
public class DSLTest {
@Parameterized.Parameters(name = "{index}: {0}")
public static Collection<Object[]> data() {
return Arrays.asList(
new String[] {
"parser",
"filter {\n" +
" json {\n" +
" abortOnFailure false // for test purpose, we want to persist all logs\n" +
" }\n" +
" text {\n" +
" abortOnFailure false // for test purpose, we want to persist all logs\n" +
" regexp $/(?s)(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.\\d{3}) \\[TID:(?<tid>.+?)] \\[(?<thread>.+?)] (?<level>\\w{4,}) (?<logger>.{1,36}) (?<msg>.+)/$" +
" }\n" +
" yaml {\n" +
" abortOnFailure false // for test purpose, we want to persist all logs\n" +
" }" +
"}",
},
new String[] {
"extractor",
"filter {\n" +
" extractor {\n" +
" service \"test\"\n" +
" instance \"test\"\n" +
" endpoint \"test\"\n" +
" traceId \"123\"\n" +
" segmentId \"123\"\n" +
" spanId \"123\"\n" +
" timestamp \"123\"\n" +
" metrics {\n" +
" name \"metricsName\"\n" +
" value 123\n" +
" timestamp \"123\"\n" +
" labels \"k1\": \"v1\"\n" +
" }\n" +
" }\n" +
"}",
},
new String[] {
"sink",
"filter {\n" +
" sink {\n" +
" enforcer {\n" +
" }\n" +
" dropper {\n" +
" }\n" +
" sampler {\n" +
" if (parsed?.commonProperties?.responseFlags) {\n" +
" // use service:errorCode as sampler id so that each service:errorCode has its own sampler,\n" +
" // e.g. checkoutservice:[upstreamConnectionFailure], checkoutservice:[upstreamRetryLimitExceeded]\n" +
" rateLimit(\"${log.service}:${log.body.json.json}:${log.tags.getData(0).key}:${parsed?.commonProperties?.responseFlags}\") {\n" +
" qps 100\n" +
" }\n" +
" } else {\n" +
" // use service:responseCode as sampler id so that each service:responseCode has its own sampler,\n" +
" // e.g. checkoutservice:500, checkoutservice:404.\n" +
" rateLimit(\"${log.service}:${log.body?.type}:${log.traceContext?.traceId}:${parsed?.response?.responseCode}\") {\n" +
" qps 100\n" +
" }\n" +
" }\n" +
" }\n" +
" }\n" +
"}",
},
new String[] {
"e2e",
"filter {\n" +
" text {\n" +
" abortOnFailure false // for test purpose, we want to persist all logs\n" +
" regexp $/(?s)(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.\\d{3}) \\[TID:(?<tid>.+?)] \\[(?<thread>.+?)] (?<level>\\w{4,}) (?<logger>.{1,36}) (?<msg>.+)/$\n" +
" }\n" +
" extractor {\n" +
" metrics {\n" +
" timestamp \"${log.timestamp}\"\n" +
" labels level: parsed.level, service: log.service, instance: log.serviceInstance\n" +
" name \"log_count\"\n" +
" value 1\n" +
" }\n" +
" }\n" +
" sink {\n" +
" }\n" +
"}\n",
},
new String[] {
"e2e",
"filter {\n" +
" json {\n" +
" }\n" +
" // only collect abnormal logs (http status code >= 300, or commonProperties?.responseFlags is not empty)\n" +
" if (parsed?.response?.responseCode as Integer < 400 && !parsed?.commonProperties?.responseFlags) {\n" +
" abort {}\n" +
" }\n" +
" extractor {\n" +
" if (parsed?.response?.responseCode) {\n" +
" tag 'status.code': parsed?.response?.responseCode as int\n" +
" }\n" +
" tag 'response.flag': (parsed?.commonProperties?.responseFlags as Map)?.keySet()\n" +
" }\n" +
" sink {\n" +
" sampler {\n" +
" if (parsed?.commonProperties?.responseFlags) {\n" +
" // use service:errorCode as sampler id so that each service:errorCode has its own sampler,\n" +
" // e.g. checkoutservice:[upstreamConnectionFailure], checkoutservice:[upstreamRetryLimitExceeded]\n" +
" rateLimit(\"${log.service}:${(parsed?.commonProperties?.responseFlags as Map)?.keySet()}\") {\n" +
" qps 100\n" +
" }\n" +
" } else {\n" +
" // use service:responseCode as sampler id so that each service:responseCode has its own sampler,\n" +
" // e.g. checkoutservice:500, checkoutservice:404.\n" +
" rateLimit(\"${log.service}:${parsed?.response?.responseCode}\") {\n" +
" qps 100\n" +
" }\n" +
" }\n" +
" }\n" +
" }\n" +
"}\n",
}
);
}
@Parameterized.Parameter()
public String name;
@Parameterized.Parameter(1)
public String script;
final ModuleManager manager = mock(ModuleManager.class);
@Before
public void setup() {
Whitebox.setInternalState(manager, "isInPrepareStage", false);
when(manager.find(anyString())).thenReturn(mock(ModuleProviderHolder.class));
when(manager.find(CoreModule.NAME).provider()).thenReturn(mock(ModuleServiceHolder.class));
when(manager.find(CoreModule.NAME).provider().getService(SourceReceiver.class))
.thenReturn(mock(SourceReceiver.class));
when(manager.find(CoreModule.NAME).provider().getService(ConfigService.class))
.thenReturn(mock(ConfigService.class));
when(manager.find(CoreModule.NAME)
.provider()
.getService(ConfigService.class)
.getSearchableLogsTags())
.thenReturn("");
}
@Test
public void testDslStaticCompile() throws ModuleStartException {
final DSL dsl = DSL.of(manager, new LogAnalyzerModuleConfig(), script);
Whitebox.setInternalState(
Whitebox.getInternalState(dsl, "filterSpec"), "factories", Collections.emptyList()
);
dsl.bind(new Binding().log(LogData.newBuilder().build()));
dsl.evaluate();
}
}

16
oap-server/analyzer/log-analyzer/src/test/resources/log-mal-rules/placeholder.yaml Normal file
View File

@ -0,0 +1,16 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Refer to examples in config-examples/log-mal.yaml

2
oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4
View File

@ -39,7 +39,7 @@ SRC_SERVICE_INSTANCE_JVM_MEMORY: 'ServiceInstanceJVMMemory';
SRC_SERVICE_INSTANCE_JVM_MEMORY_POOL: 'ServiceInstanceJVMMemoryPool';
SRC_SERVICE_INSTANCE_JVM_GC: 'ServiceInstanceJVMGC';
SRC_SERVICE_INSTANCE_JVM_THREAD: 'ServiceInstanceJVMThread';
SRC_SERVICE_INSTANCE_JVM_CLASS:'ServiceInstanceJVMClass';
SRC_SERVICE_INSTANCE_JVM_CLASS: 'ServiceInstanceJVMClass';
SRC_DATABASE_ACCESS: 'DatabaseAccess';
SRC_SERVICE_INSTANCE_CLR_CPU: 'ServiceInstanceCLRCPU';
SRC_SERVICE_INSTANCE_CLR_GC: 'ServiceInstanceCLRGC';

6
oap-server/server-bootstrap/src/main/resources/lal/envoy-als.yaml
View File

@ -20,21 +20,21 @@ rules:
json {
}
// only collect abnormal logs (http status code >= 300, or commonProperties?.responseFlags is not empty)
if (parsed?.response?.responseCode < 400 && !parsed?.commonProperties?.responseFlags) {
if (parsed?.response?.responseCode as Integer < 400 && !parsed?.commonProperties?.responseFlags) {
abort {}
}
extractor {
if (parsed?.response?.responseCode) {
tag 'status.code': parsed?.response?.responseCode as int
}
tag 'response.flag': parsed?.commonProperties?.responseFlags?.keySet()
tag 'response.flag': (parsed?.commonProperties?.responseFlags as Map)?.keySet()
}
sink {
sampler {
if (parsed?.commonProperties?.responseFlags) {
// use service:errorCode as sampler id so that each service:errorCode has its own sampler,
// e.g. checkoutservice:[upstreamConnectionFailure], checkoutservice:[upstreamRetryLimitExceeded]
rateLimit("${log.service}:${parsed?.commonProperties?.responseFlags?.keySet()}") {
rateLimit("${log.service}:${(parsed?.commonProperties?.responseFlags as Map)?.keySet()}") {
qps 100
}
} else {

2
oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/mock/MockModuleManager.java
View File

@ -37,10 +37,12 @@ public abstract class MockModuleManager extends ModuleManager {
moduleProviderHolderMap.put(name, provider);
}
@Override
public boolean has(String moduleName) {
return moduleProviderHolderMap.containsKey(moduleName);
}
@Override
public ModuleProviderHolder find(String moduleName) throws ModuleNotFoundRuntimeException {
if (!moduleProviderHolderMap.containsKey(moduleName)) {
throw new ModuleNotFoundRuntimeException("ModuleProviderHolder[" + moduleName + "] cannot found in MOCK.");

2
test/e2e/e2e-test/docker/log/lal.yaml
View File

@ -23,7 +23,7 @@ rules:
}
extractor {
metrics {
timestamp log.timestamp
timestamp log.timestamp as Long
labels level: parsed.level, service: log.service, instance: log.serviceInstance
name "log_count"
value 1

2
tools/dependencies/known-oap-backend-dependencies-es7.txt
View File

@ -47,7 +47,7 @@ flatbuffers-java-1.12.0.jar
freemarker-2.3.28.jar
graphql-java-8.0.jar
graphql-java-tools-5.2.3.jar
groovy-3.0.3.jar
groovy-3.0.8.jar
grpc-api-1.32.1.jar
grpc-context-1.32.1.jar
grpc-core-1.32.1.jar

2
tools/dependencies/known-oap-backend-dependencies.txt
View File

@ -45,7 +45,7 @@ flatbuffers-java-1.12.0.jar
freemarker-2.3.28.jar
graphql-java-8.0.jar
graphql-java-tools-5.2.3.jar
groovy-3.0.3.jar
groovy-3.0.8.jar
grpc-api-1.32.1.jar
grpc-context-1.32.1.jar
grpc-core-1.32.1.jar