diff --git a/Jenkinsfile-E2E b/Jenkinsfile-E2E index 2845776af..2d1967412 100644 --- a/Jenkinsfile-E2E +++ b/Jenkinsfile-E2E @@ -65,6 +65,13 @@ pipeline { sh 'docker ps' sh 'docker ps | grep -e "skywalking-e2e-container-${BUILD_ID}" | awk \'{print $1}\' | xargs --no-run-if-empty docker stop' sh 'docker ps | grep -e "skywalking-e2e-container-${BUILD_ID}" | awk \'{print $1}\' | xargs --no-run-if-empty docker rm' + + // Files created by docker container in the directories that are mounted from the host to + // the container can not be deleted by `deleteDir()`, we need to mount it again and delete them + // inside the container + // + // Delete `/dist` folder + sh 'docker run -v $(pwd)/dist:/sw alpine sleep 10 && rm -rf /sw/*' deleteDir() } } diff --git a/apm-dist/release-docs/LICENSE b/apm-dist/release-docs/LICENSE index 891d8054a..bddc04a56 100644 --- a/apm-dist/release-docs/LICENSE +++ b/apm-dist/release-docs/LICENSE @@ -271,6 +271,7 @@ The text of each license is the standard Apache 2.0 license. Apache: commons-io 2.4: https://github.com/apache/commons-io, Apache 2.0 Apache: commons-compress 1.18: https://github.com/apache/commons-compress, Apache 2.0 Apache: tomcat 8.5.27: https://github.com/apache/tomcat/tree/trunk, Apache 2.0 + Apache: freemaker 2.3.28: https://github.com/apache/freemarker, Apache 2.0 netty 5.5.0: https://github.com/netty/netty/blob/4.1/LICENSE.txt, Apache 2.0 annotations 13.0: http://www.jetbrains.org, Apache 2.0 compiler 0.9.3: https://github.com/spullara/mustache.java, Apache 2.0 @@ -323,6 +324,7 @@ The text of each license is the standard Apache 2.0 license. sundrio 0.9.2: https://github.com/sundrio/sundrio Apache 2.0 Ctripcorp: apollo 1.4.0: https://github.com/ctripcorp/apollo Apache 2.0 etcd4j 2.17.0: https://github.com/jurmous/etcd4j Apache 2.0 + javaassist 3.25.0-GA: https://github.com/jboss-javassist/javassist Apache 2.0 ======================================================================== MIT licenses diff --git a/apm-dist/src/main/assembly/binary.xml b/apm-dist/src/main/assembly/binary.xml index 45301c9c4..8e0cd0ff1 100644 --- a/apm-dist/src/main/assembly/binary.xml +++ b/apm-dist/src/main/assembly/binary.xml @@ -42,6 +42,7 @@ *.yml *.xml *.properties + *.oal diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/boot/AgentPackagePath.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/boot/AgentPackagePath.java index a7574ef5a..f391a2ece 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/boot/AgentPackagePath.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/boot/AgentPackagePath.java @@ -72,7 +72,8 @@ public class AgentPackagePath { return agentJarFile.getParentFile(); } } else { - String classLocation = urlString.substring(urlString.indexOf("file:"), urlString.length() - classResourcePath.length()); + int prefixLength = "file:".length(); + String classLocation = urlString.substring(prefixLength, urlString.length() - classResourcePath.length()); return new File(classLocation); } } diff --git a/docs/en/concepts-and-designs/oal.md b/docs/en/concepts-and-designs/oal.md index 5a89269d1..1721e263d 100644 --- a/docs/en/concepts-and-designs/oal.md +++ b/docs/en/concepts-and-designs/oal.md @@ -4,8 +4,12 @@ Provide OAL(Observability Analysis Language) to analysis incoming data in stream OAL focuses on metrics in Service, Service Instance and Endpoint. Because of that, the language is easy to learn and use. -Considering performance, reading and debugging, OAL is defined as a compile language. -The OAL scripts will be compiled to normal Java codes in package stage. + +Since 6.3, the OAL engine is embedded in OAP server runtime, as `oal-rt`(OAL Runtime). +OAL scripts now locate in `/config` folder, user could simply change and reboot the server to make it effective. +But still, OAL script is compile language, OAL Runtime generates java codes dynamically. + +You could open set `SW_OAL_ENGINE_DEBUG=Y` at system env, to see which classes generated. ## Grammar Scripts should be named as `*.oal` diff --git a/docs/en/guides/How-to-build.md b/docs/en/guides/How-to-build.md index 5106d30ef..2e1aca856 100644 --- a/docs/en/guides/How-to-build.md +++ b/docs/en/guides/How-to-build.md @@ -76,7 +76,6 @@ a oap image with name `bar/oap:foo`, run the following command * `grpc-java` and `java` folders in **oap-server/exporter/target/generated-sources/protobuf** * `grpc-java` and `java` folders in **oap-server/server-configuration/grpc-configuration-sync/target/generated-sources/protobuf** * `antlr4` folder in **oap-server/generate-tool-grammar/target/generated-sources** - * `oal` folder in **oap-server/generated-analysis/target/generated-sources** ## Setup your Eclipse IDE **NOTICE**: If you clone the codes from GitHub, please make sure that you had finished step 1 to 7 in section **[Build from GitHub](#build-from-github)**, if you download the source codes from the official website of SkyWalking, please make sure that you had followed the steps in section **[Build from Apache source code release](#build-from-apache-source-code-release)**. diff --git a/docs/en/guides/README.md b/docs/en/guides/README.md index a6140dfab..788c07e54 100755 --- a/docs/en/guides/README.md +++ b/docs/en/guides/README.md @@ -96,7 +96,8 @@ and private plugin developer should read this. - If you want to build a new probe or plugin in any language, please read [Component library definition and extension](Component-library-settings.md) document. - [Storage extension development guide](storage-extention.md). Help potential contributors to build a new storage implementor besides the official. -- [Customize analysis by oal script](write-oal.md). Guide you to use oal script to make your own metrics available. +- Customize analysis by oal script. OAL scripts locate in `config/official_analysis.oal`. You could change it and reboot the OAP server. Read +[Observability Analysis Language Introduction](../concepts-and-designs/oal.md) if you need to learn about OAL script. - [Source and scope extension for new metrics](source-extension.md). If you want to analysis a new metrics, which SkyWalking haven't provide. You need to add a new receiver rather than choosing [existed receiver](../setup/backend/backend-receivers.md). diff --git a/docs/en/guides/write-oal.md b/docs/en/guides/write-oal.md deleted file mode 100644 index 7d6eb2a7d..000000000 --- a/docs/en/guides/write-oal.md +++ /dev/null @@ -1,15 +0,0 @@ -# Write OAL script -Read [OAL overview](../concepts-and-designs/oal.md) to understand the oal script. - -## Find oal script in source codes -The official official_analysis.oal is [here](../../../oap-server/generated-analysis/src/main/resources/official_analysis.oal). - -## Generate tool -The `oap-server/generate-tool` module includes the source codes of compiling tool. This tool is already integrated -maven compile phase. So, unless you want to change the tool source codes, you don't need to set anything. - -Run `./mvnw compile` or `./mvnw package`, the generated codes of the oal script are in `oap-server/generate-tool/target/generated-sources/oal/*`. - -## Write and recompile -You could change the `official_analysis.oal` script, then recompile codes. -The generated codes are in **oap-server/generated-analysis/target/generated-sources/oal**. \ No newline at end of file diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/Main.java b/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/Main.java deleted file mode 100644 index 04a48c96c..000000000 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/Main.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oal.tool; - -import freemarker.template.TemplateException; -import java.io.*; -import org.apache.skywalking.apm.util.StringUtil; -import org.apache.skywalking.oal.tool.meta.*; -import org.apache.skywalking.oal.tool.output.FileGenerator; -import org.apache.skywalking.oal.tool.parser.*; -import org.apache.skywalking.oap.server.core.annotation.AnnotationScan; -import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; - -public class Main { - - public static void main(String[] args) throws IOException, TemplateException { - AnnotationScan scopeScan = new AnnotationScan(); - scopeScan.registerListener(new DefaultScopeDefine.Listener()); - scopeScan.scan(null); - - String modulePath = args[0]; - - String scriptFilePath = StringUtil.join(File.separatorChar, modulePath, "src", "main", "resources", "official_analysis.oal"); - String outputPath = StringUtil.join(File.separatorChar, modulePath, "..", "generated-analysis", "target", "generated-sources", "oal", - "org", "apache", "skywalking", "oap", "server", "core", "analysis"); - String metaFilePath = StringUtil.join(File.separatorChar, modulePath, "src", "main", "resources", "generator-scope-meta.yml"); - - MetricsHolder.init(); - - File scriptFile = new File(scriptFilePath); - if (!scriptFile.exists()) { - throw new IllegalArgumentException("OAL script file [" + scriptFilePath + "] doesn't exist"); - } - - File metaFile = new File(metaFilePath); - if (!metaFile.exists()) { - throw new IllegalArgumentException("Generator meta file [" + metaFilePath + "] doesn't exist"); - } - - MetaReader reader = new MetaReader(); - MetaSettings metaSettings = reader.read(new FileInputStream(metaFile)); - SourceColumnsFactory.setSettings(metaSettings); - - ScriptParser scriptParser = ScriptParser.createFromFile(scriptFilePath); - OALScripts oalScripts = scriptParser.parse(); - - FileGenerator generator = new FileGenerator(oalScripts, outputPath); - generator.generate(); - } -} diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/output/FileGenerator.java b/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/output/FileGenerator.java deleted file mode 100644 index deab81b58..000000000 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/output/FileGenerator.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oal.tool.output; - -import freemarker.template.*; -import java.io.*; -import java.util.*; -import org.apache.skywalking.oal.tool.parser.*; - -public class FileGenerator { - private List results; - private DisableCollection collection; - private String outputPath; - private Configuration configuration; - private AllDispatcherContext allDispatcherContext; - - public FileGenerator(OALScripts oalScripts, String outputPath) { - this.results = oalScripts.getMetricsStmts(); - this.collection = oalScripts.getDisableCollection(); - this.outputPath = outputPath; - configuration = new Configuration(new Version("2.3.28")); - configuration.setEncoding(Locale.ENGLISH, "UTF-8"); - configuration.setClassLoaderForTemplateLoading(FileGenerator.class.getClassLoader(), "/code-templates"); - allDispatcherContext = new AllDispatcherContext(); - buildDispatcherContext(); - } - - public void generate() throws IOException, TemplateException { - for (AnalysisResult result : results) { - generate(result, "Metrics.java", writer -> generateMetricsImplementor(result, writer)); - - String scopeName = result.getSourceName(); - File file = new File(outputPath, "generated/" + scopeName.toLowerCase() + "/" + scopeName + "Dispatcher.java"); - createFile(file); - generateDispatcher(result, new FileWriter(file)); - } - generateDisable(); - } - - private void generate(AnalysisResult result, String fileSuffix, - WriteWrapper writeWrapper) throws IOException, TemplateException { - File file = new File(outputPath, buildSubFolderName(result, fileSuffix)); - createFile(file); - FileWriter fileWriter = new FileWriter(file); - try { - writeWrapper.execute(fileWriter); - } finally { - fileWriter.close(); - } - } - - private void createFile(File file) throws IOException { - if (!file.exists()) { - if (!file.getParentFile().exists()) { - file.getParentFile().mkdirs(); - } - file.createNewFile(); - } - } - - private String buildSubFolderName(AnalysisResult result, String suffix) { - return "generated/" - + result.getSourceName().toLowerCase() + "/" - + result.getMetricsName() + suffix; - } - - void generateMetricsImplementor(AnalysisResult result, Writer output) throws IOException, TemplateException { - configuration.getTemplate("MetricsImplementor.ftl").process(result, output); - } - - void generateDispatcher(AnalysisResult result, Writer output) throws IOException, TemplateException { - String scopeName = result.getSourceName(); - DispatcherContext context = allDispatcherContext.getAllContext().get(scopeName); - if (context != null) { - configuration.getTemplate("DispatcherTemplate.ftl").process(context, output); - } - } - - private void buildDispatcherContext() { - for (AnalysisResult result : results) { - String sourceName = result.getSourceName(); - - DispatcherContext context = allDispatcherContext.getAllContext().get(sourceName); - if (context == null) { - context = new DispatcherContext(); - context.setSource(sourceName); - context.setPackageName(sourceName.toLowerCase()); - allDispatcherContext.getAllContext().put(sourceName, context); - } - context.getMetrics().add(result); - } - } - - private void generateDisable() throws IOException, TemplateException { - File file = new File(outputPath, "generated/DisableSourceDefinition.java"); - createFile(file); - configuration.getTemplate("DisableSourceDefinition.ftl").process(collection, new FileWriter(file)); - } -} diff --git a/oap-server/generate-tool/src/main/resources/code-templates/DisableSourceDefinition.ftl b/oap-server/generate-tool/src/main/resources/code-templates/DisableSourceDefinition.ftl deleted file mode 100644 index b1c9be5b9..000000000 --- a/oap-server/generate-tool/src/main/resources/code-templates/DisableSourceDefinition.ftl +++ /dev/null @@ -1,33 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -*/ - -package org.apache.skywalking.oap.server.core.analysis.generated; -<#if (allDisableSources?size>0)> -import org.apache.skywalking.oap.server.core.analysis.Disable; - - -/** -* This class is auto generated. Please don't change this class manually. -* -* @author Observability Analysis Language code generator -*/ -<#list allDisableSources as disableSource> -@Disable("${disableSource}") - -public class DisableSourceDefinition { -} \ No newline at end of file diff --git a/oap-server/generate-tool/src/main/resources/code-templates/DispatcherTemplate.ftl b/oap-server/generate-tool/src/main/resources/code-templates/DispatcherTemplate.ftl deleted file mode 100644 index ea6b319a6..000000000 --- a/oap-server/generate-tool/src/main/resources/code-templates/DispatcherTemplate.ftl +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oap.server.core.analysis.generated.${packageName}; - -import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher; -<#if (metrics?size>0)> -import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor; - <#list metrics as metrics> - <#if metrics.filterExpressions??> -import org.apache.skywalking.oap.server.core.analysis.metrics.expression.*; - <#break> - - - -import org.apache.skywalking.oap.server.core.source.*; - -/** - * This class is auto generated. Please don't change this class manually. - * - * @author Observability Analysis Language code generator - */ -public class ${source}Dispatcher implements SourceDispatcher<${source}> { - - @Override public void dispatch(${source} source) { -<#list metrics as metrics> - do${metrics.metricsName}(source); - - } - -<#list metrics as metrics> - private void do${metrics.metricsName}(${source} source) { - ${metrics.metricsName}Metrics metrics = new ${metrics.metricsName}Metrics(); - - <#if metrics.filterExpressions??> - <#list metrics.filterExpressions as filterExpression> - <#if filterExpression.expressionObject == "GreaterMatch" || filterExpression.expressionObject == "LessMatch" || filterExpression.expressionObject == "GreaterEqualMatch" || filterExpression.expressionObject == "LessEqualMatch"> - if (!new ${filterExpression.expressionObject}().match(${filterExpression.left}, ${filterExpression.right})) { - return; - } - <#else> - if (!new ${filterExpression.expressionObject}().setLeft(${filterExpression.left}).setRight(${filterExpression.right}).match()) { - return; - } - - - - - metrics.setTimeBucket(source.getTimeBucket()); - <#list metrics.fieldsFromSource as field> - metrics.${field.fieldSetter}(source.${field.fieldGetter}()); - - metrics.${metrics.entryMethod.methodName}(<#list metrics.entryMethod.argsExpressions as arg>${arg}<#if arg_has_next>, ); - - MetricsStreamProcessor.getInstance().in(metrics); - } - -} diff --git a/oap-server/generate-tool/src/main/resources/code-templates/MetricsImplementor.ftl b/oap-server/generate-tool/src/main/resources/code-templates/MetricsImplementor.ftl deleted file mode 100644 index bd945282d..000000000 --- a/oap-server/generate-tool/src/main/resources/code-templates/MetricsImplementor.ftl +++ /dev/null @@ -1,293 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oap.server.core.analysis.generated.${packageName}; - -import java.util.*; -<#if (fieldsFromSource?size>0) > -import lombok.*; - -<#list fieldsFromSource as sourceField> - <#if sourceField.isID()> -import org.apache.skywalking.oap.server.core.Const; - <#break> - - -import org.apache.skywalking.oap.server.core.analysis.Stream; -import org.apache.skywalking.oap.server.core.analysis.metrics.*; -import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor; -import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData; -import org.apache.skywalking.oap.server.core.storage.StorageBuilder; -import org.apache.skywalking.oap.server.core.storage.annotation.*; - -/** - * This class is auto generated. Please don't change this class manually. - * - * @author Observability Analysis Language code generator - */ -@Stream(name = "${tableName}", scopeId = ${sourceScopeId}, builder = ${metricsName}Metrics.Builder.class, processor = MetricsStreamProcessor.class) -public class ${metricsName}Metrics extends ${metricsClassName} implements WithMetadata { - -<#list fieldsFromSource as sourceField> - @Setter @Getter @Column(columnName = "${sourceField.columnName}") <#if sourceField.isID()>@IDColumn private ${sourceField.typeName} ${sourceField.fieldName}; - - - @Override public String id() { - String splitJointId = String.valueOf(getTimeBucket()); -<#list fieldsFromSource as sourceField> - <#if sourceField.isID()> - <#if sourceField.getTypeName() == "java.lang.String"> - splitJointId += Const.ID_SPLIT + ${sourceField.fieldName}; - <#else> - splitJointId += Const.ID_SPLIT + String.valueOf(${sourceField.fieldName}); - - - - return splitJointId; - } - - @Override public int hashCode() { - int result = 17; -<#list fieldsFromSource as sourceField> - <#if sourceField.isID()> - <#if sourceField.getTypeName() == "java.lang.String"> - result = 31 * result + ${sourceField.fieldName}.hashCode(); - <#else> - result += Const.ID_SPLIT + ${sourceField.fieldName}; - - - - result = 31 * result + (int)getTimeBucket(); - return result; - } - - @Override public int remoteHashCode() { - int result = 17; -<#list fieldsFromSource as sourceField> - <#if sourceField.isID()> - <#if sourceField.getTypeName() == "java.lang.String"> - result = 31 * result + ${sourceField.fieldName}.hashCode(); - <#else> - result += Const.ID_SPLIT + ${sourceField.fieldName}; - - - - return result; - } - - @Override public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - - ${metricsName}Metrics metrics = (${metricsName}Metrics)obj; -<#list fieldsFromSource as sourceField> - <#if sourceField.isID()> - <#if sourceField.getTypeName() == "java.lang.String"> - if (!${sourceField.fieldName}.equals(metrics.${sourceField.fieldName})) - <#else> - if (${sourceField.fieldName} != metrics.${sourceField.fieldName}) - - return false; - - - - if (getTimeBucket() != metrics.getTimeBucket()) - return false; - - return true; - } - - @Override public RemoteData.Builder serialize() { - RemoteData.Builder remoteBuilder = RemoteData.newBuilder(); -<#list serializeFields.stringFields as field> - remoteBuilder.addDataStrings(${field.getter}()); - - -<#list serializeFields.longFields as field> - remoteBuilder.addDataLongs(${field.getter}()); - - -<#list serializeFields.doubleFields as field> - remoteBuilder.addDataDoubles(${field.getter}()); - - -<#list serializeFields.intFields as field> - remoteBuilder.addDataIntegers(${field.getter}()); - -<#list serializeFields.intLongValuePairListFields as field> - ${field.getter}().forEach(element -> remoteBuilder.addDataIntLongPairList(element.serialize())); - - - return remoteBuilder; - } - - @Override public void deserialize(RemoteData remoteData) { -<#list serializeFields.stringFields as field> - ${field.setter}(remoteData.getDataStrings(${field?index})); - - -<#list serializeFields.longFields as field> - ${field.setter}(remoteData.getDataLongs(${field?index})); - - -<#list serializeFields.doubleFields as field> - ${field.setter}(remoteData.getDataDoubles(${field?index})); - - -<#list serializeFields.intFields as field> - ${field.setter}(remoteData.getDataIntegers(${field?index})); - - -<#list serializeFields.intLongValuePairListFields as field> - setDetailGroup(new IntKeyLongValueArray(30)); - remoteData.getDataIntLongPairListList().forEach(element -> { - getDetailGroup().add(new IntKeyLongValue(element.getKey(), element.getValue())); - }); - - - } - - @Override public MetricsMetaInfo getMeta() { - return new MetricsMetaInfo("${varName}", ${sourceScopeId}<#if (fieldsFromSource?size>0) ><#list fieldsFromSource as field><#if field.isID()>, ${field.fieldName}); - } - - @Override - public Metrics toHour() { - ${metricsName}Metrics metrics = new ${metricsName}Metrics(); -<#list fieldsFromSource as field> - <#if field.columnName == "time_bucket"> - metrics.setTimeBucket(toTimeBucketInHour()); - <#elseif field.typeName == "java.lang.String" || field.typeName == "long" || field.typeName == "int" || field.typeName == "double" || field.typeName == "float"> - metrics.${field.fieldSetter}(this.${field.fieldGetter}()); - <#else> - ${field.typeName} newValue = new ${field.typeName}(); - newValue.copyFrom(this.${field.fieldGetter}()); - metrics.${field.fieldSetter}(newValue); - - -<#list persistentFields as field> - <#if field.columnName == "time_bucket"> - metrics.setTimeBucket(toTimeBucketInHour()); - <#elseif field.typeName == "java.lang.String" || field.typeName == "long" || field.typeName == "int" || field.typeName == "double" || field.typeName == "float"> - metrics.${field.fieldSetter}(this.${field.fieldGetter}()); - <#else> - ${field.typeName} newValue = new ${field.typeName}(); - newValue.copyFrom(this.${field.fieldGetter}()); - metrics.${field.fieldSetter}(newValue); - - - return metrics; - } - - @Override - public Metrics toDay() { - ${metricsName}Metrics metrics = new ${metricsName}Metrics(); -<#list fieldsFromSource as field> - <#if field.columnName == "time_bucket"> - metrics.setTimeBucket(toTimeBucketInDay()); - <#elseif field.typeName == "java.lang.String" || field.typeName == "long" || field.typeName == "int" || field.typeName == "double" || field.typeName == "float"> - metrics.${field.fieldSetter}(this.${field.fieldGetter}()); - <#else> - ${field.typeName} newValue = new ${field.typeName}(); - newValue.copyFrom(this.${field.fieldGetter}()); - metrics.${field.fieldSetter}(newValue); - - -<#list persistentFields as field> - <#if field.columnName == "time_bucket"> - metrics.setTimeBucket(toTimeBucketInDay()); - <#elseif field.typeName == "java.lang.String" || field.typeName == "long" || field.typeName == "int" || field.typeName == "double" || field.typeName == "float"> - metrics.${field.fieldSetter}(this.${field.fieldGetter}()); - <#else> - ${field.typeName} newValue = new ${field.typeName}(); - newValue.copyFrom(this.${field.fieldGetter}()); - metrics.${field.fieldSetter}(newValue); - - - return metrics; - } - - @Override - public Metrics toMonth() { - ${metricsName}Metrics metrics = new ${metricsName}Metrics(); -<#list fieldsFromSource as field> - <#if field.columnName == "time_bucket"> - metrics.setTimeBucket(toTimeBucketInMonth()); - <#elseif field.typeName == "java.lang.String" || field.typeName == "long" || field.typeName == "int" || field.typeName == "double" || field.typeName == "float"> - metrics.${field.fieldSetter}(this.${field.fieldGetter}()); - <#else> - ${field.typeName} newValue = new ${field.typeName}(); - newValue.copyFrom(this.${field.fieldGetter}()); - metrics.${field.fieldSetter}(newValue); - - -<#list persistentFields as field> - <#if field.columnName == "time_bucket"> - metrics.setTimeBucket(toTimeBucketInMonth()); - <#elseif field.typeName == "java.lang.String" || field.typeName == "long" || field.typeName == "int" || field.typeName == "double" || field.typeName == "float"> - metrics.${field.fieldSetter}(this.${field.fieldGetter}()); - <#else> - ${field.typeName} newValue = new ${field.typeName}(); - newValue.copyFrom(this.${field.fieldGetter}()); - metrics.${field.fieldSetter}(newValue); - - - return metrics; - } - - public static class Builder implements StorageBuilder<${metricsName}Metrics> { - - @Override public Map data2Map(${metricsName}Metrics storageData) { - Map map = new HashMap<>(); - <#list fieldsFromSource as field> - map.put("${field.columnName}", storageData.${field.fieldGetter}()); - - <#list persistentFields as field> - map.put("${field.columnName}", storageData.${field.fieldGetter}()); - - return map; - } - - @Override public ${metricsName}Metrics map2Data(Map dbMap) { - ${metricsName}Metrics metrics = new ${metricsName}Metrics(); - <#list fieldsFromSource as field> - <#if field.typeName == "long" || field.typeName == "int" || field.typeName == "double" || field.typeName == "float"> - metrics.${field.fieldSetter}(((Number)dbMap.get("${field.columnName}")).${field.typeName}Value()); - <#elseif field.typeName == "java.lang.String"> - metrics.${field.fieldSetter}((String)dbMap.get("${field.columnName}")); - <#else> - metrics.${field.fieldSetter}(new ${field.typeName}((String)dbMap.get("${field.columnName}"))); - - - <#list persistentFields as field> - <#if field.typeName == "long" || field.typeName == "int" || field.typeName == "double" || field.typeName == "float"> - metrics.${field.fieldSetter}(((Number)dbMap.get("${field.columnName}")).${field.typeName}Value()); - <#elseif field.typeName == "java.lang.String"> - metrics.${field.fieldSetter}((String)dbMap.get("${field.columnName}")); - <#else> - metrics.${field.fieldSetter}(new ${field.typeName}((String)dbMap.get("${field.columnName}"))); - - - return metrics; - } - } -} diff --git a/oap-server/generate-tool/src/test/java/org/apache/skywalking/oal/tool/output/FileGeneratorTest.java b/oap-server/generate-tool/src/test/java/org/apache/skywalking/oal/tool/output/FileGeneratorTest.java deleted file mode 100644 index 88ce38c63..000000000 --- a/oap-server/generate-tool/src/test/java/org/apache/skywalking/oal/tool/output/FileGeneratorTest.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oal.tool.output; - -import freemarker.template.TemplateException; -import java.io.*; -import java.util.LinkedList; -import org.apache.skywalking.oal.tool.meta.*; -import org.apache.skywalking.oal.tool.parser.*; -import org.junit.*; - -public class FileGeneratorTest { - @BeforeClass - public static void init() { - MetaReader reader = new MetaReader(); - InputStream stream = MetaReaderTest.class.getResourceAsStream("/scope-meta.yml"); - MetaSettings metaSettings = reader.read(stream); - SourceColumnsFactory.setSettings(metaSettings); - } - - private AnalysisResult buildResult() { - AnalysisResult result = new AnalysisResult(); - result.setVarName("generate_metrics"); - result.setSourceName("Service"); - result.setSourceScopeId(1); - result.setPackageName("service.serviceavg"); - result.setTableName("service_avg"); - result.setSourceAttribute("latency"); - result.setMetricsName("ServiceAvg"); - result.setAggregationFunctionName("avg"); - result.setMetricsClassName("LongAvgMetrics"); - - FilterExpression equalExpression = new FilterExpression(); - equalExpression.setExpressionObject("EqualMatch"); - equalExpression.setLeft("source.getName()"); - equalExpression.setRight("\"/service/prod/save\""); - result.addFilterExpressions(equalExpression); - - FilterExpression greaterExpression = new FilterExpression(); - greaterExpression.setExpressionObject("GreaterMatch"); - greaterExpression.setLeft("source.getLatency()"); - greaterExpression.setRight("1000"); - result.addFilterExpressions(greaterExpression); - - EntryMethod method = new EntryMethod(); - method.setMethodName("combine"); - method.setArgsExpressions(new LinkedList<>()); - method.getArgsExpressions().add("source.getLatency()"); - method.getArgsExpressions().add("1"); - result.setEntryMethod(method); - result.addPersistentField("summation", "summation", long.class); - result.addPersistentField("count", "count", int.class); - result.addPersistentField("value", "value", long.class); - result.addPersistentField("timeBucket", "time_bucket", long.class); - result.addPersistentField("stringField", "string_field", String.class); - result.setFieldsFromSource(SourceColumnsFactory.getColumns("Service")); - result.generateSerializeFields(); - - return result; - } - - @Test - public void testGenerateMetricsImplementor() throws IOException, TemplateException { - AnalysisResult result = buildResult(); - - OALScripts oalScripts = new OALScripts(); - oalScripts.getMetricsStmts().add(result); - - FileGenerator fileGenerator = new FileGenerator(oalScripts, "."); - StringWriter writer = new StringWriter(); - fileGenerator.generateMetricsImplementor(result, writer); - Assert.assertEquals(readExpectedFile("MetricsImplementorExpected.java"), writer.toString()); - -// fileGenerator.generateMetricsImplementor(result, new OutputStreamWriter(System.out)); - } - - @Test - public void testServiceDispatcher() throws IOException, TemplateException { - AnalysisResult result = buildResult(); - - OALScripts oalScripts = new OALScripts(); - oalScripts.getMetricsStmts().add(result); - - FileGenerator fileGenerator = new FileGenerator(oalScripts, "."); - StringWriter writer = new StringWriter(); - fileGenerator.generateDispatcher(result, writer); - Assert.assertEquals(readExpectedFile("ServiceDispatcherExpected.java"), writer.toString()); - -// fileGenerator.generateDispatcher(result, new OutputStreamWriter(System.out)); - } - - private String readExpectedFile(String filename) throws IOException { - BufferedReader reader = new BufferedReader(new InputStreamReader(FileGenerator.class.getResourceAsStream("/expectedFiles/" + filename))); - - StringBuilder fileContent = new StringBuilder(); - String sCurrentLine; - - while ((sCurrentLine = reader.readLine()) != null) { - fileContent.append(sCurrentLine).append("\n"); - } - - return fileContent.toString(); - } -} diff --git a/oap-server/generated-analysis/pom.xml b/oap-server/generated-analysis/pom.xml deleted file mode 100644 index 63265e363..000000000 --- a/oap-server/generated-analysis/pom.xml +++ /dev/null @@ -1,70 +0,0 @@ - - - - - - oap-server - org.apache.skywalking - 6.3.0-SNAPSHOT - - 4.0.0 - - generated-analysis - - - - org.apache.skywalking - server-core - ${project.version} - - - - - org.apache.skywalking - generate-tool - ${project.version} - - - - - - target/generated-sources/oal - - - org.codehaus.mojo - exec-maven-plugin - ${exec-maven-plugin.version} - - - generate-sources - - java - - - org.apache.skywalking.oal.tool.Main - - ${project.basedir} - - - - - - - - \ No newline at end of file diff --git a/oap-server/generate-tool-grammar/pom.xml b/oap-server/oal-grammar/pom.xml similarity index 95% rename from oap-server/generate-tool-grammar/pom.xml rename to oap-server/oal-grammar/pom.xml index caf196d5e..3c7408f05 100644 --- a/oap-server/generate-tool-grammar/pom.xml +++ b/oap-server/oal-grammar/pom.xml @@ -25,13 +25,12 @@ 4.0.0 - generate-tool-grammar + oal-grammar org.antlr antlr4 - 4.7.1 diff --git a/oap-server/generate-tool-grammar/src/main/antlr4/org/apache/skywalking/oal/tool/grammar/OALLexer.g4 b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 similarity index 97% rename from oap-server/generate-tool-grammar/src/main/antlr4/org/apache/skywalking/oal/tool/grammar/OALLexer.g4 rename to oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 index 8ae21e244..7ab88fc6a 100644 --- a/oap-server/generate-tool-grammar/src/main/antlr4/org/apache/skywalking/oal/tool/grammar/OALLexer.g4 +++ b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 @@ -20,7 +20,7 @@ // Observability Analysis Language lexer lexer grammar OALLexer; -@Header {package org.apache.skywalking.oal.tool.grammar;} +@Header {package org.apache.skywalking.oal.rt.grammar;} // Keywords diff --git a/oap-server/generate-tool-grammar/src/main/antlr4/org/apache/skywalking/oal/tool/grammar/OALParser.g4 b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 similarity index 98% rename from oap-server/generate-tool-grammar/src/main/antlr4/org/apache/skywalking/oal/tool/grammar/OALParser.g4 rename to oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 index 79127e43d..a7103f318 100644 --- a/oap-server/generate-tool-grammar/src/main/antlr4/org/apache/skywalking/oal/tool/grammar/OALParser.g4 +++ b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 @@ -18,7 +18,7 @@ parser grammar OALParser; -@Header {package org.apache.skywalking.oal.tool.grammar;} +@Header {package org.apache.skywalking.oal.rt.grammar;} options { tokenVocab=OALLexer; } diff --git a/oap-server/generate-tool/pom.xml b/oap-server/oal-rt/pom.xml similarity index 76% rename from oap-server/generate-tool/pom.xml rename to oap-server/oal-rt/pom.xml index 46251c3aa..2ce0efca9 100644 --- a/oap-server/generate-tool/pom.xml +++ b/oap-server/oal-rt/pom.xml @@ -17,7 +17,8 @@ ~ --> - + oap-server org.apache.skywalking @@ -25,7 +26,7 @@ 4.0.0 - generate-tool + oal-rt @@ -35,28 +36,24 @@ org.apache.skywalking - generate-tool-grammar + oal-grammar ${project.version} org.antlr antlr4 - 4.7.1 - - - commons-cli - commons-cli - 1.4 - - - org.projectlombok - lombok - 1.18.0 org.freemarker freemarker - 2.3.28 + + + org.javassist + javassist + + + commons-io + commons-io \ No newline at end of file diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java new file mode 100644 index 000000000..7c983fcd6 --- /dev/null +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java @@ -0,0 +1,513 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oal.rt; + +import freemarker.template.Configuration; +import freemarker.template.Version; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.Reader; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import javassist.CannotCompileException; +import javassist.ClassPool; +import javassist.CtClass; +import javassist.CtConstructor; +import javassist.CtField; +import javassist.CtNewConstructor; +import javassist.CtNewMethod; +import javassist.NotFoundException; +import javassist.bytecode.AnnotationsAttribute; +import javassist.bytecode.ClassFile; +import javassist.bytecode.ConstPool; +import javassist.bytecode.SignatureAttribute; +import javassist.bytecode.annotation.Annotation; +import javassist.bytecode.annotation.ClassMemberValue; +import javassist.bytecode.annotation.IntegerMemberValue; +import javassist.bytecode.annotation.StringMemberValue; +import org.apache.commons.io.FileUtils; +import org.apache.skywalking.apm.util.StringUtil; +import org.apache.skywalking.oal.rt.meta.MetaReader; +import org.apache.skywalking.oal.rt.meta.MetaSettings; +import org.apache.skywalking.oal.rt.output.AllDispatcherContext; +import org.apache.skywalking.oal.rt.output.DispatcherContext; +import org.apache.skywalking.oal.rt.parser.AnalysisResult; +import org.apache.skywalking.oal.rt.parser.MetricsHolder; +import org.apache.skywalking.oal.rt.parser.OALScripts; +import org.apache.skywalking.oal.rt.parser.ScriptParser; +import org.apache.skywalking.oal.rt.parser.SourceColumn; +import org.apache.skywalking.oal.rt.parser.SourceColumnsFactory; +import org.apache.skywalking.oap.server.core.WorkPath; +import org.apache.skywalking.oap.server.core.analysis.DisableRegister; +import org.apache.skywalking.oap.server.core.analysis.DispatcherDetectorListener; +import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher; +import org.apache.skywalking.oap.server.core.analysis.Stream; +import org.apache.skywalking.oap.server.core.analysis.StreamAnnotationListener; +import org.apache.skywalking.oap.server.core.oal.rt.OALCompileException; +import org.apache.skywalking.oap.server.core.oal.rt.OALEngine; +import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import org.apache.skywalking.oap.server.core.storage.annotation.IDColumn; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.apache.skywalking.oap.server.library.util.ResourceUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * OAL Runtime is the class generation engine, which load the generated classes from OAL scrip definitions. This runtime + * is loaded dynamically. + * + * @author wusheng + */ +public class OALRuntime implements OALEngine { + private static final Logger logger = LoggerFactory.getLogger(OALRuntime.class); + + private static final String CLASS_FILE_CHARSET = "UTF-8"; + private static final String METRICS_FUNCTION_PACKAGE = "org.apache.skywalking.oap.server.core.analysis.metrics."; + private static final String DYNAMIC_METRICS_CLASS_PACKAGE = "org.apache.skywalking.oal.rt.metrics."; + private static final String DYNAMIC_METRICS_BUILDER_CLASS_PACKAGE = "org.apache.skywalking.oal.rt.metrics.builder."; + private static final String DYNAMIC_DISPATCHER_CLASS_PACKAGE = "org.apache.skywalking.oal.rt.dispatcher."; + private static final String WITH_METADATA_INTERFACE = "org.apache.skywalking.oap.server.core.analysis.metrics.WithMetadata"; + private static final String STORAGE_BUILDER_INTERFACE = "org.apache.skywalking.oap.server.core.storage.StorageBuilder"; + private static final String DISPATCHER_INTERFACE = "org.apache.skywalking.oap.server.core.analysis.SourceDispatcher"; + private static final String SOURCE_PACKAGE = "org.apache.skywalking.oap.server.core.source."; + private static final String METRICS_STREAM_PROCESSOR = "org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor"; + private static final String[] METRICS_CLASS_METHODS = + {"id", "hashCode", "remoteHashCode", "equals", "serialize", "deserialize", "getMeta", "toHour", "toDay", "toMonth"}; + private static final String[] METRICS_BUILDER_CLASS_METHODS = + {"data2Map", "map2Data"}; + private final ClassPool classPool; + private ClassLoader currentClassLoader; + private Configuration configuration; + private AllDispatcherContext allDispatcherContext; + private StreamAnnotationListener streamAnnotationListener; + private DispatcherDetectorListener dispatcherDetectorListener; + private final List metricsClasses; + private final List dispatcherClasses; + private final boolean openEngineDebug; + + public OALRuntime() { + classPool = ClassPool.getDefault(); + configuration = new Configuration(new Version("2.3.28")); + configuration.setEncoding(Locale.ENGLISH, CLASS_FILE_CHARSET); + configuration.setClassLoaderForTemplateLoading(OALRuntime.class.getClassLoader(), "/code-templates"); + allDispatcherContext = new AllDispatcherContext(); + metricsClasses = new ArrayList<>(); + dispatcherClasses = new ArrayList<>(); + openEngineDebug = !StringUtil.isEmpty(System.getenv("SW_OAL_ENGINE_DEBUG")); + } + + @Override public void setStreamListener(StreamAnnotationListener listener) throws ModuleStartException { + this.streamAnnotationListener = listener; + } + + @Override public void setDispatcherListener(DispatcherDetectorListener listener) throws ModuleStartException { + dispatcherDetectorListener = listener; + } + + @Override public void start(ClassLoader currentClassLoader) throws ModuleStartException, OALCompileException { + prepareRTTempFolder(); + + this.currentClassLoader = currentClassLoader; + Reader read; + try { + read = ResourceUtils.read("scope-meta.yml"); + } catch (FileNotFoundException e) { + throw new ModuleStartException("Can't locate scope-meta.yml", e); + } + + MetaReader reader = new MetaReader(); + MetaSettings metaSettings = reader.read(read); + SourceColumnsFactory.setSettings(metaSettings); + + try { + MetricsHolder.init(); + } catch (IOException e) { + throw new ModuleStartException("load metrics functions error.", e); + } + + try { + read = ResourceUtils.read("official_analysis.oal"); + } catch (FileNotFoundException e) { + throw new ModuleStartException("Can't locate official_analysis.oal", e); + } + + OALScripts oalScripts; + try { + ScriptParser scriptParser = ScriptParser.createFromFile(read); + oalScripts = scriptParser.parse(); + } catch (IOException e) { + throw new ModuleStartException("OAL script parse analysis failure.", e); + } + + this.generateClassAtRuntime(oalScripts); + } + + @Override public void notifyAllListeners() throws ModuleStartException { + metricsClasses.forEach(streamAnnotationListener::notify); + for (Class dispatcherClass : dispatcherClasses) { + try { + dispatcherDetectorListener.addIfAsSourceDispatcher(dispatcherClass); + } catch (Exception e) { + throw new ModuleStartException(e.getMessage(), e); + } + } + } + + private void generateClassAtRuntime(OALScripts oalScripts) throws OALCompileException { + List metricsStmts = oalScripts.getMetricsStmts(); + metricsStmts.forEach(this::buildDispatcherContext); + + for (AnalysisResult metricsStmt : metricsStmts) { + metricsClasses.add(generateMetricsClass(metricsStmt)); + generateMetricsBuilderClass(metricsStmt); + } + + for (Map.Entry entry : allDispatcherContext.getAllContext().entrySet()) { + dispatcherClasses.add(generateDispatcherClass(entry.getKey(), entry.getValue())); + } + + oalScripts.getDisableCollection().getAllDisableSources().forEach(disable -> { + DisableRegister.INSTANCE.add(disable); + }); + } + + /** + * Generate metrics class, and inject it to classloader + * + * @param metricsStmt + * @return + * @throws OALCompileException + */ + private Class generateMetricsClass(AnalysisResult metricsStmt) throws OALCompileException { + String className = metricsClassName(metricsStmt, false); + CtClass parentMetricsClass = null; + try { + parentMetricsClass = classPool.get(METRICS_FUNCTION_PACKAGE + metricsStmt.getMetricsClassName()); + } catch (NotFoundException e) { + logger.error("Can't find parent class for " + className + ".", e); + throw new OALCompileException(e.getMessage(), e); + } + CtClass metricsClass = classPool.makeClass(metricsClassName(metricsStmt, true), parentMetricsClass); + try { + metricsClass.addInterface(classPool.get(WITH_METADATA_INTERFACE)); + } catch (NotFoundException e) { + logger.error("Can't find WithMetadata interface for " + className + ".", e); + throw new OALCompileException(e.getMessage(), e); + } + + ClassFile metricsClassClassFile = metricsClass.getClassFile(); + ConstPool constPool = metricsClassClassFile.getConstPool(); + + /** + * Create empty construct + */ + try { + CtConstructor defaultConstructor = CtNewConstructor.make("public " + className + "() {}", metricsClass); + metricsClass.addConstructor(defaultConstructor); + } catch (CannotCompileException e) { + logger.error("Can't add empty constructor in " + className + ".", e); + throw new OALCompileException(e.getMessage(), e); + } + + /** + * Add fields with annotations. + * + * private ${sourceField.typeName} ${sourceField.fieldName}; + */ + for (SourceColumn field : metricsStmt.getFieldsFromSource()) { + try { + CtField newField = CtField.make("private " + field.getType().getName() + " " + field.getFieldName() + ";", metricsClass); + + metricsClass.addField(newField); + + metricsClass.addMethod(CtNewMethod.getter(field.getFieldGetter(), newField)); + metricsClass.addMethod(CtNewMethod.setter(field.getFieldSetter(), newField)); + + AnnotationsAttribute annotationsAttribute = new AnnotationsAttribute(constPool, AnnotationsAttribute.visibleTag); + /** + * Add @Column(columnName = "${sourceField.columnName}") + */ + Annotation columnAnnotation = new Annotation(Column.class.getName(), constPool); + columnAnnotation.addMemberValue("columnName", new StringMemberValue(field.getColumnName(), constPool)); + annotationsAttribute.addAnnotation(columnAnnotation); + + if (field.isID()) { + /** + * Add @IDColumn + */ + Annotation idAnnotation = new Annotation(IDColumn.class.getName(), constPool); + annotationsAttribute.addAnnotation(idAnnotation); + } + + newField.getFieldInfo().addAttribute(annotationsAttribute); + + } catch (CannotCompileException e) { + logger.error("Can't add field(including set/get) " + field.getFieldName() + " in " + className + ".", e); + throw new OALCompileException(e.getMessage(), e); + } + } + + /** + * Generate methods + */ + for (String method : METRICS_CLASS_METHODS) { + StringWriter methodEntity = new StringWriter(); + try { + configuration.getTemplate("metrics/" + method + ".ftl").process(metricsStmt, methodEntity); + metricsClass.addMethod(CtNewMethod.make(methodEntity.toString(), metricsClass)); + } catch (Exception e) { + logger.error("Can't generate method " + method + " for " + className + ".", e); + throw new OALCompileException(e.getMessage(), e); + } + } + + /** + * Add following annotation to the metrics class + * + * at Stream(name = "${tableName}", scopeId = ${sourceScopeId}, builder = ${metricsName}Metrics.Builder.class, processor = MetricsStreamProcessor.class) + */ + AnnotationsAttribute annotationsAttribute = new AnnotationsAttribute(constPool, AnnotationsAttribute.visibleTag); + Annotation streamAnnotation = new Annotation(Stream.class.getName(), constPool); + streamAnnotation.addMemberValue("name", new StringMemberValue(metricsStmt.getTableName(), constPool)); + streamAnnotation.addMemberValue("scopeId", new IntegerMemberValue(constPool, metricsStmt.getSourceScopeId())); + streamAnnotation.addMemberValue("builder", new ClassMemberValue(metricsBuilderClassName(metricsStmt, true), constPool)); + streamAnnotation.addMemberValue("processor", new ClassMemberValue(METRICS_STREAM_PROCESSOR, constPool)); + + annotationsAttribute.addAnnotation(streamAnnotation); + metricsClassClassFile.addAttribute(annotationsAttribute); + + Class targetClass; + try { + targetClass = metricsClass.toClass(currentClassLoader, null); + } catch (CannotCompileException e) { + logger.error("Can't compile/load " + className + ".", e); + throw new OALCompileException(e.getMessage(), e); + } + + logger.debug("Generate metrics class, " + metricsClass.getName()); + writeGeneratedFile(metricsClass, metricsClass.getSimpleName(), "metrics"); + + return targetClass; + } + + /** + * Generate metrics class builder and inject it to classloader + * + * @param metricsStmt + * @throws OALCompileException + */ + private void generateMetricsBuilderClass(AnalysisResult metricsStmt) throws OALCompileException { + String className = metricsBuilderClassName(metricsStmt, false); + CtClass metricsBuilderClass = classPool.makeClass(metricsBuilderClassName(metricsStmt, true)); + try { + metricsBuilderClass.addInterface(classPool.get(STORAGE_BUILDER_INTERFACE)); + } catch (NotFoundException e) { + logger.error("Can't find StorageBuilder interface for " + className + ".", e); + throw new OALCompileException(e.getMessage(), e); + } + + ClassFile metricsClassClassFile = metricsBuilderClass.getClassFile(); + ConstPool constPool = metricsClassClassFile.getConstPool(); + + /** + * Create empty construct + */ + try { + CtConstructor defaultConstructor = CtNewConstructor.make("public " + className + "() {}", metricsBuilderClass); + metricsBuilderClass.addConstructor(defaultConstructor); + } catch (CannotCompileException e) { + logger.error("Can't add empty constructor in " + className + ".", e); + throw new OALCompileException(e.getMessage(), e); + } + + /** + * Generate methods + */ + for (String method : METRICS_BUILDER_CLASS_METHODS) { + StringWriter methodEntity = new StringWriter(); + try { + configuration.getTemplate("metrics-builder/" + method + ".ftl").process(metricsStmt, methodEntity); + metricsBuilderClass.addMethod(CtNewMethod.make(methodEntity.toString(), metricsBuilderClass)); + } catch (Exception e) { + logger.error("Can't generate method " + method + " for " + className + ".", e); + throw new OALCompileException(e.getMessage(), e); + } + } + + try { + metricsBuilderClass.toClass(currentClassLoader, null); + } catch (CannotCompileException e) { + logger.error("Can't compile/load " + className + ".", e); + throw new OALCompileException(e.getMessage(), e); + } + + writeGeneratedFile(metricsBuilderClass, className, "metrics/builder"); + } + + /** + * Generate SourceDispatcher class and inject it to classloader + * + * @throws OALCompileException + */ + private Class generateDispatcherClass(String scopeName, + DispatcherContext dispatcherContext) throws OALCompileException { + + String className = dispatcherClassName(scopeName, false); + CtClass dispatcherClass = classPool.makeClass(dispatcherClassName(scopeName, true)); + try { + CtClass dispatcherInterface = classPool.get(DISPATCHER_INTERFACE); + + dispatcherClass.addInterface(dispatcherInterface); + + /** + * Set generic signature + */ + String sourceClassName = SOURCE_PACKAGE + dispatcherContext.getSource(); + SignatureAttribute.ClassSignature dispatcherSignature = new SignatureAttribute.ClassSignature(null, null, + // Set interface and its generic params + new SignatureAttribute.ClassType[] { + new SignatureAttribute.ClassType(SourceDispatcher.class.getCanonicalName(), + new SignatureAttribute.TypeArgument[] {new SignatureAttribute.TypeArgument(new SignatureAttribute.ClassType(sourceClassName))} + )}); + + dispatcherClass.setGenericSignature(dispatcherSignature.encode()); + } catch (NotFoundException e) { + logger.error("Can't find Dispatcher interface for " + className + ".", e); + throw new OALCompileException(e.getMessage(), e); + } + + /** + * Generate methods + */ + for (AnalysisResult dispatcherContextMetric : dispatcherContext.getMetrics()) { + StringWriter methodEntity = new StringWriter(); + try { + configuration.getTemplate("dispatcher/doMetrics.ftl").process(dispatcherContextMetric, methodEntity); + dispatcherClass.addMethod(CtNewMethod.make(methodEntity.toString(), dispatcherClass)); + } catch (Exception e) { + logger.error("Can't generate method do" + dispatcherContextMetric.getMetricsName() + " for " + className + ".", e); + logger.error("Method body as following" + System.lineSeparator() + "{}", methodEntity); + throw new OALCompileException(e.getMessage(), e); + } + } + + try { + StringWriter methodEntity = new StringWriter(); + configuration.getTemplate("dispatcher/dispatch.ftl").process(dispatcherContext, methodEntity); + dispatcherClass.addMethod(CtNewMethod.make(methodEntity.toString(), dispatcherClass)); + } catch (Exception e) { + logger.error("Can't generate method dispatch for " + className + ".", e); + throw new OALCompileException(e.getMessage(), e); + } + + Class targetClass; + try { + targetClass = dispatcherClass.toClass(currentClassLoader, null); + } catch (CannotCompileException e) { + logger.error("Can't compile/load " + className + ".", e); + throw new OALCompileException(e.getMessage(), e); + } + + writeGeneratedFile(dispatcherClass, className, "dispatcher"); + return targetClass; + } + + private String metricsClassName(AnalysisResult metricsStmt, boolean fullName) { + return (fullName ? DYNAMIC_METRICS_CLASS_PACKAGE : "") + metricsStmt.getMetricsName() + "Metrics"; + } + + private String metricsBuilderClassName(AnalysisResult metricsStmt, boolean fullName) { + return (fullName ? DYNAMIC_METRICS_BUILDER_CLASS_PACKAGE : "") + metricsStmt.getMetricsName() + "MetricsBuilder"; + } + + private String dispatcherClassName(String scopeName, boolean fullName) { + return (fullName ? DYNAMIC_DISPATCHER_CLASS_PACKAGE : "") + scopeName + "Dispatcher"; + } + + private void buildDispatcherContext(AnalysisResult metricsStmt) { + String sourceName = metricsStmt.getSourceName(); + + DispatcherContext context = allDispatcherContext.getAllContext().get(sourceName); + if (context == null) { + context = new DispatcherContext(); + context.setSource(sourceName); + context.setPackageName(sourceName.toLowerCase()); + allDispatcherContext.getAllContext().put(sourceName, context); + } + context.getMetrics().add(metricsStmt); + } + + private void prepareRTTempFolder() { + if (openEngineDebug) { + File workPath = WorkPath.getPath(); + File folder = new File(workPath.getParentFile(), "oal-rt/"); + if (folder.exists()) { + try { + FileUtils.deleteDirectory(folder); + } catch (IOException e) { + logger.warn("Can't delete " + folder.getAbsolutePath() + " temp folder.", e); + } + } + folder.mkdirs(); + } + } + + private void writeGeneratedFile(CtClass metricsClass, String className, String type) throws OALCompileException { + if (openEngineDebug) { + DataOutputStream printWriter = null; + try { + File workPath = WorkPath.getPath(); + File folder = new File(workPath.getParentFile(), "oal-rt/" + type); + if (!folder.exists()) { + folder.mkdirs(); + } + File file = new File(folder, className + ".class"); + if (file.exists()) { + file.delete(); + } + file.createNewFile(); + + printWriter = new DataOutputStream(new FileOutputStream(file)); + metricsClass.toBytecode(printWriter); + printWriter.flush(); + } catch (IOException e) { + logger.warn("Can't create " + className + ".txt, ignore.", e); + return; + } catch (CannotCompileException e) { + logger.warn("Can't compile " + className + ".class(should not happen), ignore.", e); + return; + } finally { + if (printWriter != null) { + try { + printWriter.close(); + } catch (IOException e) { + + } + } + } + } + + } +} diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/meta/MetaReader.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/meta/MetaReader.java similarity index 72% rename from oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/meta/MetaReader.java rename to oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/meta/MetaReader.java index 8a0f9dee9..99339223a 100644 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/meta/MetaReader.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/meta/MetaReader.java @@ -16,18 +16,26 @@ * */ -package org.apache.skywalking.oal.tool.meta; +package org.apache.skywalking.oal.rt.meta; import java.io.InputStream; +import java.io.Reader; import org.yaml.snakeyaml.Yaml; /** * @author wusheng */ public class MetaReader { - public MetaSettings read(InputStream settingFileStream) { + public MetaSettings read(Reader reader) { Yaml yaml = new Yaml(); - MetaSettings settings = yaml.loadAs(settingFileStream, MetaSettings.class); + MetaSettings settings = yaml.loadAs(reader, MetaSettings.class); + + return settings; + } + + public MetaSettings read(InputStream reader) { + Yaml yaml = new Yaml(); + MetaSettings settings = yaml.loadAs(reader, MetaSettings.class); return settings; } diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/meta/MetaSettings.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/meta/MetaSettings.java similarity index 95% rename from oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/meta/MetaSettings.java rename to oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/meta/MetaSettings.java index 7e5ea0a78..31b3cd365 100644 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/meta/MetaSettings.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/meta/MetaSettings.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oal.tool.meta; +package org.apache.skywalking.oal.rt.meta; import java.util.List; import lombok.*; diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/meta/ScopeMeta.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/meta/ScopeMeta.java similarity index 90% rename from oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/meta/ScopeMeta.java rename to oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/meta/ScopeMeta.java index f027d4343..7a674103d 100644 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/meta/ScopeMeta.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/meta/ScopeMeta.java @@ -16,11 +16,11 @@ * */ -package org.apache.skywalking.oal.tool.meta; +package org.apache.skywalking.oal.rt.meta; import java.util.*; import lombok.*; -import org.apache.skywalking.oal.tool.parser.SourceColumn; +import org.apache.skywalking.oal.rt.parser.SourceColumn; /** * @author wusheng diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/output/AllDispatcherContext.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/output/AllDispatcherContext.java similarity index 95% rename from oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/output/AllDispatcherContext.java rename to oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/output/AllDispatcherContext.java index ac11fe750..1a406ad86 100644 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/output/AllDispatcherContext.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/output/AllDispatcherContext.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oal.tool.output; +package org.apache.skywalking.oal.rt.output; import java.util.*; import lombok.*; diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/output/DispatcherContext.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/output/DispatcherContext.java similarity index 90% rename from oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/output/DispatcherContext.java rename to oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/output/DispatcherContext.java index b8913e342..be54da60f 100644 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/output/DispatcherContext.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/output/DispatcherContext.java @@ -16,11 +16,11 @@ * */ -package org.apache.skywalking.oal.tool.output; +package org.apache.skywalking.oal.rt.output; import java.util.*; import lombok.*; -import org.apache.skywalking.oal.tool.parser.AnalysisResult; +import org.apache.skywalking.oal.rt.parser.AnalysisResult; @Getter @Setter diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/AnalysisResult.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/AnalysisResult.java similarity index 96% rename from oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/AnalysisResult.java rename to oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/AnalysisResult.java index 20a7e49f1..70090a945 100644 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/AnalysisResult.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/AnalysisResult.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oal.tool.parser; +package org.apache.skywalking.oal.rt.parser; import java.util.*; import lombok.*; @@ -44,7 +44,7 @@ public class AnalysisResult { private EntryMethod entryMethod; - private List filterExpressions; + private List filterExpressions; private List filterExpressionsParserResult; @@ -74,7 +74,7 @@ public class AnalysisResult { funcConditionExpressions.add(conditionExpression); } - public void addFilterExpressions(FilterExpression filterExpression) { + public void addFilterExpressions(Expression filterExpression) { if (filterExpressions == null) { filterExpressions = new LinkedList<>(); } diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/ConditionExpression.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/ConditionExpression.java similarity index 95% rename from oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/ConditionExpression.java rename to oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/ConditionExpression.java index 96433397a..e5385672e 100644 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/ConditionExpression.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/ConditionExpression.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oal.tool.parser; +package org.apache.skywalking.oal.rt.parser; import lombok.*; diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/DataColumn.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/DataColumn.java similarity index 93% rename from oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/DataColumn.java rename to oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/DataColumn.java index 31c81887a..0a2c0b898 100644 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/DataColumn.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/DataColumn.java @@ -16,10 +16,10 @@ * */ -package org.apache.skywalking.oal.tool.parser; +package org.apache.skywalking.oal.rt.parser; import lombok.*; -import org.apache.skywalking.oal.tool.util.ClassMethodUtil; +import org.apache.skywalking.oal.rt.util.ClassMethodUtil; @Getter(AccessLevel.PUBLIC) @Setter(AccessLevel.PUBLIC) diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/DeepAnalysis.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/DeepAnalysis.java similarity index 69% rename from oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/DeepAnalysis.java rename to oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/DeepAnalysis.java index 564536072..99622fab0 100644 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/DeepAnalysis.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/DeepAnalysis.java @@ -16,12 +16,12 @@ * */ -package org.apache.skywalking.oal.tool.parser; +package org.apache.skywalking.oal.rt.parser; import java.lang.annotation.Annotation; import java.lang.reflect.*; import java.util.List; -import org.apache.skywalking.oal.tool.util.ClassMethodUtil; +import org.apache.skywalking.oal.rt.util.ClassMethodUtil; import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.*; import org.apache.skywalking.oap.server.core.storage.annotation.Column; @@ -39,7 +39,7 @@ public class DeepAnalysis { List expressions = result.getFilterExpressionsParserResult(); if (expressions != null && expressions.size() > 0) { for (ConditionExpression expression : expressions) { - FilterExpression filterExpression = new FilterExpression(); + Expression filterExpression = new Expression(); if ("booleanMatch".equals(expression.getExpressionType())) { filterExpression.setExpressionObject("EqualMatch"); filterExpression.setLeft("source." + ClassMethodUtil.toIsMethod(expression.getAttribute()) + "()"); @@ -99,39 +99,55 @@ public class DeepAnalysis { // 4. Use parameter's annotation of entrance method to generate aggregation entrance. for (Parameter parameter : entranceMethod.getParameters()) { + Class parameterType = parameter.getType(); Annotation[] parameterAnnotations = parameter.getAnnotations(); if (parameterAnnotations == null || parameterAnnotations.length == 0) { throw new IllegalArgumentException("Entrance method:" + entranceMethod + " doesn't include the annotation."); } Annotation annotation = parameterAnnotations[0]; if (annotation instanceof SourceFrom) { - entryMethod.addArg("source." + ClassMethodUtil.toGetMethod(result.getSourceAttribute()) + "()"); + entryMethod.addArg(parameterType, "source." + ClassMethodUtil.toGetMethod(result.getSourceAttribute()) + "()"); } else if (annotation instanceof ConstOne) { - entryMethod.addArg("1"); - } else if (annotation instanceof Expression) { - entryMethod.addArg("new " + parameter.getType().getName() + "()"); - } else if (annotation instanceof ExpressionArg0) { + entryMethod.addArg(parameterType, "1"); + } else if (annotation instanceof org.apache.skywalking.oap.server.core.analysis.metrics.annotation.Expression) { if (result.getFuncConditionExpressions().size() == 1) { - ConditionExpression conditionExpression = result.getFuncConditionExpressions().get(0); - if ("booleanMatch".equals(conditionExpression.getExpressionType())) { - entryMethod.addArg("source." + ClassMethodUtil.toIsMethod(conditionExpression.getAttribute()) + "()"); - } else if ("stringMatch".equals(conditionExpression.getExpressionType())) { - entryMethod.addArg("source." + ClassMethodUtil.toGetMethod(conditionExpression.getAttribute()) + "()"); + ConditionExpression expression = result.getFuncConditionExpressions().get(0); + + Expression argExpression = new Expression(); + if ("booleanMatch".equals(expression.getExpressionType())) { + argExpression.setExpressionObject("EqualMatch"); + argExpression.setLeft("source." + ClassMethodUtil.toIsMethod(expression.getAttribute()) + "()"); + argExpression.setRight(expression.getValue()); + } else if ("stringMatch".equals(expression.getExpressionType())) { + argExpression.setExpressionObject("EqualMatch"); + argExpression.setLeft("source." + ClassMethodUtil.toGetMethod(expression.getAttribute()) + "()"); + argExpression.setRight(expression.getValue()); + } else if ("greaterMatch".equals(expression.getExpressionType())) { + argExpression.setExpressionObject("GreaterMatch"); + argExpression.setLeft("source." + ClassMethodUtil.toGetMethod(expression.getAttribute()) + "()"); + argExpression.setRight(expression.getValue()); + } else if ("lessMatch".equals(expression.getExpressionType())) { + argExpression.setExpressionObject("LessMatch"); + argExpression.setLeft("source." + ClassMethodUtil.toGetMethod(expression.getAttribute()) + "()"); + argExpression.setRight(expression.getValue()); + } else if ("greaterEqualMatch".equals(expression.getExpressionType())) { + argExpression.setExpressionObject("GreaterEqualMatch"); + argExpression.setLeft("source." + ClassMethodUtil.toGetMethod(expression.getAttribute()) + "()"); + argExpression.setRight(expression.getValue()); + } else if ("lessEqualMatch".equals(expression.getExpressionType())) { + argExpression.setExpressionObject("LessEqualMatch"); + argExpression.setLeft("source." + ClassMethodUtil.toGetMethod(expression.getAttribute()) + "()"); + argExpression.setRight(expression.getValue()); } else { - throw new IllegalArgumentException("Entrance method:" + entranceMethod + " argument has @ExpressionArg0, but expression type is not supported"); + throw new IllegalArgumentException("filter expression [" + expression.getExpressionType() + "] not found"); } + + entryMethod.addArg(argExpression); } else { - throw new IllegalArgumentException("Entrance method:" + entranceMethod + " argument has @ExpressionArg0, but can't find funcParamExpression."); - } - } else if (annotation instanceof ExpressionArg1) { - if (result.getFuncConditionExpressions().size() == 1) { - ConditionExpression conditionExpression = result.getFuncConditionExpressions().get(0); - entryMethod.addArg(conditionExpression.getValue()); - } else { - throw new IllegalArgumentException("Entrance method:" + entranceMethod + " argument has @ExpressionArg0, but can't find funcParamExpression."); + throw new IllegalArgumentException("Entrance method:" + entranceMethod + " argument can't find funcParamExpression."); } } else if (annotation instanceof Arg) { - entryMethod.addArg(result.getNextFuncArg()); + entryMethod.addArg(parameterType, result.getNextFuncArg()); } else { throw new IllegalArgumentException("Entrance method:" + entranceMethod + " doesn't the expected annotation."); } diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/DisableCollection.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/DisableCollection.java similarity index 95% rename from oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/DisableCollection.java rename to oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/DisableCollection.java index 08987c244..17539ccf0 100644 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/DisableCollection.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/DisableCollection.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oal.tool.parser; +package org.apache.skywalking.oal.rt.parser; import java.util.*; import lombok.Getter; diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/EntryMethod.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/EntryMethod.java similarity index 52% rename from oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/EntryMethod.java rename to oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/EntryMethod.java index d87ad3bdf..8f08e24a4 100644 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/EntryMethod.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/EntryMethod.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oal.tool.parser; +package org.apache.skywalking.oal.rt.parser; import java.util.*; import lombok.*; @@ -24,14 +24,29 @@ import lombok.*; @Getter(AccessLevel.PUBLIC) @Setter(AccessLevel.PUBLIC) public class EntryMethod { + private static final int LITERAL_TYPE = 1; + private static final int EXPRESSION_TYPE = 2; + private String methodName; + private List argTypes = new ArrayList<>(); + private List argsExpressions = new ArrayList<>(); - private List argsExpressions; - - public void addArg(String expression) { - if (argsExpressions == null) { - argsExpressions = new LinkedList<>(); + public void addArg(Class parameterType, String expression) { + if (parameterType.equals(int.class)) { + expression = "(int)(" + expression + ")"; + } else if (parameterType.equals(long.class)) { + expression = "(long)(" + expression + ")"; + } else if (parameterType.equals(double.class)) { + expression = "(double)(" + expression + ")"; + } else if (parameterType.equals(float.class)) { + expression = "(float)(" + expression + ")"; } + argTypes.add(LITERAL_TYPE); + argsExpressions.add(expression); + } + + public void addArg(Expression expression) { + argTypes.add(EXPRESSION_TYPE); argsExpressions.add(expression); } } diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/FilterExpression.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/Expression.java similarity index 76% rename from oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/FilterExpression.java rename to oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/Expression.java index 20dc47136..ccd185660 100644 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/FilterExpression.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/Expression.java @@ -16,14 +16,24 @@ * */ -package org.apache.skywalking.oal.tool.parser; +package org.apache.skywalking.oal.rt.parser; -import lombok.*; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.Setter; @Getter(AccessLevel.PUBLIC) @Setter(AccessLevel.PUBLIC) -public class FilterExpression { +public class Expression { private String expressionObject; private String left; private String right; + + public void setLeft(String left) { + this.left = left; + } + + public void setRight(String right) { + this.right = right; + } } diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/MetricsHolder.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/MetricsHolder.java similarity index 98% rename from oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/MetricsHolder.java rename to oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/MetricsHolder.java index 56b90667b..af32d0efb 100644 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/MetricsHolder.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/MetricsHolder.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oal.tool.parser; +package org.apache.skywalking.oal.rt.parser; import com.google.common.collect.ImmutableSet; import com.google.common.reflect.ClassPath; diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/OALListener.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/OALListener.java similarity index 90% rename from oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/OALListener.java rename to oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/OALListener.java index 8af30730a..a43d723fb 100644 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/OALListener.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/OALListener.java @@ -16,11 +16,11 @@ * */ -package org.apache.skywalking.oal.tool.parser; +package org.apache.skywalking.oal.rt.parser; import java.util.List; import org.antlr.v4.runtime.misc.NotNull; -import org.apache.skywalking.oal.tool.grammar.*; +import org.apache.skywalking.oal.rt.grammar.*; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; public class OALListener extends OALParserBaseListener { @@ -120,19 +120,27 @@ public class OALListener extends OALParserBaseListener { } @Override public void enterBooleanConditionValue(OALParser.BooleanConditionValueContext ctx) { - conditionExpression.setValue(ctx.getText()); + enterConditionValue(ctx.getText()); } @Override public void enterStringConditionValue(OALParser.StringConditionValueContext ctx) { - conditionExpression.setValue(ctx.getText()); + enterConditionValue(ctx.getText()); } @Override public void enterEnumConditionValue(OALParser.EnumConditionValueContext ctx) { - conditionExpression.setValue(ctx.getText()); + enterConditionValue(ctx.getText()); } @Override public void enterNumberConditionValue(OALParser.NumberConditionValueContext ctx) { - conditionExpression.setValue(ctx.getText()); + enterConditionValue(ctx.getText()); + } + + private void enterConditionValue(String value) { + if (value.split("\\.").length == 2 && !value.startsWith("\"")) { + // Value is an enum. + value = "org.apache.skywalking.oap.server.core.source." + value; + } + conditionExpression.setValue(value); } ///////////// diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/OALScripts.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/OALScripts.java similarity index 95% rename from oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/OALScripts.java rename to oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/OALScripts.java index cfb427756..8af71be04 100644 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/OALScripts.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/OALScripts.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oal.tool.parser; +package org.apache.skywalking.oal.rt.parser; import java.util.*; import lombok.*; diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/PersistenceColumns.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/PersistenceColumns.java similarity index 97% rename from oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/PersistenceColumns.java rename to oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/PersistenceColumns.java index 8a78e3d38..a93d7f4ab 100644 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/PersistenceColumns.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/PersistenceColumns.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oal.tool.parser; +package org.apache.skywalking.oal.rt.parser; import java.util.*; diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/PersistenceField.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/PersistenceField.java similarity index 91% rename from oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/PersistenceField.java rename to oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/PersistenceField.java index 5e9c254fb..c126ef0e2 100644 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/PersistenceField.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/PersistenceField.java @@ -16,10 +16,10 @@ * */ -package org.apache.skywalking.oal.tool.parser; +package org.apache.skywalking.oal.rt.parser; import lombok.*; -import org.apache.skywalking.oal.tool.util.ClassMethodUtil; +import org.apache.skywalking.oal.rt.util.ClassMethodUtil; @Getter(AccessLevel.PUBLIC) @Setter(AccessLevel.PUBLIC) diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/ScriptParser.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/ScriptParser.java similarity index 83% rename from oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/ScriptParser.java rename to oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/ScriptParser.java index 9091fbd0c..70ff752b1 100644 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/ScriptParser.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/ScriptParser.java @@ -16,14 +16,19 @@ * */ -package org.apache.skywalking.oal.tool.parser; +package org.apache.skywalking.oal.rt.parser; import java.io.IOException; -import java.util.*; +import java.io.Reader; import org.antlr.v4.runtime.*; import org.antlr.v4.runtime.tree.*; -import org.apache.skywalking.oal.tool.grammar.*; +import org.apache.skywalking.oal.rt.grammar.*; +/** + * Script reader and parser. + * + * @author wusheng + */ public class ScriptParser { private OALLexer lexer; @@ -31,9 +36,9 @@ public class ScriptParser { } - public static ScriptParser createFromFile(String scriptFilepath) throws IOException { + public static ScriptParser createFromFile(Reader scriptReader) throws IOException { ScriptParser parser = new ScriptParser(); - parser.lexer = new OALLexer(CharStreams.fromFileName(scriptFilepath)); + parser.lexer = new OALLexer(CharStreams.fromReader(scriptReader)); return parser; } diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/SourceColumn.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumn.java similarity index 97% rename from oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/SourceColumn.java rename to oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumn.java index 74a7706c2..49e16114f 100644 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/SourceColumn.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumn.java @@ -16,11 +16,11 @@ * */ -package org.apache.skywalking.oal.tool.parser; +package org.apache.skywalking.oal.rt.parser; import java.util.Objects; import lombok.*; -import org.apache.skywalking.oal.tool.util.ClassMethodUtil; +import org.apache.skywalking.oal.rt.util.ClassMethodUtil; @Getter(AccessLevel.PUBLIC) @Setter(AccessLevel.PUBLIC) diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/SourceColumnsFactory.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumnsFactory.java similarity index 93% rename from oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/SourceColumnsFactory.java rename to oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumnsFactory.java index 7be457cbf..7955503f9 100644 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/SourceColumnsFactory.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumnsFactory.java @@ -16,10 +16,10 @@ * */ -package org.apache.skywalking.oal.tool.parser; +package org.apache.skywalking.oal.rt.parser; import java.util.*; -import org.apache.skywalking.oal.tool.meta.*; +import org.apache.skywalking.oal.rt.meta.*; /** * @author wusheng diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/util/ClassMethodUtil.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/util/ClassMethodUtil.java similarity index 96% rename from oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/util/ClassMethodUtil.java rename to oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/util/ClassMethodUtil.java index 140ab637c..4a880a6be 100644 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/util/ClassMethodUtil.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/util/ClassMethodUtil.java @@ -16,7 +16,7 @@ * */ -package org.apache.skywalking.oal.tool.util; +package org.apache.skywalking.oal.rt.util; public class ClassMethodUtil { public static String toGetMethod(String attribute) { diff --git a/oap-server/oal-rt/src/main/resources/code-templates/dispatcher/dispatch.ftl b/oap-server/oal-rt/src/main/resources/code-templates/dispatcher/dispatch.ftl new file mode 100644 index 000000000..89a365028 --- /dev/null +++ b/oap-server/oal-rt/src/main/resources/code-templates/dispatcher/dispatch.ftl @@ -0,0 +1,6 @@ +public void dispatch(org.apache.skywalking.oap.server.core.source.Source source) { + org.apache.skywalking.oap.server.core.source.${source} _source = (org.apache.skywalking.oap.server.core.source.${source})source; + <#list metrics as metrics> + do${metrics.metricsName}(_source); + +} \ No newline at end of file diff --git a/oap-server/oal-rt/src/main/resources/code-templates/dispatcher/doMetrics.ftl b/oap-server/oal-rt/src/main/resources/code-templates/dispatcher/doMetrics.ftl new file mode 100644 index 000000000..058352e82 --- /dev/null +++ b/oap-server/oal-rt/src/main/resources/code-templates/dispatcher/doMetrics.ftl @@ -0,0 +1,26 @@ +private void do${metricsName}(org.apache.skywalking.oap.server.core.source.${sourceName} source) { + org.apache.skywalking.oal.rt.metrics.${metricsName}Metrics metrics = new org.apache.skywalking.oal.rt.metrics.${metricsName}Metrics(); + + <#if filterExpressions??> + <#list filterExpressions as filterExpression> + if (!new org.apache.skywalking.oap.server.core.analysis.metrics.expression.${filterExpression.expressionObject}().match(${filterExpression.left}, ${filterExpression.right})) { + return; + } + + + + metrics.setTimeBucket(source.getTimeBucket()); + <#list fieldsFromSource as field> + metrics.${field.fieldSetter}(source.${field.fieldGetter}()); + + metrics.${entryMethod.methodName}( + <#list entryMethod.argsExpressions as arg> + <#if entryMethod.argTypes[arg_index] == 1> + ${arg} + <#else> + new org.apache.skywalking.oap.server.core.analysis.metrics.expression.${arg.expressionObject}().match(${arg.left}, ${arg.right}) + <#if arg_has_next>, + ); + + org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor.getInstance().in(metrics); +} diff --git a/oap-server/oal-rt/src/main/resources/code-templates/metrics-builder/data2Map.ftl b/oap-server/oal-rt/src/main/resources/code-templates/metrics-builder/data2Map.ftl new file mode 100644 index 000000000..8dc79ade7 --- /dev/null +++ b/oap-server/oal-rt/src/main/resources/code-templates/metrics-builder/data2Map.ftl @@ -0,0 +1,31 @@ +public java.util.Map data2Map(org.apache.skywalking.oap.server.core.storage.StorageData input) { + org.apache.skywalking.oal.rt.metrics.${metricsName}Metrics storageData = (org.apache.skywalking.oal.rt.metrics.${metricsName}Metrics)input; + java.util.Map map = new java.util.HashMap(); + <#list fieldsFromSource as field> + <#if field.typeName == "long"> + map.put((Object)"${field.columnName}", new Long(storageData.${field.fieldGetter}())); + <#elseif field.typeName == "int"> + map.put((Object)"${field.columnName}", new Integer(storageData.${field.fieldGetter}())); + <#elseif field.typeName == "double"> + map.put((Object)"${field.columnName}", new Double(storageData.${field.fieldGetter}())); + <#elseif field.typeName == "float"> + map.put((Object)"${field.columnName}", new Float(storageData.${field.fieldGetter}())); + <#else> + map.put((Object)"${field.columnName}", storageData.${field.fieldGetter}()); + + + <#list persistentFields as field> + <#if field.typeName == "long"> + map.put((Object)"${field.columnName}", new Long(storageData.${field.fieldGetter}())); + <#elseif field.typeName == "int"> + map.put((Object)"${field.columnName}", new Integer(storageData.${field.fieldGetter}())); + <#elseif field.typeName == "double"> + map.put((Object)"${field.columnName}", new Double(storageData.${field.fieldGetter}())); + <#elseif field.typeName == "float"> + map.put((Object)"${field.columnName}", new Float(storageData.${field.fieldGetter}())); + <#else> + map.put((Object)"${field.columnName}", storageData.${field.fieldGetter}()); + + + return map; +} \ No newline at end of file diff --git a/oap-server/oal-rt/src/main/resources/code-templates/metrics-builder/map2Data.ftl b/oap-server/oal-rt/src/main/resources/code-templates/metrics-builder/map2Data.ftl new file mode 100644 index 000000000..1dfbf45ac --- /dev/null +++ b/oap-server/oal-rt/src/main/resources/code-templates/metrics-builder/map2Data.ftl @@ -0,0 +1,22 @@ +public org.apache.skywalking.oap.server.core.storage.StorageData map2Data(java.util.Map dbMap) { + org.apache.skywalking.oal.rt.metrics.${metricsName}Metrics metrics = new org.apache.skywalking.oal.rt.metrics.${metricsName}Metrics(); + <#list fieldsFromSource as field> + <#if field.typeName == "long" || field.typeName == "int" || field.typeName == "double" || field.typeName == "float"> + metrics.${field.fieldSetter}(((Number)dbMap.get("${field.columnName}")).${field.typeName}Value()); + <#elseif field.typeName == "java.lang.String"> + metrics.${field.fieldSetter}((String)dbMap.get("${field.columnName}")); + <#else> + metrics.${field.fieldSetter}(new ${field.typeName}((String)dbMap.get("${field.columnName}"))); + + + <#list persistentFields as field> + <#if field.typeName == "long" || field.typeName == "int" || field.typeName == "double" || field.typeName == "float"> + metrics.${field.fieldSetter}(((Number)dbMap.get("${field.columnName}")).${field.typeName}Value()); + <#elseif field.typeName == "java.lang.String"> + metrics.${field.fieldSetter}((String)dbMap.get("${field.columnName}")); + <#else> + metrics.${field.fieldSetter}(new ${field.typeName}((String)dbMap.get("${field.columnName}"))); + + + return metrics; +} \ No newline at end of file diff --git a/oap-server/oal-rt/src/main/resources/code-templates/metrics/deserialize.ftl b/oap-server/oal-rt/src/main/resources/code-templates/metrics/deserialize.ftl new file mode 100644 index 000000000..bbd0ad049 --- /dev/null +++ b/oap-server/oal-rt/src/main/resources/code-templates/metrics/deserialize.ftl @@ -0,0 +1,27 @@ +public void deserialize(org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData remoteData) { + <#list serializeFields.stringFields as field> + ${field.setter}(remoteData.getDataStrings(${field?index})); + + + <#list serializeFields.longFields as field> + ${field.setter}(remoteData.getDataLongs(${field?index})); + + + <#list serializeFields.doubleFields as field> + ${field.setter}(remoteData.getDataDoubles(${field?index})); + + + <#list serializeFields.intFields as field> + ${field.setter}(remoteData.getDataIntegers(${field?index})); + + + <#list serializeFields.intLongValuePairListFields as field> + setDetailGroup(new org.apache.skywalking.oap.server.core.analysis.metrics.IntKeyLongValueArray(30)); + + java.util.Iterator iterator = remoteData.getDataIntLongPairListList().iterator(); + while (iterator.hasNext()) { + org.apache.skywalking.oap.server.core.remote.grpc.proto.IntKeyLongValuePair element = (org.apache.skywalking.oap.server.core.remote.grpc.proto.IntKeyLongValuePair)(iterator.next()); + super.getDetailGroup().add(new org.apache.skywalking.oap.server.core.analysis.metrics.IntKeyLongValue(element.getKey(), element.getValue())); + } + +} \ No newline at end of file diff --git a/oap-server/oal-rt/src/main/resources/code-templates/metrics/equals.ftl b/oap-server/oal-rt/src/main/resources/code-templates/metrics/equals.ftl new file mode 100644 index 000000000..15f9e1752 --- /dev/null +++ b/oap-server/oal-rt/src/main/resources/code-templates/metrics/equals.ftl @@ -0,0 +1,25 @@ +public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + + org.apache.skywalking.oal.rt.metrics.${metricsName}Metrics metrics = (org.apache.skywalking.oal.rt.metrics.${metricsName}Metrics)obj; + <#list fieldsFromSource as sourceField> + <#if sourceField.isID()> + <#if sourceField.getTypeName() == "java.lang.String"> + if (!${sourceField.fieldName}.equals(metrics.${sourceField.fieldName})) + <#else> + if (${sourceField.fieldName} != metrics.${sourceField.fieldName}) + + return false; + + + + if (getTimeBucket() != metrics.getTimeBucket()) + return false; + + return true; +} \ No newline at end of file diff --git a/oap-server/oal-rt/src/main/resources/code-templates/metrics/getMeta.ftl b/oap-server/oal-rt/src/main/resources/code-templates/metrics/getMeta.ftl new file mode 100644 index 000000000..bc7f39e26 --- /dev/null +++ b/oap-server/oal-rt/src/main/resources/code-templates/metrics/getMeta.ftl @@ -0,0 +1,3 @@ +public org.apache.skywalking.oap.server.core.analysis.metrics.MetricsMetaInfo getMeta() { + return new org.apache.skywalking.oap.server.core.analysis.metrics.MetricsMetaInfo("${varName}", ${sourceScopeId}<#if (fieldsFromSource?size>0) ><#list fieldsFromSource as field><#if field.isID()>, ${field.fieldName}); +} \ No newline at end of file diff --git a/oap-server/oal-rt/src/main/resources/code-templates/metrics/hashCode.ftl b/oap-server/oal-rt/src/main/resources/code-templates/metrics/hashCode.ftl new file mode 100644 index 000000000..ed901e75d --- /dev/null +++ b/oap-server/oal-rt/src/main/resources/code-templates/metrics/hashCode.ftl @@ -0,0 +1,14 @@ +public int hashCode() { + int result = 17; + <#list fieldsFromSource as sourceField> + <#if sourceField.isID()> + <#if sourceField.getTypeName() == "java.lang.String"> + result = 31 * result + ${sourceField.fieldName}.hashCode(); + <#else> + result += Const.ID_SPLIT + ${sourceField.fieldName}; + + + + result = 31 * result + (int)getTimeBucket(); + return result; +} \ No newline at end of file diff --git a/oap-server/oal-rt/src/main/resources/code-templates/metrics/id.ftl b/oap-server/oal-rt/src/main/resources/code-templates/metrics/id.ftl new file mode 100644 index 000000000..00cd1c855 --- /dev/null +++ b/oap-server/oal-rt/src/main/resources/code-templates/metrics/id.ftl @@ -0,0 +1,13 @@ +public String id() { + String splitJointId = String.valueOf(getTimeBucket()); + <#list fieldsFromSource as sourceField> + <#if sourceField.isID()> + <#if sourceField.getTypeName() == "java.lang.String"> + splitJointId += org.apache.skywalking.oap.server.core.Const.ID_SPLIT + ${sourceField.fieldName}; + <#else> + splitJointId += org.apache.skywalking.oap.server.core.Const.ID_SPLIT + String.valueOf(${sourceField.fieldName}); + + + + return splitJointId; +} \ No newline at end of file diff --git a/oap-server/oal-rt/src/main/resources/code-templates/metrics/remoteHashCode.ftl b/oap-server/oal-rt/src/main/resources/code-templates/metrics/remoteHashCode.ftl new file mode 100644 index 000000000..9d75ab356 --- /dev/null +++ b/oap-server/oal-rt/src/main/resources/code-templates/metrics/remoteHashCode.ftl @@ -0,0 +1,13 @@ +public int remoteHashCode() { + int result = 17; + <#list fieldsFromSource as sourceField> + <#if sourceField.isID()> + <#if sourceField.getTypeName() == "java.lang.String"> + result = 31 * result + ${sourceField.fieldName}.hashCode(); + <#else> + result += org.apache.skywalking.oap.server.core.Const.ID_SPLIT + ${sourceField.fieldName}; + + + + return result; +} \ No newline at end of file diff --git a/oap-server/oal-rt/src/main/resources/code-templates/metrics/serialize.ftl b/oap-server/oal-rt/src/main/resources/code-templates/metrics/serialize.ftl new file mode 100644 index 000000000..53af5b3b1 --- /dev/null +++ b/oap-server/oal-rt/src/main/resources/code-templates/metrics/serialize.ftl @@ -0,0 +1,26 @@ +public org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData.Builder serialize() { + org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData.Builder remoteBuilder = org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData.newBuilder(); + <#list serializeFields.stringFields as field> + remoteBuilder.addDataStrings(${field.getter}()); + + + <#list serializeFields.longFields as field> + remoteBuilder.addDataLongs(${field.getter}()); + + + <#list serializeFields.doubleFields as field> + remoteBuilder.addDataDoubles(${field.getter}()); + + + <#list serializeFields.intFields as field> + remoteBuilder.addDataIntegers(${field.getter}()); + + <#list serializeFields.intLongValuePairListFields as field> + java.util.Iterator iterator = super.getDetailGroup().iterator(); + while (iterator.hasNext()) { + remoteBuilder.addDataIntLongPairList(((org.apache.skywalking.oap.server.core.analysis.metrics.IntKeyLongValue)(iterator.next())).serialize()); + } + + + return remoteBuilder; +} \ No newline at end of file diff --git a/oap-server/oal-rt/src/main/resources/code-templates/metrics/toDay.ftl b/oap-server/oal-rt/src/main/resources/code-templates/metrics/toDay.ftl new file mode 100644 index 000000000..8dcd474d5 --- /dev/null +++ b/oap-server/oal-rt/src/main/resources/code-templates/metrics/toDay.ftl @@ -0,0 +1,26 @@ +public org.apache.skywalking.oap.server.core.analysis.metrics.Metrics toDay() { + org.apache.skywalking.oal.rt.metrics.${metricsName}Metrics metrics = new org.apache.skywalking.oal.rt.metrics.${metricsName}Metrics(); + <#list fieldsFromSource as field> + <#if field.columnName == "time_bucket"> + metrics.setTimeBucket(toTimeBucketInDay()); + <#elseif field.typeName == "java.lang.String" || field.typeName == "long" || field.typeName == "int" || field.typeName == "double" || field.typeName == "float"> + metrics.${field.fieldSetter}(this.${field.fieldGetter}()); + <#else> + ${field.typeName} newValue = new ${field.typeName}(); + newValue.copyFrom(this.${field.fieldGetter}()); + metrics.${field.fieldSetter}(newValue); + + + <#list persistentFields as field> + <#if field.columnName == "time_bucket"> + metrics.setTimeBucket(toTimeBucketInDay()); + <#elseif field.typeName == "java.lang.String" || field.typeName == "long" || field.typeName == "int" || field.typeName == "double" || field.typeName == "float"> + metrics.${field.fieldSetter}(this.${field.fieldGetter}()); + <#else> + ${field.typeName} newValue = new ${field.typeName}(); + newValue.copyFrom(this.${field.fieldGetter}()); + metrics.${field.fieldSetter}(newValue); + + + return metrics; +} \ No newline at end of file diff --git a/oap-server/oal-rt/src/main/resources/code-templates/metrics/toHour.ftl b/oap-server/oal-rt/src/main/resources/code-templates/metrics/toHour.ftl new file mode 100644 index 000000000..1328eb930 --- /dev/null +++ b/oap-server/oal-rt/src/main/resources/code-templates/metrics/toHour.ftl @@ -0,0 +1,26 @@ +public org.apache.skywalking.oap.server.core.analysis.metrics.Metrics toHour() { + org.apache.skywalking.oal.rt.metrics.${metricsName}Metrics metrics = new org.apache.skywalking.oal.rt.metrics.${metricsName}Metrics(); + <#list fieldsFromSource as field> + <#if field.columnName == "time_bucket"> + metrics.setTimeBucket(toTimeBucketInHour()); + <#elseif field.typeName == "java.lang.String" || field.typeName == "long" || field.typeName == "int" || field.typeName == "double" || field.typeName == "float"> + metrics.${field.fieldSetter}(this.${field.fieldGetter}()); + <#else> + ${field.typeName} newValue = new ${field.typeName}(); + newValue.copyFrom(this.${field.fieldGetter}()); + metrics.${field.fieldSetter}(newValue); + + + <#list persistentFields as field> + <#if field.columnName == "time_bucket"> + metrics.setTimeBucket(toTimeBucketInHour()); + <#elseif field.typeName == "java.lang.String" || field.typeName == "long" || field.typeName == "int" || field.typeName == "double" || field.typeName == "float"> + metrics.${field.fieldSetter}(this.${field.fieldGetter}()); + <#else> + ${field.typeName} newValue = new ${field.typeName}(); + newValue.copyFrom(this.${field.fieldGetter}()); + metrics.${field.fieldSetter}(newValue); + + + return metrics; +} \ No newline at end of file diff --git a/oap-server/oal-rt/src/main/resources/code-templates/metrics/toMonth.ftl b/oap-server/oal-rt/src/main/resources/code-templates/metrics/toMonth.ftl new file mode 100644 index 000000000..b0363857e --- /dev/null +++ b/oap-server/oal-rt/src/main/resources/code-templates/metrics/toMonth.ftl @@ -0,0 +1,26 @@ +public org.apache.skywalking.oap.server.core.analysis.metrics.Metrics toMonth() { + org.apache.skywalking.oal.rt.metrics.${metricsName}Metrics metrics = new org.apache.skywalking.oal.rt.metrics.${metricsName}Metrics(); + <#list fieldsFromSource as field> + <#if field.columnName == "time_bucket"> + metrics.setTimeBucket(toTimeBucketInMonth()); + <#elseif field.typeName == "java.lang.String" || field.typeName == "long" || field.typeName == "int" || field.typeName == "double" || field.typeName == "float"> + metrics.${field.fieldSetter}(this.${field.fieldGetter}()); + <#else> + ${field.typeName} newValue = new ${field.typeName}(); + newValue.copyFrom(this.${field.fieldGetter}()); + metrics.${field.fieldSetter}(newValue); + + + <#list persistentFields as field> + <#if field.columnName == "time_bucket"> + metrics.setTimeBucket(toTimeBucketInMonth()); + <#elseif field.typeName == "java.lang.String" || field.typeName == "long" || field.typeName == "int" || field.typeName == "double" || field.typeName == "float"> + metrics.${field.fieldSetter}(this.${field.fieldGetter}()); + <#else> + ${field.typeName} newValue = new ${field.typeName}(); + newValue.copyFrom(this.${field.fieldGetter}()); + metrics.${field.fieldSetter}(newValue); + + + return metrics; +} \ No newline at end of file diff --git a/oap-server/generate-tool/src/test/java/org/apache/skywalking/oal/tool/meta/MetaReaderTest.java b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/meta/MetaReaderTest.java similarity index 94% rename from oap-server/generate-tool/src/test/java/org/apache/skywalking/oal/tool/meta/MetaReaderTest.java rename to oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/meta/MetaReaderTest.java index bb1ac4ce6..17f506d8d 100644 --- a/oap-server/generate-tool/src/test/java/org/apache/skywalking/oal/tool/meta/MetaReaderTest.java +++ b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/meta/MetaReaderTest.java @@ -16,11 +16,11 @@ * */ -package org.apache.skywalking.oal.tool.meta; +package org.apache.skywalking.oal.rt.meta; import java.io.InputStream; import java.util.List; -import org.apache.skywalking.oal.tool.parser.SourceColumn; +import org.apache.skywalking.oal.rt.parser.SourceColumn; import org.junit.*; public class MetaReaderTest { diff --git a/oap-server/generate-tool/src/test/java/org/apache/skywalking/oal/tool/meta/MockSourceColumnsFactory.java b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/meta/MockSourceColumnsFactory.java similarity index 98% rename from oap-server/generate-tool/src/test/java/org/apache/skywalking/oal/tool/meta/MockSourceColumnsFactory.java rename to oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/meta/MockSourceColumnsFactory.java index 593090791..3d3c4b58e 100644 --- a/oap-server/generate-tool/src/test/java/org/apache/skywalking/oal/tool/meta/MockSourceColumnsFactory.java +++ b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/meta/MockSourceColumnsFactory.java @@ -16,10 +16,10 @@ * */ -package org.apache.skywalking.oal.tool.meta; +package org.apache.skywalking.oal.rt.meta; import java.util.*; -import org.apache.skywalking.oal.tool.parser.SourceColumn; +import org.apache.skywalking.oal.rt.parser.SourceColumn; public class MockSourceColumnsFactory { public static List getColumns(String source) { diff --git a/oap-server/generate-tool/src/test/java/org/apache/skywalking/oal/tool/parser/DeepAnalysisTest.java b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/DeepAnalysisTest.java similarity index 85% rename from oap-server/generate-tool/src/test/java/org/apache/skywalking/oal/tool/parser/DeepAnalysisTest.java rename to oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/DeepAnalysisTest.java index 0334ae7ed..18b035ab3 100644 --- a/oap-server/generate-tool/src/test/java/org/apache/skywalking/oal/tool/parser/DeepAnalysisTest.java +++ b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/DeepAnalysisTest.java @@ -16,11 +16,11 @@ * */ -package org.apache.skywalking.oal.tool.parser; +package org.apache.skywalking.oal.rt.parser; import java.io.*; import java.util.List; -import org.apache.skywalking.oal.tool.meta.*; +import org.apache.skywalking.oal.rt.meta.*; import org.junit.*; public class DeepAnalysisTest { @@ -47,8 +47,8 @@ public class DeepAnalysisTest { EntryMethod method = result.getEntryMethod(); Assert.assertEquals("combine", method.getMethodName()); - Assert.assertEquals("source.getLatency()", method.getArgsExpressions().get(0)); - Assert.assertEquals("1", method.getArgsExpressions().get(1)); + Assert.assertEquals("(long)(source.getLatency())", method.getArgsExpressions().get(0)); + Assert.assertEquals("(long)(1)", method.getArgsExpressions().get(1)); List source = result.getFieldsFromSource(); Assert.assertEquals(1, source.size()); @@ -71,8 +71,8 @@ public class DeepAnalysisTest { EntryMethod method = result.getEntryMethod(); Assert.assertEquals("combine", method.getMethodName()); - Assert.assertEquals("source.getLatency()", method.getArgsExpressions().get(0)); - Assert.assertEquals("1", method.getArgsExpressions().get(1)); + Assert.assertEquals("(long)(source.getLatency())", method.getArgsExpressions().get(0)); + Assert.assertEquals("(long)(1)", method.getArgsExpressions().get(1)); List source = result.getFieldsFromSource(); Assert.assertEquals(3, source.size()); @@ -100,8 +100,8 @@ public class DeepAnalysisTest { EntryMethod method = result.getEntryMethod(); Assert.assertEquals("combine", method.getMethodName()); - Assert.assertEquals("source.getLatency()", method.getArgsExpressions().get(0)); - Assert.assertEquals("1", method.getArgsExpressions().get(1)); + Assert.assertEquals("(long)(source.getLatency())", method.getArgsExpressions().get(0)); + Assert.assertEquals("(long)(1)", method.getArgsExpressions().get(1)); List source = result.getFieldsFromSource(); Assert.assertEquals(3, source.size()); @@ -109,9 +109,9 @@ public class DeepAnalysisTest { List persistentFields = result.getPersistentFields(); Assert.assertEquals(4, persistentFields.size()); - List filterExpressions = result.getFilterExpressions(); + List filterExpressions = result.getFilterExpressions(); Assert.assertEquals(1, filterExpressions.size()); - FilterExpression filterExpression = filterExpressions.get(0); + Expression filterExpression = filterExpressions.get(0); Assert.assertEquals("EqualMatch", filterExpression.getExpressionObject()); Assert.assertEquals("source.getName()", filterExpression.getLeft()); Assert.assertEquals("\"/service/prod/save\"", filterExpression.getRight()); diff --git a/oap-server/generate-tool/src/test/java/org/apache/skywalking/oal/tool/parser/ScriptParserTest.java b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/ScriptParserTest.java similarity index 95% rename from oap-server/generate-tool/src/test/java/org/apache/skywalking/oal/tool/parser/ScriptParserTest.java rename to oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/ScriptParserTest.java index 59c384914..35464db03 100644 --- a/oap-server/generate-tool/src/test/java/org/apache/skywalking/oal/tool/parser/ScriptParserTest.java +++ b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/ScriptParserTest.java @@ -16,11 +16,11 @@ * */ -package org.apache.skywalking.oal.tool.parser; +package org.apache.skywalking.oal.rt.parser; import java.io.*; import java.util.List; -import org.apache.skywalking.oal.tool.meta.*; +import org.apache.skywalking.oal.rt.meta.*; import org.apache.skywalking.oap.server.core.annotation.AnnotationScan; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; import org.junit.*; @@ -36,7 +36,7 @@ public class ScriptParserTest { AnnotationScan scopeScan = new AnnotationScan(); scopeScan.registerListener(new DefaultScopeDefine.Listener()); - scopeScan.scan(null); + scopeScan.scan(); } @AfterClass @@ -80,10 +80,8 @@ public class ScriptParserTest { Assert.assertEquals("*", endpointPercent.getSourceAttribute()); Assert.assertEquals("percent", endpointPercent.getAggregationFunctionName()); EntryMethod entryMethod = endpointPercent.getEntryMethod(); - List methodArgsExpressions = entryMethod.getArgsExpressions(); - Assert.assertEquals(3, methodArgsExpressions.size()); - Assert.assertEquals("source.isStatus()", methodArgsExpressions.get(1)); - Assert.assertEquals("true", methodArgsExpressions.get(2)); + List methodArgsExpressions = entryMethod.getArgsExpressions(); + Assert.assertEquals(1, methodArgsExpressions.size()); } @Test diff --git a/oap-server/generate-tool/src/test/resources/expectedFiles/MetricsImplementorExpected.java b/oap-server/oal-rt/src/test/resources/expectedFiles/MetricsImplementorExpected.java similarity index 100% rename from oap-server/generate-tool/src/test/resources/expectedFiles/MetricsImplementorExpected.java rename to oap-server/oal-rt/src/test/resources/expectedFiles/MetricsImplementorExpected.java diff --git a/oap-server/generate-tool/src/test/resources/expectedFiles/ServiceDispatcherExpected.java b/oap-server/oal-rt/src/test/resources/expectedFiles/ServiceDispatcherExpected.java similarity index 100% rename from oap-server/generate-tool/src/test/resources/expectedFiles/ServiceDispatcherExpected.java rename to oap-server/oal-rt/src/test/resources/expectedFiles/ServiceDispatcherExpected.java diff --git a/oap-server/generate-tool/src/test/resources/oal_test.oal b/oap-server/oal-rt/src/test/resources/oal_test.oal similarity index 100% rename from oap-server/generate-tool/src/test/resources/oal_test.oal rename to oap-server/oal-rt/src/test/resources/oal_test.oal diff --git a/oap-server/generate-tool/src/test/resources/scope-meta.yml b/oap-server/oal-rt/src/test/resources/scope-meta.yml similarity index 100% rename from oap-server/generate-tool/src/test/resources/scope-meta.yml rename to oap-server/oal-rt/src/test/resources/scope-meta.yml diff --git a/oap-server/pom.xml b/oap-server/pom.xml index 8816919a9..b2e218696 100644 --- a/oap-server/pom.xml +++ b/oap-server/pom.xml @@ -37,10 +37,9 @@ server-query-plugin server-alarm-plugin server-testing - generated-analysis - generate-tool + oal-rt server-telemetry - generate-tool-grammar + oal-grammar exporter server-configuration @@ -83,6 +82,9 @@ 2.12.0 2.17.0 v3.2.3 + 4.7.1 + 2.3.28 + 3.25.0-GA 3.5 @@ -408,6 +410,22 @@ test + + + org.antlr + antlr4 + ${antlr.version} + + + org.freemarker + freemarker + ${freemarker.version} + + + org.javassist + javassist + ${javaassist.version} + diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java index 5c60cd334..2c8f78a0f 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java @@ -25,6 +25,8 @@ import org.apache.skywalking.oap.server.core.annotation.AnnotationScan; import org.apache.skywalking.oap.server.core.cache.*; import org.apache.skywalking.oap.server.core.cluster.*; import org.apache.skywalking.oap.server.core.config.*; +import org.apache.skywalking.oap.server.core.oal.rt.OALEngine; +import org.apache.skywalking.oap.server.core.oal.rt.OALEngineLoader; import org.apache.skywalking.oap.server.core.query.*; import org.apache.skywalking.oap.server.core.register.service.*; import org.apache.skywalking.oap.server.core.remote.*; @@ -56,6 +58,8 @@ public class CoreModuleProvider extends ModuleProvider { private final StorageModels storageModels; private final StreamDataMapping streamDataMapping; private final SourceReceiverImpl receiver; + private StreamAnnotationListener streamAnnotationListener; + private OALEngine oalEngine; public CoreModuleProvider() { super(); @@ -79,12 +83,26 @@ public class CoreModuleProvider extends ModuleProvider { } @Override public void prepare() throws ServiceNotProvidedException, ModuleStartException { + streamAnnotationListener = new StreamAnnotationListener(getManager()); + AnnotationScan scopeScan = new AnnotationScan(); scopeScan.registerListener(new DefaultScopeDefine.Listener()); - scopeScan.registerListener(DisableRegister.INSTANCE); - scopeScan.registerListener(new DisableRegister.SingleDisableScanListener()); try { - scopeScan.scan(null); + scopeScan.scan(); + + oalEngine = OALEngineLoader.get(); + oalEngine.setStreamListener(streamAnnotationListener); + oalEngine.setDispatcherListener(receiver.getDispatcherManager()); + oalEngine.start(getClass().getClassLoader()); + } catch (Exception e) { + throw new ModuleStartException(e.getMessage(), e); + } + + AnnotationScan oalDisable = new AnnotationScan(); + oalDisable.registerListener(DisableRegister.INSTANCE); + oalDisable.registerListener(new DisableRegister.SingleDisableScanListener()); + try { + oalDisable.scan(); } catch (IOException e) { throw new ModuleStartException(e.getMessage(), e); } @@ -144,7 +162,7 @@ public class CoreModuleProvider extends ModuleProvider { this.registerServiceImplementation(AlarmQueryService.class, new AlarmQueryService(getManager())); this.registerServiceImplementation(TopNRecordsQueryService.class, new TopNRecordsQueryService(getManager())); - annotationScan.registerListener(new StreamAnnotationListener(getManager())); + annotationScan.registerListener(streamAnnotationListener); this.remoteClientManager = new RemoteClientManager(getManager()); this.registerServiceImplementation(RemoteClientManager.class, remoteClientManager); @@ -157,9 +175,10 @@ public class CoreModuleProvider extends ModuleProvider { try { receiver.scan(); + annotationScan.scan(); + + oalEngine.notifyAllListeners(); - annotationScan.scan(() -> { - }); streamDataMapping.init(); } catch (IOException | IllegalAccessException | InstantiationException e) { throw new ModuleStartException(e.getMessage(), e); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/WorkPath.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/WorkPath.java new file mode 100644 index 000000000..d92d2eedd --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/WorkPath.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core; + +import java.io.File; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Locate the base work path of OAP backend. + * + * @author wusheng + */ +public class WorkPath { + private static final Logger logger = LoggerFactory.getLogger(WorkPath.class); + + private static File PATH; + + public static File getPath() { + if (PATH == null) { + PATH = findPath(); + } + return PATH; + } + + private static File findPath() { + String classResourcePath = WorkPath.class.getName().replaceAll("\\.", "/") + ".class"; + + URL resource = ClassLoader.getSystemClassLoader().getResource(classResourcePath); + if (resource != null) { + String urlString = resource.toString(); + + logger.debug("The beacon class location is {}.", urlString); + + int insidePathIndex = urlString.indexOf('!'); + boolean isInJar = insidePathIndex > -1; + + if (isInJar) { + urlString = urlString.substring(urlString.indexOf("file:"), insidePathIndex); + File agentJarFile = null; + try { + agentJarFile = new File(new URL(urlString).toURI()); + } catch (MalformedURLException e) { + throw new UnexpectedException("Can not locate oap core jar file by url:" + urlString, e); + } catch (URISyntaxException e) { + throw new UnexpectedException("Can not locate oap core jar file by url:" + urlString, e); + } + if (agentJarFile.exists()) { + return agentJarFile.getParentFile(); + } + } else { + int prefixLength = "file:".length(); + String classLocation = urlString.substring(prefixLength, urlString.length() - classResourcePath.length()); + return new File(classLocation); + } + } + + throw new UnexpectedException("Can not locate oap core jar file by path:" + classResourcePath); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/DisableRegister.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/DisableRegister.java index d90462e08..b87dd79fd 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/DisableRegister.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/DisableRegister.java @@ -43,11 +43,15 @@ public class DisableRegister implements AnnotationListener { Disable[] valueList = annotation.value(); if (valueList != null) { for (Disable disable : valueList) { - disableEntitySet.add(disable.value()); + add(disable.value()); } } } + public void add(String name) { + disableEntitySet.add(name); + } + public boolean include(String name) { return disableEntitySet.contains(name); } diff --git a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/output/WriteWrapper.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/DispatcherDetectorListener.java similarity index 77% rename from oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/output/WriteWrapper.java rename to oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/DispatcherDetectorListener.java index d212040bb..0c29033e1 100644 --- a/oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/output/WriteWrapper.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/DispatcherDetectorListener.java @@ -16,11 +16,11 @@ * */ -package org.apache.skywalking.oal.tool.output; +package org.apache.skywalking.oap.server.core.analysis; -import freemarker.template.TemplateException; -import java.io.*; - -public interface WriteWrapper { - void execute(FileWriter fileWriter) throws IOException, TemplateException; +/** + * @author wusheng + */ +public interface DispatcherDetectorListener { + void addIfAsSourceDispatcher(Class aClass) throws IllegalAccessException, InstantiationException; } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/DispatcherManager.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/DispatcherManager.java index f84ee9db1..48356de4f 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/DispatcherManager.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/DispatcherManager.java @@ -21,16 +21,21 @@ package org.apache.skywalking.oap.server.core.analysis; import com.google.common.collect.ImmutableSet; import com.google.common.reflect.ClassPath; import java.io.IOException; -import java.lang.reflect.*; -import java.util.*; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.skywalking.oap.server.core.UnexpectedException; import org.apache.skywalking.oap.server.core.source.Source; -import org.slf4j.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @author peng-yongsheng, wusheng */ -public class DispatcherManager { +public class DispatcherManager implements DispatcherDetectorListener { private static final Logger logger = LoggerFactory.getLogger(DispatcherManager.class); @@ -75,39 +80,44 @@ public class DispatcherManager { for (ClassPath.ClassInfo classInfo : classes) { Class aClass = classInfo.load(); - if (!aClass.isInterface() && SourceDispatcher.class.isAssignableFrom(aClass)) { - Type[] genericInterfaces = aClass.getGenericInterfaces(); - for (Type genericInterface : genericInterfaces) { - ParameterizedType anInterface = (ParameterizedType)genericInterface; - if (anInterface.getRawType().getTypeName().equals(SourceDispatcher.class.getName())) { - Type[] arguments = anInterface.getActualTypeArguments(); + addIfAsSourceDispatcher(aClass); + } + } - if (arguments.length != 1) { - throw new UnexpectedException("unexpected type argument number, class " + aClass.getName()); - } - Type argument = arguments[0]; + @Override + public void addIfAsSourceDispatcher(Class aClass) throws IllegalAccessException, InstantiationException { + if (!aClass.isInterface() && SourceDispatcher.class.isAssignableFrom(aClass)) { + Type[] genericInterfaces = aClass.getGenericInterfaces(); + for (Type genericInterface : genericInterfaces) { + ParameterizedType anInterface = (ParameterizedType)genericInterface; + if (anInterface.getRawType().getTypeName().equals(SourceDispatcher.class.getName())) { + Type[] arguments = anInterface.getActualTypeArguments(); - Object source = ((Class)argument).newInstance(); - - if (!Source.class.isAssignableFrom(source.getClass())) { - throw new UnexpectedException("unexpected type argument of class " + aClass.getName() + ", should be `org.apache.skywalking.oap.server.core.source.Source`. "); - } - - Source dispatcherSource = (Source)source; - SourceDispatcher dispatcher = (SourceDispatcher)aClass.newInstance(); - - int scopeId = dispatcherSource.scope(); - - List dispatchers = this.dispatcherMap.get(scopeId); - if (dispatchers == null) { - dispatchers = new ArrayList<>(); - this.dispatcherMap.put(scopeId, dispatchers); - } - - dispatchers.add(dispatcher); - - logger.info("Dispatcher {} is added into DefaultScopeDefine {}.", dispatcher.getClass().getName(), scopeId); + if (arguments.length != 1) { + throw new UnexpectedException("unexpected type argument number, class " + aClass.getName()); } + Type argument = arguments[0]; + + Object source = ((Class)argument).newInstance(); + + if (!Source.class.isAssignableFrom(source.getClass())) { + throw new UnexpectedException("unexpected type argument of class " + aClass.getName() + ", should be `org.apache.skywalking.oap.server.core.source.Source`. "); + } + + Source dispatcherSource = (Source)source; + SourceDispatcher dispatcher = (SourceDispatcher)aClass.newInstance(); + + int scopeId = dispatcherSource.scope(); + + List dispatchers = this.dispatcherMap.get(scopeId); + if (dispatchers == null) { + dispatchers = new ArrayList<>(); + this.dispatcherMap.put(scopeId, dispatchers); + } + + dispatchers.add(dispatcher); + + logger.info("Dispatcher {} is added into DefaultScopeDefine {}.", dispatcher.getClass().getName(), scopeId); } } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DoubleAvgMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DoubleAvgMetrics.java index 9e7080082..850c58093 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DoubleAvgMetrics.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DoubleAvgMetrics.java @@ -34,11 +34,11 @@ public abstract class DoubleAvgMetrics extends Metrics implements DoubleValueHol protected static final String VALUE = "value"; @Getter @Setter @Column(columnName = SUMMATION) private double summation; - @Getter @Setter @Column(columnName = COUNT) private int count; + @Getter @Setter @Column(columnName = COUNT) private long count; @Getter @Setter @Column(columnName = VALUE, isValue = true, function = Function.Avg) private double value; @Entrance - public final void combine(@SourceFrom double summation, @ConstOne int count) { + public final void combine(@SourceFrom double summation, @ConstOne long count) { this.summation += summation; this.count += count; } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/LongAvgMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/LongAvgMetrics.java index fce1e58b9..a0d5c3a61 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/LongAvgMetrics.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/LongAvgMetrics.java @@ -34,11 +34,11 @@ public abstract class LongAvgMetrics extends Metrics implements LongValueHolder protected static final String VALUE = "value"; @Getter @Setter @Column(columnName = SUMMATION) private long summation; - @Getter @Setter @Column(columnName = COUNT) private int count; + @Getter @Setter @Column(columnName = COUNT) private long count; @Getter @Setter @Column(columnName = VALUE, isValue = true, function = Function.Avg) private long value; @Entrance - public final void combine(@SourceFrom long summation, @ConstOne int count) { + public final void combine(@SourceFrom long summation, @ConstOne long count) { this.summation += summation; this.count += count; } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentMetrics.java index 1224d92ac..e88539ce8 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentMetrics.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentMetrics.java @@ -18,9 +18,11 @@ package org.apache.skywalking.oap.server.core.analysis.metrics; -import lombok.*; -import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.*; -import org.apache.skywalking.oap.server.core.analysis.metrics.expression.EqualMatch; +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.Entrance; +import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.Expression; +import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.MetricsFunction; import org.apache.skywalking.oap.server.core.query.sql.Function; import org.apache.skywalking.oap.server.core.storage.annotation.Column; @@ -38,11 +40,8 @@ public abstract class PercentMetrics extends Metrics implements IntValueHolder { @Getter @Setter @Column(columnName = MATCH) private long match; @Entrance - public final void combine(@Expression EqualMatch expression, @ExpressionArg0 Object leftValue, - @ExpressionArg1 Object rightValue) { - expression.setLeft(leftValue); - expression.setRight(rightValue); - if (expression.match()) { + public final void combine(@Expression boolean isMatch) { + if (isMatch) { match++; } total++; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/annotation/ExpressionArg1.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/annotation/ExpressionArg1.java deleted file mode 100644 index 364ac7e1c..000000000 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/annotation/ExpressionArg1.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oap.server.core.analysis.metrics.annotation; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** - * @author wusheng - */ -@Target(ElementType.PARAMETER) -@Retention(RetentionPolicy.RUNTIME) -public @interface ExpressionArg1 { -} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/EqualMatch.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/EqualMatch.java index f5916f6b7..c40783ecf 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/EqualMatch.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/EqualMatch.java @@ -19,12 +19,50 @@ package org.apache.skywalking.oap.server.core.analysis.metrics.expression; /** - * - * * @author wusheng */ -public class EqualMatch extends BinaryMatchExpression { - @Override public boolean match() { +public class EqualMatch { + public boolean match(int left, int right) { + return left == right; + } + + public boolean match(long left, long right) { + return left == right; + } + + public boolean match(float left, float right) { + return left == right; + } + + public boolean match(double left, double right) { + return left == right; + } + + public boolean match(Integer left, Integer right) { + return left == right; + } + + public boolean match(Long left, Long right) { + return left == right; + } + + public boolean match(Float left, Float right) { + return left == right; + } + + public boolean match(Double left, Double right) { + return left == right; + } + + public boolean match(Boolean left, Boolean right) { + return left == right; + } + + public boolean match(boolean left, boolean right) { + return left == right; + } + + public boolean match(Object left, Object right) { return left.equals(right); } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/GreaterEqualMatch.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/GreaterEqualMatch.java index f26aac78c..7c8f45561 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/GreaterEqualMatch.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/GreaterEqualMatch.java @@ -22,6 +22,21 @@ package org.apache.skywalking.oap.server.core.analysis.metrics.expression; * @author peng-yongsheng */ public class GreaterEqualMatch { + public boolean match(int left, int right) { + return left >= right; + } + + public boolean match(long left, long right) { + return left >= right; + } + + public boolean match(float left, float right) { + return left >= right; + } + + public boolean match(double left, double right) { + return left >= right; + } public boolean match(Integer left, Integer right) { return left >= right; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/GreaterMatch.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/GreaterMatch.java index 87b06c54b..7a82dc0fe 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/GreaterMatch.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/GreaterMatch.java @@ -22,6 +22,21 @@ package org.apache.skywalking.oap.server.core.analysis.metrics.expression; * @author peng-yongsheng */ public class GreaterMatch { + public boolean match(int left, int right) { + return left > right; + } + + public boolean match(long left, long right) { + return left > right; + } + + public boolean match(float left, float right) { + return left > right; + } + + public boolean match(double left, double right) { + return left > right; + } public boolean match(Integer left, Integer right) { return left > right; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/LessEqualMatch.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/LessEqualMatch.java index 00f64f1f9..4590dab5d 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/LessEqualMatch.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/LessEqualMatch.java @@ -22,6 +22,21 @@ package org.apache.skywalking.oap.server.core.analysis.metrics.expression; * @author peng-yongsheng */ public class LessEqualMatch { + public boolean match(int left, int right) { + return left <= right; + } + + public boolean match(long left, long right) { + return left <= right; + } + + public boolean match(float left, float right) { + return left <= right; + } + + public boolean match(double left, double right) { + return left <= right; + } public boolean match(Integer left, Integer right) { return left <= right; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/LessMatch.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/LessMatch.java index 5583def78..924d2fbb3 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/LessMatch.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/LessMatch.java @@ -22,6 +22,21 @@ package org.apache.skywalking.oap.server.core.analysis.metrics.expression; * @author peng-yongsheng */ public class LessMatch { + public boolean match(int left, int right) { + return left < right; + } + + public boolean match(long left, long right) { + return left < right; + } + + public boolean match(float left, float right) { + return left < right; + } + + public boolean match(double left, double right) { + return left < right; + } public boolean match(Integer left, Integer right) { return left < right; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/annotation/AnnotationScan.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/annotation/AnnotationScan.java index 7b8aea837..da68bc5dd 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/annotation/AnnotationScan.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/annotation/AnnotationScan.java @@ -25,7 +25,9 @@ import java.lang.annotation.Annotation; import java.util.*; /** - * @author peng-yongsheng + * Scan the annotation, and notify the listener(s) + * + * @author peng-yongsheng, wusheng */ public class AnnotationScan { @@ -35,11 +37,20 @@ public class AnnotationScan { this.listeners = new LinkedList<>(); } + /** + * Register the callback listener + * @param listener to be called after class found w/ annotation + */ public void registerListener(AnnotationListener listener) { listeners.add(new AnnotationListenerCache(listener)); } - public void scan(Runnable callBack) throws IOException { + /** + * Begin to scan classes. + * + * @throws IOException + */ + public void scan() throws IOException { ClassPath classpath = ClassPath.from(this.getClass().getClassLoader()); ImmutableSet classes = classpath.getTopLevelClassesRecursive("org.apache.skywalking"); for (ClassPath.ClassInfo classInfo : classes) { @@ -53,10 +64,6 @@ public class AnnotationScan { } listeners.forEach(AnnotationListenerCache::complete); - - if (callBack != null) { - callBack.run(); - } } private class AnnotationListenerCache { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/annotation/ExpressionArg0.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/oal/rt/OALCompileException.java similarity index 71% rename from oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/annotation/ExpressionArg0.java rename to oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/oal/rt/OALCompileException.java index 1e4384f3b..b72321e24 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/annotation/ExpressionArg0.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/oal/rt/OALCompileException.java @@ -16,17 +16,17 @@ * */ -package org.apache.skywalking.oap.server.core.analysis.metrics.annotation; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; +package org.apache.skywalking.oap.server.core.oal.rt; /** * @author wusheng */ -@Target(ElementType.PARAMETER) -@Retention(RetentionPolicy.RUNTIME) -public @interface ExpressionArg0 { -} +public class OALCompileException extends Exception { + public OALCompileException(String message) { + super(message); + } + + public OALCompileException(String message, Throwable cause) { + super(message, cause); + } +} \ No newline at end of file diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/BinaryMatchExpression.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/oal/rt/OALEngine.java similarity index 53% rename from oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/BinaryMatchExpression.java rename to oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/oal/rt/OALEngine.java index db7422a14..7eaf81437 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/BinaryMatchExpression.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/oal/rt/OALEngine.java @@ -16,27 +16,23 @@ * */ -package org.apache.skywalking.oap.server.core.analysis.metrics.expression; +package org.apache.skywalking.oap.server.core.oal.rt; + +import org.apache.skywalking.oap.server.core.analysis.DispatcherDetectorListener; +import org.apache.skywalking.oap.server.core.analysis.StreamAnnotationListener; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; /** - * BinaryMatchExpression accepts two calculate factors, - * and return the True/False result. + * OALEngine defines the main entrance of the oal script engine runtime. * * @author wusheng */ -public abstract class BinaryMatchExpression { - protected Object left; - protected Object right; +public interface OALEngine { + void setStreamListener(StreamAnnotationListener listener) throws ModuleStartException; - public BinaryMatchExpression setLeft(Object left) { - this.left = left; - return this; - } + void setDispatcherListener(DispatcherDetectorListener listener) throws ModuleStartException; - public BinaryMatchExpression setRight(Object right) { - this.right = right; - return this; - } + void start(ClassLoader currentClassLoader) throws ModuleStartException, OALCompileException; - public abstract boolean match(); + void notifyAllListeners() throws ModuleStartException; } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/oal/rt/OALEngineLoader.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/oal/rt/OALEngineLoader.java new file mode 100644 index 000000000..a92163541 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/oal/rt/OALEngineLoader.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.oal.rt; + +import java.util.concurrent.locks.ReentrantLock; + +/** + * Load the OAL Engine runtime, because runtime module depends on core, so we have to use class::forname to locate it. + * + * @author wusheng + */ +public class OALEngineLoader { + private static volatile OALEngine ENGINE = null; + private static ReentrantLock INIT_LOCK = new ReentrantLock(); + + public static OALEngine get() throws ClassNotFoundException, InstantiationException, IllegalAccessException { + if (ENGINE == null) { + INIT_LOCK.lock(); + try { + if (ENGINE == null) { + init(); + } + } finally { + INIT_LOCK.unlock(); + } + } + return ENGINE; + } + + private static void init() throws ClassNotFoundException, IllegalAccessException, InstantiationException { + Class engineRTClass = Class.forName("org.apache.skywalking.oal.rt.OALRuntime"); + ENGINE = (OALEngine)engineRTClass.newInstance(); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java index e5d5cb1f2..8d83b1644 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java @@ -64,7 +64,7 @@ public class DefaultScopeDefine { public static final int HTTP_ACCESS_LOG = 25; /** - * Catalog of scope, the metrics processor could use this to group all generated metrics by oal tool. + * Catalog of scope, the metrics processor could use this to group all generated metrics by oal rt. */ public static final String SERVICE_CATALOG_NAME = "SERVICE"; public static final String SERVICE_INSTANCE_CATALOG_NAME = "SERVICE_INSTANCE"; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/SourceReceiverImpl.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/SourceReceiverImpl.java index 4c48efdfb..425e7b527 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/SourceReceiverImpl.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/SourceReceiverImpl.java @@ -19,13 +19,14 @@ package org.apache.skywalking.oap.server.core.source; import java.io.IOException; +import lombok.Getter; import org.apache.skywalking.oap.server.core.analysis.DispatcherManager; /** * @author peng-yongsheng */ public class SourceReceiverImpl implements SourceReceiver { - + @Getter private final DispatcherManager dispatcherManager; public SourceReceiverImpl() { diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentMetricsTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentMetricsTest.java index b5bb0f086..9e976dd10 100644 --- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentMetricsTest.java +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentMetricsTest.java @@ -30,18 +30,18 @@ public class PercentMetricsTest { @Test public void testEntranceCombine() { PercentMetricsImpl impl = new PercentMetricsImpl(); - impl.combine(new EqualMatch(), true, true); - impl.combine(new EqualMatch(), true, false); - impl.combine(new EqualMatch(), true, false); + impl.combine(new EqualMatch().match(true,true)); + impl.combine(new EqualMatch().match(true,false)); + impl.combine(new EqualMatch().match(true,false)); impl.calculate(); Assert.assertEquals(3333, impl.getValue()); impl = new PercentMetricsImpl(); - impl.combine(new EqualMatch(), true, true); - impl.combine(new EqualMatch(), true, true); - impl.combine(new EqualMatch(), true, false); + impl.combine(new EqualMatch().match(true,true)); + impl.combine(new EqualMatch().match(true,true)); + impl.combine(new EqualMatch().match(true,false)); impl.calculate(); @@ -51,14 +51,14 @@ public class PercentMetricsTest { @Test public void testSelfCombine() { PercentMetricsImpl impl = new PercentMetricsImpl(); - impl.combine(new EqualMatch(), true, true); - impl.combine(new EqualMatch(), true, false); - impl.combine(new EqualMatch(), true, false); + impl.combine(new EqualMatch().match(true,true)); + impl.combine(new EqualMatch().match(true,false)); + impl.combine(new EqualMatch().match(true,false)); PercentMetricsImpl impl2 = new PercentMetricsImpl(); - impl2.combine(new EqualMatch(), true, true); - impl2.combine(new EqualMatch(), true, true); - impl2.combine(new EqualMatch(), true, false); + impl2.combine(new EqualMatch().match(true,true)); + impl2.combine(new EqualMatch().match(true,true)); + impl2.combine(new EqualMatch().match(true,false)); impl.combine(impl2); diff --git a/oap-server/server-starter/pom.xml b/oap-server/server-starter/pom.xml index 89a80369d..761c37709 100644 --- a/oap-server/server-starter/pom.xml +++ b/oap-server/server-starter/pom.xml @@ -35,19 +35,13 @@ ${project.version} - + org.apache.skywalking - generated-analysis + oal-rt ${project.version} - - - org.apache.skywalking - generate-tool - - - + @@ -230,6 +224,8 @@ component-libraries.yml datasource-settings.properties endpoint_naming_rules.properties + official_analysis.oal + scope-meta.yml diff --git a/oap-server/server-starter/src/main/assembly/assembly.xml b/oap-server/server-starter/src/main/assembly/assembly.xml index 9f73bfe76..e2c30a77f 100644 --- a/oap-server/server-starter/src/main/assembly/assembly.xml +++ b/oap-server/server-starter/src/main/assembly/assembly.xml @@ -46,15 +46,10 @@ src/main/resources component-libraries.yml + official_analysis.oal + scope-meta.yml /config - - ${project.basedir}/../apm-collector-instrument/target - - collector-instrument-agent.jar - - /agent - diff --git a/oap-server/generated-analysis/src/main/resources/official_analysis.oal b/oap-server/server-starter/src/main/resources/official_analysis.oal similarity index 100% rename from oap-server/generated-analysis/src/main/resources/official_analysis.oal rename to oap-server/server-starter/src/main/resources/official_analysis.oal diff --git a/oap-server/generated-analysis/src/main/resources/generator-scope-meta.yml b/oap-server/server-starter/src/main/resources/scope-meta.yml similarity index 100% rename from oap-server/generated-analysis/src/main/resources/generator-scope-meta.yml rename to oap-server/server-starter/src/main/resources/scope-meta.yml diff --git a/pom.xml b/pom.xml index 811866e04..9e68add64 100644 --- a/pom.xml +++ b/pom.xml @@ -416,8 +416,9 @@ **/src/main/proto/protoc-gen-swagger/** **/src/main/proto/validate/validate.proto - - **/src/main/antlr4/org/apache/skywalking/oal/tool/grammar/OALLexer.tokens + + **/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.tokens + **/src/main/resources/code-templates/*/*.ftl .mvn/wrapper/maven-wrapper.properties @@ -510,7 +511,7 @@ org/apache/skywalking/apm/network/servicemesh/**/*.java, org/apache/skywalking/apm/network/language/**/*.java, org/apache/skywalking/oap/server/core/remote/grpc/proto/*.java, - org/apache/skywalking/oal/tool/grammar/*.java, + org/apache/skywalking/oal/rt/grammar/*.java, org/apache/skywalking/oap/server/exporter/grpc/*.java, org/apache/skywalking/oap/server/configuration/service/*.java