kazusa
/
skywalking
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move `metrics-baseline` into `ai-pipeline` module (#13041)

This commit is contained in:
mrproliu 2025-02-18 20:16:14 +08:00 committed by GitHub
parent 68ab2edd77
commit 7785c4164b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
38 changed files with 164 additions and 369 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/en/api/metrics-query-expression.md
View File

@ -511,7 +511,7 @@ baseline(service_resp_time, upper)
```
**Notice**:
- This feature is required to enable the `baseline module` and deploy a baseline service. And the baseline service should implement the protocol of the [baseline.proto](../../../oap-server/metrics-baseline/src/main/proto/baseline.proto).
- This feature is required to enable the `baseline module` and deploy a baseline service. And the baseline service should implement the protocol of the [baseline.proto](../../../oap-server/ai-pipeline/src/main/proto/baseline.proto).
Otherwise, the result will be empty.
- The baseline operation requires the relative metrics declared through baseline service.
Otherwise, the result will be empty, which means there is no baseline or predicated value.

1
docs/en/changes/changes.md
View File

@ -15,6 +15,7 @@
production-ready. Don't need H2 as default storage anymore.
* [Breaking Change] Bump up BanyanDB server version to 0.8.0. This version is not compatible with the previous
versions. Please upgrade the BanyanDB server to 0.8.0 before upgrading OAP to 10.2.0.
* Move `metrics-baseline` module into `ai-pipeline` module.
#### OAP Server

17
oap-server/ai-pipeline/pom.xml
View File

@ -62,6 +62,21 @@
<artifactId>library-client</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.skywalking</groupId>
<artifactId>server-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-testing</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-reflect</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
@ -95,4 +110,4 @@
</plugin>
</plugins>
</build>
</project>
</project>

2
oap-server/ai-pipeline/src/main/java/org/apache/skywalking/oap/server/ai/pipeline/AIPipelineConfig.java
View File

@ -27,4 +27,6 @@ import org.apache.skywalking.oap.server.library.module.ModuleConfig;
public class AIPipelineConfig extends ModuleConfig {
private String uriRecognitionServerAddr;
private int uriRecognitionServerPort = 17128;
private String baselineServerAddr;
private int baselineServerPort = 18080;
}

6
oap-server/ai-pipeline/src/main/java/org/apache/skywalking/oap/server/ai/pipeline/AIPipelineModule.java
View File

@ -18,7 +18,7 @@
package org.apache.skywalking.oap.server.ai.pipeline;
import org.apache.skywalking.oap.server.ai.pipeline.services.api.HttpUriRecognition;
import org.apache.skywalking.oap.server.ai.pipeline.services.BaselineQueryService;
import org.apache.skywalking.oap.server.library.module.ModuleDefine;
public class AIPipelineModule extends ModuleDefine {
@ -30,8 +30,6 @@ public class AIPipelineModule extends ModuleDefine {
@Override
public Class[] services() {
return new Class[]{
HttpUriRecognition.class
};
return new Class[]{BaselineQueryService.class};
}
}

27
oap-server/ai-pipeline/src/main/java/org/apache/skywalking/oap/server/ai/pipeline/AIPipelineProvider.java
View File

@ -18,8 +18,11 @@
package org.apache.skywalking.oap.server.ai.pipeline;
import org.apache.skywalking.oap.server.ai.pipeline.services.BaselineQueryService;
import org.apache.skywalking.oap.server.ai.pipeline.services.BaselineQueryServiceImpl;
import org.apache.skywalking.oap.server.ai.pipeline.services.HttpUriRecognitionService;
import org.apache.skywalking.oap.server.ai.pipeline.services.api.HttpUriRecognition;
import org.apache.skywalking.oap.server.core.CoreModule;
import org.apache.skywalking.oap.server.core.config.group.EndpointNameGroupService;
import org.apache.skywalking.oap.server.library.module.ModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleDefine;
import org.apache.skywalking.oap.server.library.module.ModuleProvider;
@ -56,25 +59,31 @@ public class AIPipelineProvider extends ModuleProvider {
@Override
public void prepare() throws ServiceNotProvidedException, ModuleStartException {
final HttpUriRecognitionService httpUriRecognitionService = new HttpUriRecognitionService(
aiPipelineConfig.getUriRecognitionServerAddr(),
aiPipelineConfig.getUriRecognitionServerPort()
);
this.registerServiceImplementation(HttpUriRecognition.class, httpUriRecognitionService);
this.registerServiceImplementation(BaselineQueryService.class, new BaselineQueryServiceImpl(
aiPipelineConfig.getBaselineServerAddr(),
aiPipelineConfig.getBaselineServerPort()
));
}
@Override
public void start() throws ServiceNotProvidedException, ModuleStartException {
final HttpUriRecognitionService httpUriRecognitionService = new HttpUriRecognitionService(
aiPipelineConfig.getUriRecognitionServerAddr(),
aiPipelineConfig.getUriRecognitionServerPort()
);
getManager().find(CoreModule.NAME).provider()
.getService(EndpointNameGroupService.class)
.startHttpUriRecognitionSvr(httpUriRecognitionService);
}
@Override
public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException {
}
@Override
public String[] requiredModules() {
return new String[0];
return new String[] {
CoreModule.NAME
};
}
}

2
oap-server/metrics-baseline/src/main/java/org/apache/skywalking/oap/server/baseline/service/BaselineQueryService.java → oap-server/ai-pipeline/src/main/java/org/apache/skywalking/oap/server/ai/pipeline/services/BaselineQueryService.java
View File

@ -16,7 +16,7 @@
*
*/
package org.apache.skywalking.oap.server.baseline.service;
package org.apache.skywalking.oap.server.ai.pipeline.services;
import java.util.Map;
import org.apache.skywalking.oap.server.library.module.Service;

2
oap-server/metrics-baseline/src/main/java/org/apache/skywalking/oap/server/baseline/service/BaselineQueryServiceImpl.java → oap-server/ai-pipeline/src/main/java/org/apache/skywalking/oap/server/ai/pipeline/services/BaselineQueryServiceImpl.java
View File

@ -16,7 +16,7 @@
*
*/
package org.apache.skywalking.oap.server.baseline.service;
package org.apache.skywalking.oap.server.ai.pipeline.services;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;

4
oap-server/ai-pipeline/src/main/java/org/apache/skywalking/oap/server/ai/pipeline/services/HttpUriRecognitionService.java
View File

@ -28,8 +28,8 @@ import org.apache.skywalking.oap.server.ai.pipeline.grpc.HttpUriRecognitionReque
import org.apache.skywalking.oap.server.ai.pipeline.grpc.HttpUriRecognitionResponse;
import org.apache.skywalking.oap.server.ai.pipeline.grpc.HttpUriRecognitionServiceGrpc;
import org.apache.skywalking.oap.server.ai.pipeline.grpc.HttpUriRecognitionSyncRequest;
import org.apache.skywalking.oap.server.ai.pipeline.services.api.HttpUriPattern;
import org.apache.skywalking.oap.server.ai.pipeline.services.api.HttpUriRecognition;
import org.apache.skywalking.oap.server.core.config.group.ai.HttpUriPattern;
import org.apache.skywalking.oap.server.core.config.group.ai.HttpUriRecognition;
import org.apache.skywalking.oap.server.library.client.grpc.GRPCClient;
import org.apache.skywalking.oap.server.library.util.StringUtil;

2
oap-server/metrics-baseline/src/main/java/org/apache/skywalking/oap/server/baseline/service/PredictServiceMetrics.java → oap-server/ai-pipeline/src/main/java/org/apache/skywalking/oap/server/ai/pipeline/services/PredictServiceMetrics.java
View File

@ -16,7 +16,7 @@
*
*/
package org.apache.skywalking.oap.server.baseline.service;
package org.apache.skywalking.oap.server.ai.pipeline.services;
import lombok.Builder;
import lombok.Data;

2
oap-server/metrics-baseline/src/main/java/org/apache/skywalking/oap/server/baseline/service/ServiceMetrics.java → oap-server/ai-pipeline/src/main/java/org/apache/skywalking/oap/server/ai/pipeline/services/ServiceMetrics.java
View File

@ -16,7 +16,7 @@
*
*/
package org.apache.skywalking.oap.server.baseline.service;
package org.apache.skywalking.oap.server.ai.pipeline.services;
import lombok.Builder;
import lombok.Data;

0
oap-server/metrics-baseline/src/main/proto/baseline.proto → oap-server/ai-pipeline/src/main/proto/baseline.proto
View File

10
oap-server/metrics-baseline/src/test/java/org/apache/skywalking/oap/server/baseline/BaselineQueryServer.java → oap-server/ai-pipeline/src/test/java/BaselineQueryServer.java
View File

@ -16,19 +16,17 @@
*
*/
package org.apache.skywalking.oap.server.baseline;
import com.google.protobuf.Empty;
import io.grpc.stub.StreamObserver;
import lombok.Builder;
import lombok.Data;
import org.apache.skywalking.apm.baseline.v3.AlarmBaselineMetricsNames;
import org.apache.skywalking.oap.server.core.analysis.DownSampling;
import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
import org.apache.skywalking.apm.baseline.v3.AlarmBaselineServiceGrpc;
import org.apache.skywalking.apm.baseline.v3.AlarmBaselineLabeledValue;
import org.apache.skywalking.apm.baseline.v3.AlarmBaselineMetricsNames;
import org.apache.skywalking.apm.baseline.v3.AlarmBaselineServiceGrpc;
import org.apache.skywalking.apm.baseline.v3.AlarmBaselineValue;
import org.apache.skywalking.apm.baseline.v3.KeyStringValuePair;
import org.apache.skywalking.oap.server.core.analysis.DownSampling;
import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
import java.util.ArrayList;
import java.util.Calendar;

8
oap-server/metrics-baseline/src/test/java/org/apache/skywalking/oap/server/baseline/BaselineServerTest.java → oap-server/ai-pipeline/src/test/java/BaselineServerTest.java
View File

@ -16,17 +16,15 @@
*
*/
package org.apache.skywalking.oap.server.baseline;
import io.grpc.ManagedChannel;
import io.grpc.Server;
import io.grpc.inprocess.InProcessChannelBuilder;
import io.grpc.inprocess.InProcessServerBuilder;
import io.grpc.util.MutableHandlerRegistry;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.oap.server.baseline.service.BaselineQueryServiceImpl;
import org.apache.skywalking.oap.server.baseline.service.PredictServiceMetrics;
import org.apache.skywalking.oap.server.baseline.service.ServiceMetrics;
import org.apache.skywalking.oap.server.ai.pipeline.services.BaselineQueryServiceImpl;
import org.apache.skywalking.oap.server.ai.pipeline.services.PredictServiceMetrics;
import org.apache.skywalking.oap.server.ai.pipeline.services.ServiceMetrics;
import org.apache.skywalking.oap.server.core.analysis.DownSampling;
import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
import org.junit.jupiter.api.AfterEach;

77
oap-server/metrics-baseline/pom.xml
View File

@ -1,77 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
~
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>oap-server</artifactId>
<groupId>org.apache.skywalking</groupId>
<version>${revision}</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>metrics-baseline</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.skywalking</groupId>
<artifactId>server-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-testing</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-reflect</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
<version>${protobuf-maven-plugin.version}</version>
<configuration>
<!--
The version of protoc must match protobuf-java. If you don't depend on
protobuf-java directly, you will be transitively depending on the
protobuf-java version that grpc depends on.
-->
<protocArtifact>com.google.protobuf:protoc:${com.google.protobuf.protoc.version}:exe:${os.detected.classifier}
</protocArtifact>
<pluginId>grpc-java</pluginId>
<pluginArtifact>io.grpc:protoc-gen-grpc-java:${protoc-gen-grpc-java.plugin.version}:exe:${os.detected.classifier}
</pluginArtifact>
</configuration>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>compile-custom</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

36
oap-server/metrics-baseline/src/main/java/org/apache/skywalking/oap/server/baseline/BaselineModule.java
View File

@ -1,36 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.baseline;
import org.apache.skywalking.oap.server.baseline.service.BaselineQueryService;
import org.apache.skywalking.oap.server.library.module.ModuleDefine;
public class BaselineModule extends ModuleDefine {
public static final String NAME = "baseline";
public BaselineModule() {
super(NAME);
}
@Override
public Class[] services() {
return new Class[]{BaselineQueryService.class};
}
}

77
oap-server/metrics-baseline/src/main/java/org/apache/skywalking/oap/server/baseline/BaselineModuleProvider.java
View File

@ -1,77 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.baseline;
import org.apache.skywalking.oap.server.baseline.service.BaselineQueryService;
import org.apache.skywalking.oap.server.baseline.service.BaselineQueryServiceImpl;
import org.apache.skywalking.oap.server.library.module.ModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleDefine;
import org.apache.skywalking.oap.server.library.module.ModuleProvider;
import org.apache.skywalking.oap.server.library.module.ModuleStartException;
import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException;
public class BaselineModuleProvider extends ModuleProvider {
private BaselineModuleConfig config;
@Override
public String name() {
return "default";
}
@Override
public Class<? extends ModuleDefine> module() {
return BaselineModule.class;
}
@Override
public ConfigCreator<? extends ModuleConfig> newConfigCreator() {
return new ConfigCreator<BaselineModuleConfig>() {
@Override
public Class type() {
return BaselineModuleConfig.class;
}
@Override
public void onInitialized(BaselineModuleConfig moduleConfig) {
config = moduleConfig;
}
};
}
@Override
public void prepare() throws ServiceNotProvidedException, ModuleStartException {
this.registerServiceImplementation(BaselineQueryService.class, new BaselineQueryServiceImpl(config.getServiceHost(),
config.getServicePort()));
}
@Override
public void start() throws ServiceNotProvidedException, ModuleStartException {
}
@Override
public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException {
}
@Override
public String[] requiredModules() {
return new String[0];
}
}

19
oap-server/metrics-baseline/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine
View File

@ -1,19 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
org.apache.skywalking.oap.server.baseline.BaselineModule

19
oap-server/metrics-baseline/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider
View File

@ -1,19 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
org.apache.skywalking.oap.server.baseline.BaselineModuleProvider

2
oap-server/mqe-rt/pom.xml
View File

@ -46,7 +46,7 @@
</dependency>
<dependency>
<groupId>org.apache.skywalking</groupId>
<artifactId>metrics-baseline</artifactId>
<artifactId>ai-pipeline</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>

8
oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/MQEVisitorBase.java
View File

@ -40,9 +40,9 @@ import org.apache.skywalking.mqe.rt.operation.SortLabelValuesOp;
import org.apache.skywalking.mqe.rt.operation.SortValuesOp;
import org.apache.skywalking.mqe.rt.operation.TopNOfOp;
import org.apache.skywalking.mqe.rt.operation.TrendOp;
import org.apache.skywalking.oap.server.baseline.BaselineModule;
import org.apache.skywalking.oap.server.baseline.service.BaselineQueryService;
import org.apache.skywalking.oap.server.baseline.service.PredictServiceMetrics;
import org.apache.skywalking.oap.server.ai.pipeline.AIPipelineModule;
import org.apache.skywalking.oap.server.ai.pipeline.services.BaselineQueryService;
import org.apache.skywalking.oap.server.ai.pipeline.services.PredictServiceMetrics;
import org.apache.skywalking.oap.server.core.analysis.metrics.DataLabel;
import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
import org.apache.skywalking.oap.server.core.query.mqe.ExpressionResult;
@ -87,7 +87,7 @@ public abstract class MQEVisitorBase extends MQEParserBaseVisitor<ExpressionResu
private BaselineQueryService getBaselineQueryService() {
if (baselineQueryService == null) {
this.baselineQueryService = moduleManager.find(BaselineModule.NAME)
this.baselineQueryService = moduleManager.find(AIPipelineModule.NAME)
.provider()
.getService(BaselineQueryService.class);
}

1
oap-server/pom.xml
View File

@ -49,7 +49,6 @@
<module>ai-pipeline</module>
<module>mqe-grammar</module>
<module>mqe-rt</module>
<module>metrics-baseline</module>
</modules>
<profiles>

5
oap-server/server-core/pom.xml
View File

@ -102,11 +102,6 @@
<groupId>io.zipkin.zipkin2</groupId>
<artifactId>zipkin</artifactId>
</dependency>
<dependency>
<groupId>org.apache.skywalking</groupId>
<artifactId>ai-pipeline</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.groovy</groupId>
<artifactId>groovy</artifactId>

2
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java
View File

@ -30,6 +30,7 @@ import org.apache.skywalking.oap.server.core.config.DownSamplingConfigService;
import org.apache.skywalking.oap.server.core.config.HierarchyDefinitionService;
import org.apache.skywalking.oap.server.core.config.IComponentLibraryCatalogService;
import org.apache.skywalking.oap.server.core.config.NamingControl;
import org.apache.skywalking.oap.server.core.config.group.EndpointNameGroupService;
import org.apache.skywalking.oap.server.core.hierarchy.HierarchyService;
import org.apache.skywalking.oap.server.core.management.ui.menu.UIMenuManagementService;
import org.apache.skywalking.oap.server.core.management.ui.template.UITemplateManagementService;
@ -108,6 +109,7 @@ public class CoreModule extends ModuleDefine {
classes.add(CommandService.class);
classes.add(HierarchyService.class);
classes.add(EndpointNameGroupService.class);
return classes.toArray(new Class[]{});
}

11
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java
View File

@ -20,8 +20,6 @@ package org.apache.skywalking.oap.server.core;
import java.io.FileNotFoundException;
import java.io.IOException;
import org.apache.skywalking.oap.server.ai.pipeline.AIPipelineModule;
import org.apache.skywalking.oap.server.ai.pipeline.services.api.HttpUriRecognition;
import org.apache.skywalking.oap.server.configuration.api.ConfigurationModule;
import org.apache.skywalking.oap.server.configuration.api.DynamicConfigurationService;
import org.apache.skywalking.oap.server.core.analysis.ApdexThresholdConfig;
@ -48,6 +46,7 @@ import org.apache.skywalking.oap.server.core.config.DownSamplingConfigService;
import org.apache.skywalking.oap.server.core.config.HierarchyDefinitionService;
import org.apache.skywalking.oap.server.core.config.IComponentLibraryCatalogService;
import org.apache.skywalking.oap.server.core.config.NamingControl;
import org.apache.skywalking.oap.server.core.config.group.EndpointNameGroupService;
import org.apache.skywalking.oap.server.core.config.group.EndpointNameGrouping;
import org.apache.skywalking.oap.server.core.config.group.EndpointNameGroupingRuleWatcher;
import org.apache.skywalking.oap.server.core.config.group.openapi.EndpointNameGroupingRule4OpenapiWatcher;
@ -190,6 +189,7 @@ public class CoreModuleProvider extends ModuleProvider {
endpointNameGrouping
);
this.registerServiceImplementation(NamingControl.class, namingControl);
this.registerServiceImplementation(EndpointNameGroupService.class, endpointNameGrouping);
MeterEntity.setNamingControl(namingControl);
try {
endpointNameGroupingRuleWatcher = new EndpointNameGroupingRuleWatcher(
@ -396,11 +396,7 @@ public class CoreModuleProvider extends ModuleProvider {
grpcServer.addHandler(new HealthCheckServiceHandler());
grpcServer.addInterceptor(WatermarkGRPCInterceptor.INSTANCE);
endpointNameGrouping.startHttpUriRecognitionSvr(
getManager()
.find(AIPipelineModule.NAME)
.provider()
.getService(HttpUriRecognition.class),
endpointNameGrouping.prepareForHTTPUrlRecognition(
getService(MetadataQueryService.class),
moduleConfig.getSyncPeriodHttpUriRecognitionPattern(),
moduleConfig.getTrainingPeriodHttpUriRecognitionPattern(),
@ -490,7 +486,6 @@ public class CoreModuleProvider extends ModuleProvider {
return new String[] {
TelemetryModule.NAME,
ConfigurationModule.NAME,
AIPipelineModule.NAME
};
}
}

19
oap-server/metrics-baseline/src/main/java/org/apache/skywalking/oap/server/baseline/BaselineModuleConfig.java → oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGroupService.java
View File

@ -16,17 +16,14 @@
*
*/
package org.apache.skywalking.oap.server.baseline;
package org.apache.skywalking.oap.server.core.config.group;
import lombok.Getter;
import lombok.Setter;
import org.apache.skywalking.oap.server.library.module.ModuleConfig;
import org.apache.skywalking.oap.server.core.config.group.ai.HttpUriRecognition;
import org.apache.skywalking.oap.server.library.module.Service;
public class BaselineModuleConfig extends ModuleConfig {
@Setter
@Getter
private String serviceHost = "";
@Setter
@Getter
private int servicePort = 0;
public interface EndpointNameGroupService extends Service {
/**
* Start the HTTP URL Recognition service to group endpoint name
*/
void startHttpUriRecognitionSvr(final HttpUriRecognition httpUriRecognitionSvr);
}

138
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java
View File

@ -30,8 +30,11 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.skywalking.oap.server.ai.pipeline.services.api.HttpUriPattern;
import org.apache.skywalking.oap.server.ai.pipeline.services.api.HttpUriRecognition;
import lombok.AllArgsConstructor;
import lombok.Data;
import org.apache.skywalking.oap.server.core.config.group.ai.HttpUriPattern;
import org.apache.skywalking.oap.server.core.config.group.ai.HttpUriRecognition;
import org.apache.skywalking.oap.server.core.config.group.openapi.EndpointGroupingRule4Openapi;
import org.apache.skywalking.oap.server.core.config.group.uri.quickmatch.QuickUriGroupingRule;
import org.apache.skywalking.oap.server.core.query.MetadataQueryService;
@ -46,7 +49,7 @@ import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class EndpointNameGrouping {
public class EndpointNameGrouping implements EndpointNameGroupService {
public static final String ABANDONED_ENDPOINT_NAME = "_abandoned";
/**
@ -83,6 +86,10 @@ public class EndpointNameGrouping {
* The max number of HTTP URIs per service for further URI pattern recognition.
*/
private int maxHttpUrisNumberPerService = 3000;
/**
* Prepare for start HTTP URL Recognition config
*/
private HTTPUrlRecognitionConfig httpUrlRecognitionConfig;
/**
* Format the endpoint name according to the API patterns.
@ -185,68 +192,85 @@ public class EndpointNameGrouping {
return new Tuple2<>(formatResult.getReplacedName(), formatResult.isMatch());
}
public void startHttpUriRecognitionSvr(final HttpUriRecognition httpUriRecognitionSvr,
final MetadataQueryService metadataQueryService,
int syncPeriodHttpUriRecognitionPattern,
int trainingPeriodHttpUriRecognitionPattern,
int maxEndpointCandidatePerSvr) {
public void prepareForHTTPUrlRecognition(final MetadataQueryService metadataQueryService,
int syncPeriodHttpUriRecognitionPattern,
int trainingPeriodHttpUriRecognitionPattern,
int maxEndpointCandidatePerSvr) {
this.maxHttpUrisNumberPerService = maxEndpointCandidatePerSvr;
if (!httpUriRecognitionSvr.isInitialized()) {
this.httpUrlRecognitionConfig = new HTTPUrlRecognitionConfig(
metadataQueryService, syncPeriodHttpUriRecognitionPattern, trainingPeriodHttpUriRecognitionPattern,
maxEndpointCandidatePerSvr
);
}
@Override
public void startHttpUriRecognitionSvr(HttpUriRecognition httpUriRecognitionSvr) {
if (this.httpUrlRecognitionConfig == null || !httpUriRecognitionSvr.isInitialized()) {
return;
}
this.quickUriGroupingRule = new QuickUriGroupingRule();
HTTPUrlRecognitionConfig config = this.httpUrlRecognitionConfig;
Executors.newSingleThreadScheduledExecutor()
.scheduleWithFixedDelay(
new RunnableWithExceptionProtection(
() -> {
int currentExecutionCounter = aiPipelineExecutionCounter.incrementAndGet();
if (currentExecutionCounter % trainingPeriodHttpUriRecognitionPattern == 0) {
// Send the cached URIs to the recognition server to build new patterns.
cachedHttpUris.forEach((serviceName, httpUris) -> {
final List<HttpUriRecognition.HTTPUri> candidates4UriPatterns = new ArrayList<>(
3000);
httpUris.forEach((uri, candidates) -> {
if (candidates.size() == 0) {
//unrecognized uri
candidates4UriPatterns.add(new HttpUriRecognition.HTTPUri(uri));
} else {
String candidateUri;
while ((candidateUri = candidates.poll()) != null) {
candidates4UriPatterns.add(
new HttpUriRecognition.HTTPUri(candidateUri));
}
}
});
.scheduleWithFixedDelay(
new RunnableWithExceptionProtection(
() -> {
int currentExecutionCounter = aiPipelineExecutionCounter.incrementAndGet();
if (currentExecutionCounter % config.trainingPeriodHttpUriRecognitionPattern == 0) {
// Send the cached URIs to the recognition server to build new patterns.
cachedHttpUris.forEach((serviceName, httpUris) -> {
final List<HttpUriRecognition.HTTPUri> candidates4UriPatterns = new ArrayList<>(
3000);
httpUris.forEach((uri, candidates) -> {
if (candidates.size() == 0) {
//unrecognized uri
candidates4UriPatterns.add(new HttpUriRecognition.HTTPUri(uri));
} else {
String candidateUri;
while ((candidateUri = candidates.poll()) != null) {
candidates4UriPatterns.add(
new HttpUriRecognition.HTTPUri(candidateUri));
}
}
});
// Reset the cache once the URIs are sent to the recognition server.
httpUris.clear();
httpUriRecognitionSvr.feedRawData(serviceName, candidates4UriPatterns);
});
}
if (currentExecutionCounter % syncPeriodHttpUriRecognitionPattern == 0) {
// Sync with the recognition server per 1 min to get the latest patterns.
try {
metadataQueryService.listServices(null, null).forEach(
service -> {
final List<HttpUriPattern> patterns
= httpUriRecognitionSvr.fetchAllPatterns(service.getName());
if (CollectionUtils.isNotEmpty(patterns)) {
patterns.forEach(
p -> quickUriGroupingRule.addRule(
service.getName(), p.getPattern()));
// Reset the cache once the URIs are sent to the recognition server.
httpUris.clear();
httpUriRecognitionSvr.feedRawData(serviceName, candidates4UriPatterns);
});
}
if (currentExecutionCounter % config.syncPeriodHttpUriRecognitionPattern == 0) {
// Sync with the recognition server per 1 min to get the latest patterns.
try {
config.metadataQueryService.listServices(null, null).forEach(
service -> {
final List<HttpUriPattern> patterns
= httpUriRecognitionSvr.fetchAllPatterns(service.getName());
if (CollectionUtils.isNotEmpty(patterns)) {
patterns.forEach(
p -> quickUriGroupingRule.addRule(
service.getName(), p.getPattern()));
}
}
);
} catch (IOException e) {
log.error("Fail to load all services.", e);
}
}
}
);
} catch (IOException e) {
log.error("Fail to load all services.", e);
}
}
},
t -> log.error("Fail to recognize URI patterns.", t)
), 60, 1, TimeUnit.SECONDS
);
}
},
t -> log.error("Fail to recognize URI patterns.", t)
), 60, 1, TimeUnit.SECONDS
);
}
@Data
@AllArgsConstructor
private static class HTTPUrlRecognitionConfig {
private MetadataQueryService metadataQueryService;
private int syncPeriodHttpUriRecognitionPattern;
private int trainingPeriodHttpUriRecognitionPattern;
private int maxEndpointCandidatePerSvr;
}
}

2
oap-server/ai-pipeline/src/main/java/org/apache/skywalking/oap/server/ai/pipeline/services/api/HttpUriPattern.java → oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/ai/HttpUriPattern.java
View File

@ -16,7 +16,7 @@
*
*/
package org.apache.skywalking.oap.server.ai.pipeline.services.api;
package org.apache.skywalking.oap.server.core.config.group.ai;
import lombok.Getter;
import lombok.RequiredArgsConstructor;

5
oap-server/ai-pipeline/src/main/java/org/apache/skywalking/oap/server/ai/pipeline/services/api/HttpUriRecognition.java → oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/ai/HttpUriRecognition.java
View File

@ -16,17 +16,16 @@
*
*/
package org.apache.skywalking.oap.server.ai.pipeline.services.api;
package org.apache.skywalking.oap.server.core.config.group.ai;
import java.util.List;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import org.apache.skywalking.oap.server.library.module.Service;
/**
* HttpUriRecognition is a service to recognize the patterns of HTTP URIs
*/
public interface HttpUriRecognition extends Service {
public interface HttpUriRecognition {
/**
* @return true if the service is initialized and active.

2
oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/CoreModuleTest.java
View File

@ -26,6 +26,6 @@ public class CoreModuleTest {
public void testOpenServiceList() {
CoreModule coreModule = new CoreModule();
Assertions.assertEquals(48, coreModule.services().length);
Assertions.assertEquals(49, coreModule.services().length);
}
}

5
oap-server/server-starter/pom.xml
View File

@ -296,11 +296,6 @@
<artifactId>storage-elasticsearch-plugin</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.skywalking</groupId>
<artifactId>metrics-baseline</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<build>

8
oap-server/server-starter/src/main/resources/application.yml
View File

@ -693,9 +693,5 @@ ai-pipeline:
default:
uriRecognitionServerAddr: ${SW_AI_PIPELINE_URI_RECOGNITION_SERVER_ADDR:}
uriRecognitionServerPort: ${SW_AI_PIPELINE_URI_RECOGNITION_SERVER_PORT:17128}
baseline:
selector: ${SW_BASELINE:default}
default:
serviceHost: ${SW_BASELINE_SERVICE_HOST:}
servicePort: ${SW_BASELINE_SERVICE_PORT:18080}
baselineServerAddr: ${SW_API_PIPELINE_BASELINE_SERVICE_HOST:}
baselineServerPort: ${SW_API_PIPELINE_BASELINE_SERVICE_PORT:18080}

2
test/e2e-v2/cases/alarm/banyandb/docker-compose.yml
View File

@ -30,7 +30,7 @@ services:
environment:
SW_STORAGE: banyandb
SW_SEARCHABLE_ALARM_TAG_KEYS: level,receivers
SW_BASELINE_SERVICE_HOST: baseline-server
SW_API_PIPELINE_BASELINE_SERVICE_HOST: baseline-server
ports:
- 12800
depends_on:

2
test/e2e-v2/cases/alarm/es/docker-compose.yml
View File

@ -37,7 +37,7 @@ services:
environment:
SW_STORAGE: elasticsearch
SW_SEARCHABLE_ALARM_TAG_KEYS: level,receivers
SW_BASELINE_SERVICE_HOST: baseline-server
SW_API_PIPELINE_BASELINE_SERVICE_HOST: baseline-server
ports:
- 12800
depends_on:

2
test/e2e-v2/cases/alarm/es/es-sharding/docker-compose.yml
View File

@ -38,7 +38,7 @@ services:
SW_STORAGE: elasticsearch
SW_SEARCHABLE_ALARM_TAG_KEYS: level,receivers
SW_STORAGE_ES_LOGIC_SHARDING: "true"
SW_BASELINE_SERVICE_HOST: baseline-server
SW_API_PIPELINE_BASELINE_SERVICE_HOST: baseline-server
ports:
- 12800
depends_on:

2
test/e2e-v2/cases/alarm/mysql/docker-compose.yml
View File

@ -39,7 +39,7 @@ services:
environment:
SW_STORAGE: mysql
SW_SEARCHABLE_ALARM_TAG_KEYS: level,receivers
SW_BASELINE_SERVICE_HOST: baseline-server
SW_API_PIPELINE_BASELINE_SERVICE_HOST: baseline-server
entrypoint: ['sh', '-c', '/download-mysql.sh /skywalking/oap-libs && /skywalking/docker-entrypoint.sh']
ports:
- 12800

2
test/e2e-v2/cases/alarm/postgres/docker-compose.yml
View File

@ -39,7 +39,7 @@ services:
SW_STORAGE: postgresql
SW_JDBC_URL: "jdbc:postgresql://postgres:5432/skywalking"
SW_SEARCHABLE_ALARM_TAG_KEYS: level,receivers
SW_BASELINE_SERVICE_HOST: baseline-server
SW_API_PIPELINE_BASELINE_SERVICE_HOST: baseline-server
ports:
- 12800
depends_on:

2
test/e2e-v2/cases/mqe/docker-compose.yml
View File

@ -23,7 +23,7 @@ services:
volumes:
- ./oal/core.oal:/skywalking/config/oal/core.oal
environment:
SW_BASELINE_SERVICE_HOST: baseline-server
SW_API_PIPELINE_BASELINE_SERVICE_HOST: baseline-server
depends_on:
banyandb:
condition: service_healthy