kazusa
/
skywalking
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

BanyanDB: support using native term searching for `keyword` in query `findEndpoint` and `getAlarm`. (#12633)

This commit is contained in:
Wan Kai 2024-09-24 14:57:38 +08:00 committed by GitHub
parent b857fb0091
commit 15ad71d633
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 139 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/en/changes/changes.md
View File

@ -69,6 +69,7 @@
* Add SkyWalking Java Agent self observability dashboard.
* Add Component ID(5022) for the GoFrame framework.
* Bump up protobuf java dependencies to 3.25.5.
* BanyanDB: support using native term searching for `keyword` in query `findEndpoint` and `getAlarm`.
#### UI

2
oap-server-bom/pom.xml
View File

@ -73,7 +73,7 @@
<httpcore.version>4.4.13</httpcore.version>
<httpasyncclient.version>4.1.5</httpasyncclient.version>
<commons-compress.version>1.21</commons-compress.version>
<banyandb-java-client.version>0.7-rc3</banyandb-java-client.version>
<banyandb-java-client.version>0.7.0-rc4</banyandb-java-client.version>
<kafka-clients.version>3.4.0</kafka-clients.version>
<spring-kafka-test.version>2.4.6.RELEASE</spring-kafka-test.version>
<consul.client.version>1.5.3</consul.client.version>

1
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java
View File

@ -82,6 +82,7 @@ public class AlarmRecord extends Record {
private long startTime;
@Column(name = ALARM_MESSAGE, length = 512)
@ElasticSearch.MatchQuery
@BanyanDB.MatchQuery(analyzer = BanyanDB.MatchQuery.AnalyzerType.SIMPLE)
private String alarmMessage;
@Column(name = RULE_NAME)
private String ruleName;

1
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java
View File

@ -59,6 +59,7 @@ public class EndpointTraffic extends Metrics {
@Column(name = NAME)
@ElasticSearch.Column(legacyName = "name")
@ElasticSearch.MatchQuery
@BanyanDB.MatchQuery(analyzer = BanyanDB.MatchQuery.AnalyzerType.URL)
@BanyanDB.SeriesID(index = 1)
private String name = Const.EMPTY_STRING;

33
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDB.java
View File

@ -177,4 +177,37 @@ public @interface BanyanDB {
*/
int countersNumber() default 1000;
}
/**
* Match query is designed for BanyanDB match query with specific analyzer. It is a fuzzy query implementation
* powered by analyzer.
*
* @since 10.1.0
*/
@Target({ElementType.FIELD})
@Retention(RetentionPolicy.RUNTIME)
@interface MatchQuery {
AnalyzerType analyzer();
enum AnalyzerType {
/**
* Keyword analyzer is a “noop” analyzer which returns the entire input string as a single token.
*/
KEYWORD,
/**
* Standard analyzer provides grammar based tokenization
*/
STANDARD,
/**
* Simple analyzer breaks text into tokens at any non-letter character,
* such as numbers, spaces, hyphens and apostrophes, discards non-letter characters,
* and changes uppercase to lowercase.
*/
SIMPLE,
/**
* URL analyzer breaks test into tokens at any non-letter and non-digit character
*/
URL
}
}
}

6
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java
View File

@ -61,6 +61,12 @@ public class BanyanDBExtension {
@Getter
private final boolean isMeasureField;
/**
* The analyzer policy appointed to fuzzy query, especially for BanyanDB.
*/
@Getter
private final BanyanDB.MatchQuery.AnalyzerType analyzer;
/**
* @return true if this column is a part of sharding key
*/

5
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java
View File

@ -99,6 +99,11 @@ public class ModelColumn {
}
this.indexOnly = indexOnly;
this.banyanDBExtension = banyanDBExtension;
if (!this.banyanDBExtension.shouldIndex() && this.banyanDBExtension.getAnalyzer() != null) {
throw new IllegalArgumentException(
"The column " + columnName + " should be indexed if require MatchQuery.");
}
}
/**

8
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
View File

@ -215,11 +215,15 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
BanyanDB.MeasureField.class);
final BanyanDB.TopNAggregation topNAggregation = field.getAnnotation(
BanyanDB.TopNAggregation.class);
final BanyanDB.MatchQuery analyzer = field.getAnnotation(
BanyanDB.MatchQuery.class);
final boolean shouldIndex = (banyanDBNoIndex == null) && !column.storageOnly();
BanyanDBExtension banyanDBExtension = new BanyanDBExtension(
banyanDBSeriesID == null ? -1 : banyanDBSeriesID.index(),
banyanDBNoIndex == null && !column.storageOnly(),
shouldIndex,
banyanDBIndexRule == null ? BanyanDB.IndexRule.IndexType.INVERTED : banyanDBIndexRule.indexType(),
banyanDBMeasureField != null
banyanDBMeasureField != null,
analyzer == null ? null : analyzer.analyzer()
);
if (topNAggregation != null) {

26
oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java
View File

@ -49,7 +49,7 @@ public class ModelColumnTest {
new SQLDatabaseExtension(),
new ElasticSearchExtension(
ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, null, false, false),
new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false)
new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB.MatchQuery.AnalyzerType.SIMPLE)
);
Assertions.assertTrue(column.isStorageOnly());
Assertions.assertEquals("abc", column.getColumnName().getName());
@ -58,7 +58,7 @@ public class ModelColumnTest {
false, false, true, 200,
new SQLDatabaseExtension(),
new ElasticSearchExtension(ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, null, false, false),
new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false)
new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB. MatchQuery.AnalyzerType.SIMPLE)
);
Assertions.assertTrue(column.isStorageOnly());
Assertions.assertEquals("abc", column.getColumnName().getName());
@ -68,7 +68,7 @@ public class ModelColumnTest {
false, false, true, 200,
new SQLDatabaseExtension(),
new ElasticSearchExtension(ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, null, false, false),
new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false)
new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB.MatchQuery.AnalyzerType.SIMPLE)
);
Assertions.assertFalse(column.isStorageOnly());
Assertions.assertEquals("abc", column.getColumnName().getName());
@ -82,7 +82,7 @@ public class ModelColumnTest {
new SQLDatabaseExtension(),
new ElasticSearchExtension(
ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, "abc", false, false),
new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false)
new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB.MatchQuery.AnalyzerType.SIMPLE)
);
});
}
@ -95,7 +95,23 @@ public class ModelColumnTest {
new SQLDatabaseExtension(),
new ElasticSearchExtension(
ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, "abc", false, false),
new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false)
new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB.MatchQuery.AnalyzerType.SIMPLE)
);
});
}
@Test
public void testConflictDefinitionStorageOnly() {
assertThrows(IllegalArgumentException.class, () -> {
new ModelColumn(new ColumnName(c), String.class, String.class,
true, false, false, 200,
new SQLDatabaseExtension(),
new ElasticSearchExtension(
ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, "abc", false, false),
new BanyanDBExtension(
-1, false, BanyanDB.IndexRule.IndexType.INVERTED, false,
BanyanDB.MatchQuery.AnalyzerType.SIMPLE
)
);
});
}

51
oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/MetadataRegistry.java
View File

@ -79,6 +79,7 @@ import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.config.ConfigService;
import org.apache.skywalking.oap.server.core.query.enumeration.Step;
import org.apache.skywalking.oap.server.core.storage.StorageException;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata;
import org.apache.skywalking.oap.server.core.storage.model.Model;
@ -167,7 +168,7 @@ public enum MetadataRegistry {
.collect(Collectors.toList());
if (model.getBanyanDBModelExtension().isStoreIDTag()) {
indexRules.add(indexRule(schemaMetadata.group, BanyanDBConverter.ID));
indexRules.add(indexRule(schemaMetadata.group, BanyanDBConverter.ID, null));
}
final Measure.Builder builder = Measure.newBuilder();
@ -299,10 +300,29 @@ public enum MetadataRegistry {
}
}
IndexRule indexRule(String group, String tagName) {
return IndexRule.newBuilder()
.setMetadata(Metadata.newBuilder().setName(tagName).setGroup(group))
.setType(IndexRule.Type.TYPE_INVERTED).addTags(tagName).build();
IndexRule indexRule(String group, String tagName, BanyanDB.MatchQuery.AnalyzerType analyzer) {
IndexRule.Builder builder = IndexRule.newBuilder()
.setMetadata(Metadata.newBuilder().setName(tagName).setGroup(group))
.setType(IndexRule.Type.TYPE_INVERTED).addTags(tagName);
if (analyzer != null) {
switch (analyzer) {
case KEYWORD:
builder.setAnalyzer("keyword");
break;
case STANDARD:
builder.setAnalyzer("standard");
break;
case SIMPLE:
builder.setAnalyzer("simple");
break;
case URL:
builder.setAnalyzer("url");
break;
default:
throw new UnsupportedOperationException("unsupported analyzer type: " + analyzer);
}
}
return builder.build();
}
/**
@ -341,8 +361,13 @@ public enum MetadataRegistry {
final TagSpec tagSpec = parseTagSpec(col);
builder.spec(columnStorageName, new ColumnSpec(ColumnType.TAG, col.getType()));
String colName = col.getColumnName().getStorageName();
if (!shardingColumns.contains(colName) && col.getBanyanDBExtension().shouldIndex()) {
tagMetadataList.add(new TagMetadata(indexRule(group, tagSpec.getName()), tagSpec));
if (col.getBanyanDBExtension().shouldIndex()) {
if (!shardingColumns.contains(colName) || null != col.getBanyanDBExtension().getAnalyzer()) {
tagMetadataList.add(new TagMetadata(
indexRule(group, tagSpec.getName(), col.getBanyanDBExtension().getAnalyzer()), tagSpec));
} else {
tagMetadataList.add(new TagMetadata(null, tagSpec));
}
} else {
tagMetadataList.add(new TagMetadata(null, tagSpec));
}
@ -382,7 +407,17 @@ public enum MetadataRegistry {
final TagSpec tagSpec = parseTagSpec(col);
builder.spec(columnStorageName, new ColumnSpec(ColumnType.TAG, col.getType()));
String colName = col.getColumnName().getStorageName();
result.tag(new TagMetadata(!shardingColumns.contains(colName) && col.getBanyanDBExtension().shouldIndex() ? indexRule(group, tagSpec.getName()) : null, tagSpec));
if (col.getBanyanDBExtension().shouldIndex()) {
if (!shardingColumns.contains(colName) || null != col.getBanyanDBExtension().getAnalyzer()) {
result.tag(new TagMetadata(
indexRule(group, tagSpec.getName(), col.getBanyanDBExtension().getAnalyzer()), tagSpec));
} else {
result.tag(new TagMetadata(null, tagSpec));
}
} else {
result.tag(new TagMetadata(null, tagSpec));
}
}
return result.build();

11
oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBMetadataQueryDAO.java
View File

@ -24,6 +24,7 @@ import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import org.apache.commons.lang3.StringUtils;
import org.apache.skywalking.banyandb.model.v1.BanyandbModel;
import org.apache.skywalking.banyandb.v1.client.AbstractCriteria;
import org.apache.skywalking.banyandb.v1.client.AbstractQuery;
import org.apache.skywalking.banyandb.v1.client.DataPoint;
@ -186,6 +187,12 @@ public class BanyanDBMetadataQueryDAO extends AbstractBanyanDBDAO implements IMe
if (StringUtil.isNotEmpty(serviceId)) {
query.and(eq(EndpointTraffic.SERVICE_ID, serviceId));
}
if (StringUtil.isNotEmpty(keyword)) {
query.and(match(EndpointTraffic.NAME, keyword,
BanyandbModel.Condition.MatchOption.newBuilder().setOperator(
BanyandbModel.Condition.MatchOption.Operator.OPERATOR_AND).build()
));
}
query.setOrderBy(new AbstractQuery.OrderBy(AbstractQuery.Sort.DESC));
}
});
@ -194,10 +201,6 @@ public class BanyanDBMetadataQueryDAO extends AbstractBanyanDBDAO implements IMe
for (final DataPoint dataPoint : resp.getDataPoints()) {
endpoints.add(buildEndpoint(dataPoint, schema));
}
if (StringUtil.isNotEmpty(keyword)) {
return endpoints.stream().filter(e -> e.getName().contains(keyword)).collect(Collectors.toList());
}
return endpoints;
}

9
oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java
View File

@ -19,6 +19,7 @@
package org.apache.skywalking.oap.server.storage.plugin.banyandb.stream;
import com.google.gson.Gson;
import org.apache.skywalking.banyandb.model.v1.BanyandbModel;
import org.apache.skywalking.banyandb.v1.client.AbstractCriteria;
import org.apache.skywalking.banyandb.v1.client.AbstractQuery;
import org.apache.skywalking.banyandb.v1.client.And;
@ -297,6 +298,14 @@ public abstract class AbstractBanyanDBDAO extends AbstractDAO<BanyanDBStorageCli
return PairQueryCondition.StringQueryCondition.eq(name, value);
}
protected PairQueryCondition<String> match(String name, String value) {
return PairQueryCondition.StringQueryCondition.match(name, value);
}
protected PairQueryCondition<String> match(String name, String value, BanyandbModel.Condition.MatchOption matchOption) {
return PairQueryCondition.StringQueryCondition.match(name, value, matchOption);
}
protected PairQueryCondition<List<String>> in(String name, List<String> values) {
return PairQueryCondition.StringArrayQueryCondition.in(name, values);
}

5
oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBAlarmQueryDAO.java
View File

@ -32,6 +32,7 @@ import org.apache.skywalking.oap.server.core.query.type.AlarmMessage;
import org.apache.skywalking.oap.server.core.query.type.Alarms;
import org.apache.skywalking.oap.server.core.storage.query.IAlarmQueryDAO;
import org.apache.skywalking.oap.server.library.util.CollectionUtils;
import org.apache.skywalking.oap.server.library.util.StringUtil;
import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBConverter;
import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageClient;
@ -71,7 +72,9 @@ public class BanyanDBAlarmQueryDAO extends AbstractBanyanDBDAO implements IAlarm
if (Objects.nonNull(scopeId)) {
query.and(eq(AlarmRecord.SCOPE, (long) scopeId));
}
if (StringUtil.isNotEmpty(keyword)) {
query.and(match(AlarmRecord.ALARM_MESSAGE, keyword));
}
if (CollectionUtils.isNotEmpty(tags)) {
List<String> tagsConditions = new ArrayList<>(tags.size());
for (final Tag tag : tags) {

2
test/e2e-v2/script/env
View File

@ -23,7 +23,7 @@ SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449
SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016
SW_KUBERNETES_COMMIT_SHA=1335f15bf821a40a7cd71448fa805f0be265afcc
SW_ROVER_COMMIT=6bbd39aa701984482330d9dfb4dbaaff0527d55c
SW_BANYANDB_COMMIT=0e734c462571dcf55dbb7761211c07d8b156521e
SW_BANYANDB_COMMIT=9770bf48f9b88574683f56e2a137555cd68066d9
SW_AGENT_PHP_COMMIT=3192c553002707d344bd6774cfab5bc61f67a1d3
SW_CTL_COMMIT=d5f3597733aa5217373986d776a3ee5ee8b3c468