diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 79b9ae7579..378b56e607 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -69,6 +69,7 @@ * Add SkyWalking Java Agent self observability dashboard. * Add Component ID(5022) for the GoFrame framework. * Bump up protobuf java dependencies to 3.25.5. +* BanyanDB: support using native term searching for `keyword` in query `findEndpoint` and `getAlarm`. #### UI diff --git a/oap-server-bom/pom.xml b/oap-server-bom/pom.xml index d9da1b915d..a2aced4116 100644 --- a/oap-server-bom/pom.xml +++ b/oap-server-bom/pom.xml @@ -73,7 +73,7 @@ 4.4.13 4.1.5 1.21 - 0.7-rc3 + 0.7.0-rc4 3.4.0 2.4.6.RELEASE 1.5.3 diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java index 9476ca8a3c..be4bbd0f66 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java @@ -82,6 +82,7 @@ public class AlarmRecord extends Record { private long startTime; @Column(name = ALARM_MESSAGE, length = 512) @ElasticSearch.MatchQuery + @BanyanDB.MatchQuery(analyzer = BanyanDB.MatchQuery.AnalyzerType.SIMPLE) private String alarmMessage; @Column(name = RULE_NAME) private String ruleName; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java index a8c72c9363..a2beaca1d3 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java @@ -59,6 +59,7 @@ public class EndpointTraffic extends Metrics { @Column(name = NAME) @ElasticSearch.Column(legacyName = "name") @ElasticSearch.MatchQuery + @BanyanDB.MatchQuery(analyzer = BanyanDB.MatchQuery.AnalyzerType.URL) @BanyanDB.SeriesID(index = 1) private String name = Const.EMPTY_STRING; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDB.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDB.java index 6eb11c4864..ab0bfe2f9c 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDB.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDB.java @@ -177,4 +177,37 @@ public @interface BanyanDB { */ int countersNumber() default 1000; } + + /** + * Match query is designed for BanyanDB match query with specific analyzer. It is a fuzzy query implementation + * powered by analyzer. + * + * @since 10.1.0 + */ + @Target({ElementType.FIELD}) + @Retention(RetentionPolicy.RUNTIME) + @interface MatchQuery { + AnalyzerType analyzer(); + + enum AnalyzerType { + /** + * Keyword analyzer is a “noop” analyzer which returns the entire input string as a single token. + */ + KEYWORD, + /** + * Standard analyzer provides grammar based tokenization + */ + STANDARD, + /** + * Simple analyzer breaks text into tokens at any non-letter character, + * such as numbers, spaces, hyphens and apostrophes, discards non-letter characters, + * and changes uppercase to lowercase. + */ + SIMPLE, + /** + * URL analyzer breaks test into tokens at any non-letter and non-digit character + */ + URL + } + } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java index 0f47586d3e..576d41469c 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java @@ -61,6 +61,12 @@ public class BanyanDBExtension { @Getter private final boolean isMeasureField; + /** + * The analyzer policy appointed to fuzzy query, especially for BanyanDB. + */ + @Getter + private final BanyanDB.MatchQuery.AnalyzerType analyzer; + /** * @return true if this column is a part of sharding key */ diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java index bfc5ba3221..9712a8fca1 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java @@ -99,6 +99,11 @@ public class ModelColumn { } this.indexOnly = indexOnly; this.banyanDBExtension = banyanDBExtension; + + if (!this.banyanDBExtension.shouldIndex() && this.banyanDBExtension.getAnalyzer() != null) { + throw new IllegalArgumentException( + "The column " + columnName + " should be indexed if require MatchQuery."); + } } /** diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java index 51aadd4c1b..eabcfdc350 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java @@ -215,11 +215,15 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula BanyanDB.MeasureField.class); final BanyanDB.TopNAggregation topNAggregation = field.getAnnotation( BanyanDB.TopNAggregation.class); + final BanyanDB.MatchQuery analyzer = field.getAnnotation( + BanyanDB.MatchQuery.class); + final boolean shouldIndex = (banyanDBNoIndex == null) && !column.storageOnly(); BanyanDBExtension banyanDBExtension = new BanyanDBExtension( banyanDBSeriesID == null ? -1 : banyanDBSeriesID.index(), - banyanDBNoIndex == null && !column.storageOnly(), + shouldIndex, banyanDBIndexRule == null ? BanyanDB.IndexRule.IndexType.INVERTED : banyanDBIndexRule.indexType(), - banyanDBMeasureField != null + banyanDBMeasureField != null, + analyzer == null ? null : analyzer.analyzer() ); if (topNAggregation != null) { diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java index f1949cbece..cab20c4969 100644 --- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java @@ -49,7 +49,7 @@ public class ModelColumnTest { new SQLDatabaseExtension(), new ElasticSearchExtension( ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, null, false, false), - new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false) + new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB.MatchQuery.AnalyzerType.SIMPLE) ); Assertions.assertTrue(column.isStorageOnly()); Assertions.assertEquals("abc", column.getColumnName().getName()); @@ -58,7 +58,7 @@ public class ModelColumnTest { false, false, true, 200, new SQLDatabaseExtension(), new ElasticSearchExtension(ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, null, false, false), - new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false) + new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB. MatchQuery.AnalyzerType.SIMPLE) ); Assertions.assertTrue(column.isStorageOnly()); Assertions.assertEquals("abc", column.getColumnName().getName()); @@ -68,7 +68,7 @@ public class ModelColumnTest { false, false, true, 200, new SQLDatabaseExtension(), new ElasticSearchExtension(ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, null, false, false), - new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false) + new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB.MatchQuery.AnalyzerType.SIMPLE) ); Assertions.assertFalse(column.isStorageOnly()); Assertions.assertEquals("abc", column.getColumnName().getName()); @@ -82,7 +82,7 @@ public class ModelColumnTest { new SQLDatabaseExtension(), new ElasticSearchExtension( ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, "abc", false, false), - new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false) + new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB.MatchQuery.AnalyzerType.SIMPLE) ); }); } @@ -95,7 +95,23 @@ public class ModelColumnTest { new SQLDatabaseExtension(), new ElasticSearchExtension( ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, "abc", false, false), - new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false) + new BanyanDBExtension(-1, true, BanyanDB.IndexRule.IndexType.INVERTED, false, BanyanDB.MatchQuery.AnalyzerType.SIMPLE) + ); + }); + } + + @Test + public void testConflictDefinitionStorageOnly() { + assertThrows(IllegalArgumentException.class, () -> { + new ModelColumn(new ColumnName(c), String.class, String.class, + true, false, false, 200, + new SQLDatabaseExtension(), + new ElasticSearchExtension( + ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER, "abc", false, false), + new BanyanDBExtension( + -1, false, BanyanDB.IndexRule.IndexType.INVERTED, false, + BanyanDB.MatchQuery.AnalyzerType.SIMPLE + ) ); }); } diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/MetadataRegistry.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/MetadataRegistry.java index 145a52c8b4..77e8b1263a 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/MetadataRegistry.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/MetadataRegistry.java @@ -79,6 +79,7 @@ import org.apache.skywalking.oap.server.core.analysis.record.Record; import org.apache.skywalking.oap.server.core.config.ConfigService; import org.apache.skywalking.oap.server.core.query.enumeration.Step; import org.apache.skywalking.oap.server.core.storage.StorageException; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB; import org.apache.skywalking.oap.server.core.storage.annotation.Column; import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata; import org.apache.skywalking.oap.server.core.storage.model.Model; @@ -167,7 +168,7 @@ public enum MetadataRegistry { .collect(Collectors.toList()); if (model.getBanyanDBModelExtension().isStoreIDTag()) { - indexRules.add(indexRule(schemaMetadata.group, BanyanDBConverter.ID)); + indexRules.add(indexRule(schemaMetadata.group, BanyanDBConverter.ID, null)); } final Measure.Builder builder = Measure.newBuilder(); @@ -299,10 +300,29 @@ public enum MetadataRegistry { } } - IndexRule indexRule(String group, String tagName) { - return IndexRule.newBuilder() - .setMetadata(Metadata.newBuilder().setName(tagName).setGroup(group)) - .setType(IndexRule.Type.TYPE_INVERTED).addTags(tagName).build(); + IndexRule indexRule(String group, String tagName, BanyanDB.MatchQuery.AnalyzerType analyzer) { + IndexRule.Builder builder = IndexRule.newBuilder() + .setMetadata(Metadata.newBuilder().setName(tagName).setGroup(group)) + .setType(IndexRule.Type.TYPE_INVERTED).addTags(tagName); + if (analyzer != null) { + switch (analyzer) { + case KEYWORD: + builder.setAnalyzer("keyword"); + break; + case STANDARD: + builder.setAnalyzer("standard"); + break; + case SIMPLE: + builder.setAnalyzer("simple"); + break; + case URL: + builder.setAnalyzer("url"); + break; + default: + throw new UnsupportedOperationException("unsupported analyzer type: " + analyzer); + } + } + return builder.build(); } /** @@ -341,8 +361,13 @@ public enum MetadataRegistry { final TagSpec tagSpec = parseTagSpec(col); builder.spec(columnStorageName, new ColumnSpec(ColumnType.TAG, col.getType())); String colName = col.getColumnName().getStorageName(); - if (!shardingColumns.contains(colName) && col.getBanyanDBExtension().shouldIndex()) { - tagMetadataList.add(new TagMetadata(indexRule(group, tagSpec.getName()), tagSpec)); + if (col.getBanyanDBExtension().shouldIndex()) { + if (!shardingColumns.contains(colName) || null != col.getBanyanDBExtension().getAnalyzer()) { + tagMetadataList.add(new TagMetadata( + indexRule(group, tagSpec.getName(), col.getBanyanDBExtension().getAnalyzer()), tagSpec)); + } else { + tagMetadataList.add(new TagMetadata(null, tagSpec)); + } } else { tagMetadataList.add(new TagMetadata(null, tagSpec)); } @@ -382,7 +407,17 @@ public enum MetadataRegistry { final TagSpec tagSpec = parseTagSpec(col); builder.spec(columnStorageName, new ColumnSpec(ColumnType.TAG, col.getType())); String colName = col.getColumnName().getStorageName(); - result.tag(new TagMetadata(!shardingColumns.contains(colName) && col.getBanyanDBExtension().shouldIndex() ? indexRule(group, tagSpec.getName()) : null, tagSpec)); + + if (col.getBanyanDBExtension().shouldIndex()) { + if (!shardingColumns.contains(colName) || null != col.getBanyanDBExtension().getAnalyzer()) { + result.tag(new TagMetadata( + indexRule(group, tagSpec.getName(), col.getBanyanDBExtension().getAnalyzer()), tagSpec)); + } else { + result.tag(new TagMetadata(null, tagSpec)); + } + } else { + result.tag(new TagMetadata(null, tagSpec)); + } } return result.build(); diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBMetadataQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBMetadataQueryDAO.java index 3a73581cab..ea710f854c 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBMetadataQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBMetadataQueryDAO.java @@ -24,6 +24,7 @@ import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import org.apache.commons.lang3.StringUtils; +import org.apache.skywalking.banyandb.model.v1.BanyandbModel; import org.apache.skywalking.banyandb.v1.client.AbstractCriteria; import org.apache.skywalking.banyandb.v1.client.AbstractQuery; import org.apache.skywalking.banyandb.v1.client.DataPoint; @@ -186,6 +187,12 @@ public class BanyanDBMetadataQueryDAO extends AbstractBanyanDBDAO implements IMe if (StringUtil.isNotEmpty(serviceId)) { query.and(eq(EndpointTraffic.SERVICE_ID, serviceId)); } + if (StringUtil.isNotEmpty(keyword)) { + query.and(match(EndpointTraffic.NAME, keyword, + BanyandbModel.Condition.MatchOption.newBuilder().setOperator( + BanyandbModel.Condition.MatchOption.Operator.OPERATOR_AND).build() + )); + } query.setOrderBy(new AbstractQuery.OrderBy(AbstractQuery.Sort.DESC)); } }); @@ -194,10 +201,6 @@ public class BanyanDBMetadataQueryDAO extends AbstractBanyanDBDAO implements IMe for (final DataPoint dataPoint : resp.getDataPoints()) { endpoints.add(buildEndpoint(dataPoint, schema)); } - - if (StringUtil.isNotEmpty(keyword)) { - return endpoints.stream().filter(e -> e.getName().contains(keyword)).collect(Collectors.toList()); - } return endpoints; } diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java index 8d8d053388..fbf6077e0b 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java @@ -19,6 +19,7 @@ package org.apache.skywalking.oap.server.storage.plugin.banyandb.stream; import com.google.gson.Gson; +import org.apache.skywalking.banyandb.model.v1.BanyandbModel; import org.apache.skywalking.banyandb.v1.client.AbstractCriteria; import org.apache.skywalking.banyandb.v1.client.AbstractQuery; import org.apache.skywalking.banyandb.v1.client.And; @@ -297,6 +298,14 @@ public abstract class AbstractBanyanDBDAO extends AbstractDAO match(String name, String value) { + return PairQueryCondition.StringQueryCondition.match(name, value); + } + + protected PairQueryCondition match(String name, String value, BanyandbModel.Condition.MatchOption matchOption) { + return PairQueryCondition.StringQueryCondition.match(name, value, matchOption); + } + protected PairQueryCondition> in(String name, List values) { return PairQueryCondition.StringArrayQueryCondition.in(name, values); } diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBAlarmQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBAlarmQueryDAO.java index 5a04c044e0..2b5d15932e 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBAlarmQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBAlarmQueryDAO.java @@ -32,6 +32,7 @@ import org.apache.skywalking.oap.server.core.query.type.AlarmMessage; import org.apache.skywalking.oap.server.core.query.type.Alarms; import org.apache.skywalking.oap.server.core.storage.query.IAlarmQueryDAO; import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.oap.server.library.util.StringUtil; import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBConverter; import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageClient; @@ -71,7 +72,9 @@ public class BanyanDBAlarmQueryDAO extends AbstractBanyanDBDAO implements IAlarm if (Objects.nonNull(scopeId)) { query.and(eq(AlarmRecord.SCOPE, (long) scopeId)); } - + if (StringUtil.isNotEmpty(keyword)) { + query.and(match(AlarmRecord.ALARM_MESSAGE, keyword)); + } if (CollectionUtils.isNotEmpty(tags)) { List tagsConditions = new ArrayList<>(tags.size()); for (final Tag tag : tags) { diff --git a/test/e2e-v2/script/env b/test/e2e-v2/script/env index a91bbc4222..719ae1fe48 100644 --- a/test/e2e-v2/script/env +++ b/test/e2e-v2/script/env @@ -23,7 +23,7 @@ SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449 SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016 SW_KUBERNETES_COMMIT_SHA=1335f15bf821a40a7cd71448fa805f0be265afcc SW_ROVER_COMMIT=6bbd39aa701984482330d9dfb4dbaaff0527d55c -SW_BANYANDB_COMMIT=0e734c462571dcf55dbb7761211c07d8b156521e +SW_BANYANDB_COMMIT=9770bf48f9b88574683f56e2a137555cd68066d9 SW_AGENT_PHP_COMMIT=3192c553002707d344bd6774cfab5bc61f67a1d3 SW_CTL_COMMIT=d5f3597733aa5217373986d776a3ee5ee8b3c468