From 0a032ced18af4be79772930b538d3c698d415f7d Mon Sep 17 00:00:00 2001 From: Wan Kai Date: Wed, 3 Aug 2022 21:18:30 +0800 Subject: [PATCH] Elasticsearch storage support merge all metrics/meter indices into one physical index (#9416) --- .github/workflows/skywalking.yaml | 2 + docs/en/changes/changes.md | 4 + docs/en/setup/backend/backend-storage.md | 5 + .../setup/backend/configuration-vocabulary.md | 3 +- .../manual/endpoint/EndpointTraffic.java | 2 +- .../manual/instance/InstanceTraffic.java | 2 +- .../manual/service/ServiceTraffic.java | 2 +- .../meter/function/PercentileFunction.java | 2 +- .../function/avg/AvgHistogramFunction.java | 4 +- .../avg/AvgHistogramPercentileFunction.java | 6 +- .../function/avg/AvgLabeledFunction.java | 6 +- .../sum/SumHistogramPercentileFunction.java | 6 +- .../core/analysis/metrics/ApdexMetrics.java | 2 +- .../analysis/metrics/DoubleAvgMetrics.java | 4 +- .../core/analysis/metrics/PercentMetrics.java | 2 +- .../analysis/metrics/PercentileMetrics.java | 2 +- ...BPFProcessProfilingScheduleDispatcher.java | 7 +- .../storage/EBPFProfilingScheduleRecord.java | 15 +- .../oap/server/core/storage/model/Model.java | 6 +- .../core/storage/model/StorageModels.java | 3 +- .../src/main/resources/application.yml | 2 + .../measure/BanyanDBTopologyQueryDAO.java | 3 +- .../StorageModuleElasticsearchConfig.java | 8 ++ .../StorageModuleElasticsearchProvider.java | 2 + .../elasticsearch/base/IndexController.java | 72 ++++++++-- .../base/StorageEsInstaller.java | 11 +- .../cache/NetworkAddressAliasEsDAO.java | 22 +-- .../query/AggregationQueryEsDAO.java | 4 +- .../query/EBPFProfilingScheduleEsDAO.java | 8 +- .../query/MetadataQueryEsDAO.java | 37 ++++- .../query/MetricsQueryEsDAO.java | 6 +- .../query/ServiceLabelEsDAO.java | 6 +- .../query/TagAutoCompleteQueryDAO.java | 6 + .../query/TopologyQueryEsDAO.java | 69 ++++++---- .../query/zipkin/ZipkinQueryEsDAO.java | 9 ++ .../base/TimeSeriesUtilsTest.java | 12 +- .../ebpf/oncpu/opensearch/docker-compose.yml | 3 +- test/e2e-v2/cases/storage/banyandb/e2e.yaml | 5 +- test/e2e-v2/cases/storage/es/e2e.yaml | 5 +- .../storage/es/es-sharding/docker-compose.yml | 76 +++++++++++ .../cases/storage/es/es-sharding/e2e.yaml | 50 +++++++ .../expected/dependency-endpoint-consumer.yml | 53 +++++++ .../expected/dependency-endpoint-provider.yml | 40 ++++++ .../storage/expected/dependency-instance.yml | 41 ++++++ ...s.yml => dependency-services-consumer.yml} | 19 ++- .../expected/dependency-services-provider.yml | 48 +++++++ .../expected/metrics-has-value-percentile.yml | 47 +++++++ .../storage/expected/metrics-has-value.yml | 2 +- ...oint.yml => service-endpoint-consumer.yml} | 6 +- .../expected/service-endpoint-provider.yml | 19 +++ .../expected/service-instance-consumer.yml | 40 ++++++ ...ance.yml => service-instance-provider.yml} | 6 +- .../storage/expected/trace-info-detail.yml | 68 --------- .../storage/expected/trace-users-detail.yml | 54 +++++++- .../cases/storage/expected/traces-list.yml | 6 +- test/e2e-v2/cases/storage/h2/e2e.yaml | 5 +- test/e2e-v2/cases/storage/mysql/e2e.yaml | 5 +- test/e2e-v2/cases/storage/opensearch/e2e.yaml | 5 +- test/e2e-v2/cases/storage/postgres/e2e.yaml | 5 +- test/e2e-v2/cases/storage/storage-cases.yaml | 129 ++++++++++++++---- test/e2e-v2/cases/storage/tidb/e2e.yaml | 5 +- 61 files changed, 894 insertions(+), 210 deletions(-) create mode 100644 test/e2e-v2/cases/storage/es/es-sharding/docker-compose.yml create mode 100644 test/e2e-v2/cases/storage/es/es-sharding/e2e.yaml create mode 100644 test/e2e-v2/cases/storage/expected/dependency-endpoint-consumer.yml create mode 100644 test/e2e-v2/cases/storage/expected/dependency-endpoint-provider.yml create mode 100644 test/e2e-v2/cases/storage/expected/dependency-instance.yml rename test/e2e-v2/cases/storage/expected/{dependency-services.yml => dependency-services-consumer.yml} (82%) create mode 100644 test/e2e-v2/cases/storage/expected/dependency-services-provider.yml create mode 100644 test/e2e-v2/cases/storage/expected/metrics-has-value-percentile.yml rename test/e2e-v2/cases/storage/expected/{service-endpoint.yml => service-endpoint-consumer.yml} (87%) create mode 100644 test/e2e-v2/cases/storage/expected/service-endpoint-provider.yml create mode 100644 test/e2e-v2/cases/storage/expected/service-instance-consumer.yml rename test/e2e-v2/cases/storage/expected/{service-instance.yml => service-instance-provider.yml} (95%) delete mode 100644 test/e2e-v2/cases/storage/expected/trace-info-detail.yml diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index 9adb41c716..8db48051ec 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -322,6 +322,8 @@ jobs: - name: Storage OpenSearch 1.2.0 config: test/e2e-v2/cases/storage/opensearch/e2e.yaml env: OPENSEARCH_VERSION=1.2.0 + - name: Storage ES Sharding + config: test/e2e-v2/cases/storage/es/es-sharding/e2e.yaml - name: Alarm H2 config: test/e2e-v2/cases/alarm/h2/e2e.yaml diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 0ff13f0df7..7b190dc656 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -29,6 +29,10 @@ * Support `SUM_PER_MIN` downsampling in `MAL`. * Support `sumHistogramPercentile` in `MAL`. * Add `VIRTUAL_CACHE` to Layer, to fix conjectured Redis server, which icon can't show on the topology. +* [Breaking Change] Elasticsearch storage merge all metrics/meter indices into one physical index template `metrics-all` on the default setting. + Provide system environment variable(`SW_STORAGE_ES_LOGIC_SHARDING`) to shard metrics indices into multi-physical indices as the previous versions(one index template per metric/meter aggregation function). + In the current one index mode, users still could choose to adjust ElasticSearch's shard number(`SW_STORAGE_ES_INDEX_SHARDS_NUMBER`) to scale out. +* [Breaking Change] Many columns of metrics model names are changed, The H2/Mysql/Tidb/Postgres storage users are required to remove all metrics-related tables for OAP to re-create or use a new database instance. #### UI diff --git a/docs/en/setup/backend/backend-storage.md b/docs/en/setup/backend/backend-storage.md index 32e374bb92..6f9e27b0ab 100644 --- a/docs/en/setup/backend/backend-storage.md +++ b/docs/en/setup/backend/backend-storage.md @@ -49,6 +49,10 @@ Since 8.8.0, SkyWalking rebuilds the ElasticSearch client on top of ElasticSearc correct request formats according to the server-side version, hence you don't need to download different binaries and don't need to configure different storage selectors for different ElasticSearch server-side versions anymore. +Since 9.2.0, SkyWalking merges all metrics/meter indices into one physical index template `metrics-all` on the default setting. +Provide system environment variable(`SW_STORAGE_ES_LOGIC_SHARDING`) to shard metrics indices into multi-physical indices as the previous versions(one index template per metric/meter aggregation function). +In the current one index mode, users still could choose to adjust ElasticSearch's shard number(`SW_STORAGE_ES_INDEX_SHARDS_NUMBER`) to scale out. + For now, SkyWalking supports ElasticSearch 6.x, ElasticSearch 7.x, ElasticSearch 8.x, and OpenSearch 1.x, their configurations are as follows: @@ -83,6 +87,7 @@ storage: oapAnalyzer: ${SW_STORAGE_ES_OAP_ANALYZER:"{\"analyzer\":{\"oap_analyzer\":{\"type\":\"stop\"}}}"} # the oap analyzer. oapLogAnalyzer: ${SW_STORAGE_ES_OAP_LOG_ANALYZER:"{\"analyzer\":{\"oap_log_analyzer\":{\"type\":\"standard\"}}}"} # the oap log analyzer. It could be customized by the ES analyzer configuration to support more language log formats, such as Chinese log, Japanese log and etc. advanced: ${SW_STORAGE_ES_ADVANCED:""} + logicSharding: ${SW_STORAGE_ES_LOGIC_SHARDING:false} ``` ### ElasticSearch With Https SSL Encrypting communications. diff --git a/docs/en/setup/backend/configuration-vocabulary.md b/docs/en/setup/backend/configuration-vocabulary.md index 85ec7ecec4..babb14c47d 100644 --- a/docs/en/setup/backend/configuration-vocabulary.md +++ b/docs/en/setup/backend/configuration-vocabulary.md @@ -114,6 +114,7 @@ The Configuration Vocabulary lists all available configurations provided by `app | - | - | profileTaskQueryMaxSize | The maximum size of profile task per query. | SW_STORAGE_ES_QUERY_PROFILE_TASK_SIZE | 200 | | - | - | profileDataQueryScrollBatchSize | The batch size of query profiling data. | SW_STORAGE_ES_QUERY_PROFILE_DATA_BATCH_SIZE | 100 | | - | - | advanced | All settings of ElasticSearch index creation. The value should be in JSON format. | SW_STORAGE_ES_ADVANCED | - | +| - | - | logicSharding | Shard metrics indices into multi-physical indices, one index template per metric/meter aggregation function. | SW_STORAGE_ES_LOGIC_SHARDING | false | | - | h2 | - | H2 storage is designed for demonstration and running in short term (i.e. 1-2 hours) only. | - | - | | - | - | driver | H2 JDBC driver. | SW_STORAGE_H2_DRIVER | org.h2.jdbcx.JdbcDataSource | | - | - | url | H2 connection URL. Defaults to H2 memory mode. | SW_STORAGE_H2_URL | jdbc:h2:mem:skywalking-oap-db | @@ -238,7 +239,7 @@ The Configuration Vocabulary lists all available configurations provided by `app | - | - | enableLogTestTool | Enable the log testing API to test the LAL. **NOTE**: This API evaluates untrusted code on the OAP server. A malicious script can do significant damage (steal keys and secrets, remove files and directories, install malware, etc). As such, please enable this API only when you completely trust your users. | SW_QUERY_GRAPHQL_ENABLE_LOG_TEST_TOOL | false | | - | - | maxQueryComplexity | Maximum complexity allowed for the GraphQL query that can be used to abort a query if the total number of data fields queried exceeds the defined threshold. | SW_QUERY_MAX_QUERY_COMPLEXITY | 1000 | | - | - | enableUpdateUITemplate | Allow user add，disable and update UI template. | SW_ENABLE_UPDATE_UI_TEMPLATE | false | -| - | - | enableOnDemandPodLog | Ondemand Pod log: fetch the Pod logs on users' demand, the logs are fetched and displayed in real time, and are not persisted in any kind. This is helpful when users want to do some experiments and monitor the logs and see what's happing inside the service. Note: if you print secrets in the logs, they are also visible to the UI, so for the sake of security, this feature is disabled by default, please set this configuration to enable the feature manually. | SW_ENABLE_ON_DEMAND_POD_LOG | false | +| - | - | enableOnDemandPodLog | Ondemand Pod log: fetch the Pod logs on users' demand, the logs are fetched and displayed in real time, and are not persisted in any kind. This is helpful when users want to do some experiments and monitor the logs and see what's happing inside the service. Note: if you print secrets in the logs, they are also visible to the UI, so for the sake of security, this feature is disabled by default, please set this configuration to enable the feature manually. | SW_ENABLE_ON_DEMAND_POD_LOG | false | | query | graphql | - | GraphQL query implementation. | - | | | - | - | restHost | Binding IP of RESTful services. | SW_QUERY_ZIPKIN_REST_HOST | 0.0.0.0 | | - | - | restPort | Binding port of RESTful services. | SW_QUERY_ZIPKIN_REST_PORT | 9412 | diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java index d0189e2c1f..fa67c3358d 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java @@ -45,7 +45,7 @@ public class EndpointTraffic extends Metrics { public static final String INDEX_NAME = "endpoint_traffic"; public static final String SERVICE_ID = "service_id"; - public static final String NAME = "name"; + public static final String NAME = "endpoint_traffic_name"; @Setter @Getter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/instance/InstanceTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/instance/InstanceTraffic.java index ef38bc995f..14098da9ce 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/instance/InstanceTraffic.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/instance/InstanceTraffic.java @@ -48,7 +48,7 @@ import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SE public class InstanceTraffic extends Metrics { public static final String INDEX_NAME = "instance_traffic"; public static final String SERVICE_ID = "service_id"; - public static final String NAME = "name"; + public static final String NAME = "instance_traffic_name"; public static final String LAST_PING_TIME_BUCKET = "last_ping"; public static final String PROPERTIES = "properties"; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java index 9e12488318..c7bd000695 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java @@ -49,7 +49,7 @@ import static org.apache.skywalking.oap.server.core.Const.DOUBLE_COLONS_SPLIT; public class ServiceTraffic extends Metrics { public static final String INDEX_NAME = "service_traffic"; - public static final String NAME = "name"; + public static final String NAME = "service_traffic_name"; public static final String SHORT_NAME = "short_name"; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/PercentileFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/PercentileFunction.java index 31734d8a8d..6fa1656838 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/PercentileFunction.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/PercentileFunction.java @@ -52,7 +52,7 @@ import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; public abstract class PercentileFunction extends Meter implements AcceptableValue, MultiIntValuesHolder { public static final String DATASET = "dataset"; public static final String RANKS = "ranks"; - public static final String VALUE = "value"; + public static final String VALUE = "datatable_value"; @Setter @Getter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramFunction.java index d71f352777..6c487013c7 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramFunction.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramFunction.java @@ -55,8 +55,8 @@ import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; @ToString public abstract class AvgHistogramFunction extends Meter implements AcceptableValue { public static final String DATASET = "dataset"; - protected static final String SUMMATION = "summation"; - protected static final String COUNT = "count"; + protected static final String SUMMATION = "datatable_summation"; + protected static final String COUNT = "datatable_count"; @Setter @Getter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramPercentileFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramPercentileFunction.java index e87a817033..71aa4c3bbb 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramPercentileFunction.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramPercentileFunction.java @@ -69,9 +69,9 @@ public abstract class AvgHistogramPercentileFunction extends Meter implements Ac private static final String DEFAULT_GROUP = "pD"; public static final String DATASET = "dataset"; public static final String RANKS = "ranks"; - public static final String VALUE = "value"; - protected static final String SUMMATION = "summation"; - protected static final String COUNT = "count"; + public static final String VALUE = "datatable_value"; + protected static final String SUMMATION = "datatable_summation"; + protected static final String COUNT = "datatable_count"; @Setter @Getter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgLabeledFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgLabeledFunction.java index 0bde64db62..b69c956b28 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgLabeledFunction.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgLabeledFunction.java @@ -43,9 +43,9 @@ import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; @MeterFunction(functionName = "avgLabeled") @ToString public abstract class AvgLabeledFunction extends Meter implements AcceptableValue, LabeledValueHolder { - protected static final String SUMMATION = "summation"; - protected static final String COUNT = "count"; - protected static final String VALUE = "value"; + protected static final String SUMMATION = "datatable_summation"; + protected static final String COUNT = "datatable_count"; + protected static final String VALUE = "datatable_value"; @Setter @Getter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/sum/SumHistogramPercentileFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/sum/SumHistogramPercentileFunction.java index 497c85a40b..b3750352ba 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/sum/SumHistogramPercentileFunction.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/sum/SumHistogramPercentileFunction.java @@ -61,8 +61,8 @@ public abstract class SumHistogramPercentileFunction extends Meter implements Ac private static final String DEFAULT_GROUP = "pD"; public static final String DATASET = "dataset"; public static final String RANKS = "ranks"; - public static final String VALUE = "value"; - protected static final String SUMMATION = "summation"; + public static final String VALUE = "datatable_value"; + protected static final String SUMMATION = "datatable_summation"; @Setter @Getter @@ -340,4 +340,4 @@ public abstract class SumHistogramPercentileFunction extends Meter implements Ac public int hashCode() { return Objects.hash(entityId, getTimeBucket()); } -} \ No newline at end of file +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ApdexMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ApdexMetrics.java index 14b4750bc8..1fb724df30 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ApdexMetrics.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ApdexMetrics.java @@ -44,7 +44,7 @@ public abstract class ApdexMetrics extends Metrics implements IntValueHolder { protected static final String S_NUM = "s_num"; // Level: tolerated protected static final String T_NUM = "t_num"; - protected static final String VALUE = "value"; + protected static final String VALUE = "int_value"; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DoubleAvgMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DoubleAvgMetrics.java index 9356c21198..f11036a36d 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DoubleAvgMetrics.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DoubleAvgMetrics.java @@ -30,9 +30,9 @@ import org.apache.skywalking.oap.server.core.storage.annotation.Column; @MetricsFunction(functionName = "doubleAvg") public abstract class DoubleAvgMetrics extends Metrics implements DoubleValueHolder { - protected static final String SUMMATION = "summation"; + protected static final String SUMMATION = "double_summation"; protected static final String COUNT = "count"; - protected static final String VALUE = "value"; + protected static final String VALUE = "double_value"; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentMetrics.java index aa6d32f96f..2efd2a6ee3 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentMetrics.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentMetrics.java @@ -34,7 +34,7 @@ public abstract class PercentMetrics extends Metrics implements IntValueHolder { @Getter @Setter - @Column(columnName = TOTAL) + @Column(columnName = TOTAL, storageOnly = true) private long total; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentileMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentileMetrics.java index 1f3ca142bc..3808330bee 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentileMetrics.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentileMetrics.java @@ -38,7 +38,7 @@ import org.apache.skywalking.oap.server.core.storage.annotation.Column; @MetricsFunction(functionName = "percentile") public abstract class PercentileMetrics extends Metrics implements MultiIntValuesHolder { protected static final String DATASET = "dataset"; - protected static final String VALUE = "value"; + protected static final String VALUE = "datatable_value"; protected static final String PRECISION = "precision"; private static final int[] RANKS = { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProcessProfilingScheduleDispatcher.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProcessProfilingScheduleDispatcher.java index 10e3fd11dd..c19f1326bd 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProcessProfilingScheduleDispatcher.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProcessProfilingScheduleDispatcher.java @@ -18,6 +18,8 @@ package org.apache.skywalking.oap.server.core.profiling.ebpf.storage; +import com.google.common.hash.Hashing; +import com.linecorp.armeria.internal.shaded.guava.base.Charsets; import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor; @@ -32,6 +34,9 @@ public class EBPFProcessProfilingScheduleDispatcher implements SourceDispatcher< traffic.setStartTime(source.getStartTime()); traffic.setEndTime(source.getCurrentTime()); traffic.setTimeBucket(TimeBucket.getMinuteTimeBucket(source.getCurrentTime())); + traffic.setScheduleId(Hashing.sha256().newHasher() + .putString(String.format("%s_%s_%d", source.getTaskId(), source.getProcessId(), source.getStartTime()), Charsets.UTF_8) + .hash().toString()); MetricsStreamProcessor.getInstance().in(traffic); } -} \ No newline at end of file +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingScheduleRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingScheduleRecord.java index 494627e4e6..253ea9ccd0 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingScheduleRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingScheduleRecord.java @@ -18,8 +18,6 @@ package org.apache.skywalking.oap.server.core.profiling.ebpf.storage; -import com.google.common.hash.Hashing; -import com.linecorp.armeria.internal.shaded.guava.base.Charsets; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; @@ -57,6 +55,7 @@ public class EBPFProfilingScheduleRecord extends Metrics { public static final String PROCESS_ID = "process_id"; public static final String START_TIME = "start_time"; public static final String END_TIME = "end_time"; + public static final String EBPF_PROFILING_SCHEDULE_ID = "ebpf_profiling_schedule_id"; @Column(columnName = TASK_ID, length = 600) private String taskId; @@ -66,6 +65,8 @@ public class EBPFProfilingScheduleRecord extends Metrics { private long startTime; @Column(columnName = END_TIME) private long endTime; + @Column(columnName = EBPF_PROFILING_SCHEDULE_ID) + private String scheduleId; @Override public boolean combine(Metrics metrics) { @@ -92,15 +93,14 @@ public class EBPFProfilingScheduleRecord extends Metrics { @Override protected String id0() { - return Hashing.sha256().newHasher() - .putString(String.format("%s_%s_%d", taskId, processId, startTime), Charsets.UTF_8) - .hash().toString(); + return scheduleId; } @Override public void deserialize(RemoteData remoteData) { setTaskId(remoteData.getDataStrings(0)); setProcessId(remoteData.getDataStrings(1)); + setScheduleId(remoteData.getDataStrings(2)); setStartTime(remoteData.getDataLongs(0)); setEndTime(remoteData.getDataLongs(1)); setTimeBucket(remoteData.getDataLongs(2)); @@ -111,6 +111,7 @@ public class EBPFProfilingScheduleRecord extends Metrics { final RemoteData.Builder builder = RemoteData.newBuilder(); builder.addDataStrings(taskId); builder.addDataStrings(processId); + builder.addDataStrings(scheduleId); builder.addDataLongs(startTime); builder.addDataLongs(endTime); builder.addDataLongs(getTimeBucket()); @@ -129,6 +130,7 @@ public class EBPFProfilingScheduleRecord extends Metrics { final EBPFProfilingScheduleRecord executeTraffic = new EBPFProfilingScheduleRecord(); executeTraffic.setTaskId((String) converter.get(TASK_ID)); executeTraffic.setProcessId((String) converter.get(PROCESS_ID)); + executeTraffic.setScheduleId((String) converter.get(EBPF_PROFILING_SCHEDULE_ID)); executeTraffic.setStartTime(((Number) converter.get(START_TIME)).longValue()); executeTraffic.setEndTime(((Number) converter.get(END_TIME)).longValue()); executeTraffic.setTimeBucket(((Number) converter.get(TIME_BUCKET)).longValue()); @@ -139,9 +141,10 @@ public class EBPFProfilingScheduleRecord extends Metrics { public void entity2Storage(final EBPFProfilingScheduleRecord storageData, final Convert2Storage converter) { converter.accept(TASK_ID, storageData.getTaskId()); converter.accept(PROCESS_ID, storageData.getProcessId()); + converter.accept(EBPF_PROFILING_SCHEDULE_ID, storageData.getScheduleId()); converter.accept(START_TIME, storageData.getStartTime()); converter.accept(END_TIME, storageData.getEndTime()); converter.accept(TIME_BUCKET, storageData.getTimeBucket()); } } -} \ No newline at end of file +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/Model.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/Model.java index eea3a61f80..4e37d756f3 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/Model.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/Model.java @@ -36,7 +36,7 @@ public class Model { private final boolean record; private final boolean superDataset; private final boolean isTimeSeries; - private final String aggregationFunctionName; + private final Class streamClass; private final boolean timeRelativeID; private final SQLDatabaseModelExtension sqlDBModelExtension; @@ -46,7 +46,7 @@ public class Model { final DownSampling downsampling, final boolean record, final boolean superDataset, - final String aggregationFunctionName, + final Class streamClass, boolean timeRelativeID, final SQLDatabaseModelExtension sqlDBModelExtension) { this.name = name; @@ -56,7 +56,7 @@ public class Model { this.isTimeSeries = !DownSampling.None.equals(downsampling); this.record = record; this.superDataset = superDataset; - this.aggregationFunctionName = aggregationFunctionName; + this.streamClass = streamClass; this.timeRelativeID = timeRelativeID; this.sqlDBModelExtension = sqlDBModelExtension; } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java index bbf223b8ce..28a47de1ea 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java @@ -24,7 +24,6 @@ import java.util.HashMap; import java.util.List; import java.util.Objects; import lombok.extern.slf4j.Slf4j; -import org.apache.skywalking.oap.server.core.analysis.FunctionCategory; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; import org.apache.skywalking.oap.server.core.storage.StorageException; import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB; @@ -69,7 +68,7 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula storage.getDownsampling(), record, isSuperDatasetModel(aClass), - FunctionCategory.uniqueFunctionName(aClass), + aClass, storage.isTimeRelativeID(), sqlDBModelExtension ); diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index 632bc702a9..a5294394d4 100755 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -166,6 +166,8 @@ storage: oapAnalyzer: ${SW_STORAGE_ES_OAP_ANALYZER:"{\"analyzer\":{\"oap_analyzer\":{\"type\":\"stop\"}}}"} # the oap analyzer. oapLogAnalyzer: ${SW_STORAGE_ES_OAP_LOG_ANALYZER:"{\"analyzer\":{\"oap_log_analyzer\":{\"type\":\"standard\"}}}"} # the oap log analyzer. It could be customized by the ES analyzer configuration to support more language log formats, such as Chinese log, Japanese log and etc. advanced: ${SW_STORAGE_ES_ADVANCED:""} + # Enable shard metrics indices into multi-physical indices, one index template per metric/meter aggregation function. + logicSharding: ${SW_STORAGE_ES_LOGIC_SHARDING:false} h2: driver: ${SW_STORAGE_H2_DRIVER:org.h2.jdbcx.JdbcDataSource} url: ${SW_STORAGE_H2_URL:jdbc:h2:mem:skywalking-oap-db;DB_CLOSE_DELAY=-1} diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTopologyQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTopologyQueryDAO.java index 5dfd13d404..7afc5c80e2 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTopologyQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTopologyQueryDAO.java @@ -25,6 +25,7 @@ import org.apache.skywalking.banyandb.v1.client.TimestampRange; import org.apache.skywalking.oap.server.core.UnexpectedException; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.manual.relation.endpoint.EndpointRelationServerSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.instance.ServiceInstanceRelationClientSideMetrics; import org.apache.skywalking.oap.server.core.analysis.manual.relation.instance.ServiceInstanceRelationServerSideMetrics; import org.apache.skywalking.oap.server.core.analysis.manual.relation.process.ProcessRelationClientSideMetrics; import org.apache.skywalking.oap.server.core.analysis.manual.relation.process.ProcessRelationServerSideMetrics; @@ -169,7 +170,7 @@ public class BanyanDBTopologyQueryDAO extends AbstractBanyanDBDAO implements ITo timestampRange = new TimestampRange(TimeBucket.getTimestamp(startTB), TimeBucket.getTimestamp(endTB)); } final String modelName = detectPoint == DetectPoint.SERVER ? ServiceInstanceRelationServerSideMetrics.INDEX_NAME : - ServiceRelationClientSideMetrics.INDEX_NAME; + ServiceInstanceRelationClientSideMetrics.INDEX_NAME; final Map callMap = new HashMap<>(); for (final QueryBuilder q : queryBuilderList) { MeasureQueryResponse resp = query(modelName, diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java index 50fccf6aab..8010ace25e 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java @@ -142,4 +142,12 @@ public class StorageModuleElasticsearchConfig extends ModuleConfig { * If the value is <= 0, the number of available processors will be used. */ private int numHttpClientThread; + + /** + * If disabled, all metrics would be persistent in one physical index template, to reduce the number of physical indices. + * If enabled, shard metrics indices into multi-physical indices, one index template per metric/meter aggregation function. + * + * @since 9.2.0 + */ + private boolean logicSharding = false; } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java index 679ba121a8..451c481f58 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java @@ -60,6 +60,7 @@ import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedExcepti import org.apache.skywalking.oap.server.library.util.MultipleFilesChangeMonitor; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.BatchProcessEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.HistoryDeleteEsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.StorageEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.StorageEsInstaller; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.TimeSeriesUtils; @@ -217,6 +218,7 @@ public class StorageModuleElasticsearchProvider extends ModuleProvider { ITagAutoCompleteQueryDAO.class, new TagAutoCompleteQueryDAO(elasticSearchClient)); this.registerServiceImplementation( IZipkinQueryDAO.class, new ZipkinQueryEsDAO(elasticSearchClient)); + IndexController.INSTANCE.setLogicSharding(config.isLogicSharding()); } @Override diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexController.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexController.java index 90c05ed64c..96d8bb291a 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexController.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexController.java @@ -18,13 +18,20 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.core.analysis.FunctionCategory; +import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; +import org.apache.skywalking.oap.server.core.storage.model.ModelColumn; import org.apache.skywalking.oap.server.core.Const; import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.library.util.StringUtil; /** * The metrics data, that generated by OAL or MAL, would be partitioned to storage by the functions of the OAL or MAL. @@ -33,9 +40,18 @@ import org.apache.skywalking.oap.server.core.storage.model.Model; @Slf4j public enum IndexController { INSTANCE; + /** + * Init in StorageModuleElasticsearchProvider.prepare() and the value from the config. + */ + @Setter + private boolean logicSharding = false; public String getTableName(Model model) { - return isMetricModel(model) ? model.getAggregationFunctionName() : model.getName(); + if (!logicSharding) { + return isMetricModel(model) ? "metrics-all" : model.getName(); + } + String aggFuncName = FunctionCategory.uniqueFunctionName(model.getStreamClass()); + return StringUtil.isNotBlank(aggFuncName) ? aggFuncName : model.getName(); } /** @@ -46,6 +62,9 @@ public enum IndexController { if (!isMetricModel(model)) { return originalID; } + if (logicSharding && !isFunctionMetric(model)) { + return originalID; + } return this.generateDocId(model.getName(), originalID); } @@ -60,12 +79,16 @@ public enum IndexController { * Check the mode of the Model definition. */ public boolean isMetricModel(Model model) { - return StringUtil.isNotBlank(model.getAggregationFunctionName()); + return Metrics.class.isAssignableFrom(model.getStreamClass()); + } + + public boolean isFunctionMetric(Model model) { + return StringUtil.isNotBlank(FunctionCategory.uniqueFunctionName(model.getStreamClass())); } /** * When a model is the metric storage mode, a column named {@link LogicIndicesRegister#METRIC_TABLE_NAME} would be - * append to the physical index. The value of the column is the original table name in other storages, such as the + * appended to the physical index. The value of the column is the original table name in other storages, such as the * OAL name. */ public Map appendMetricTableColumn(Model model, Map columns) { @@ -81,7 +104,10 @@ public enum IndexController { /** * The relations of the logic table and the physical table. */ - private static final Map LOGIC_INDICES_CATALOG = new ConcurrentHashMap<>(); + private static final Map LOGIC_INDICES_CATALOG = new HashMap<>(); + + private static final Map> PHYSICAL_INDICES_COLUMNS = new HashMap<>(); + /** * The metric table name in aggregation physical storage. */ @@ -91,12 +117,42 @@ public enum IndexController { return Optional.ofNullable(LOGIC_INDICES_CATALOG.get(logicName)).orElse(logicName); } - public static void registerRelation(String logicName, String physicalName) { - LOGIC_INDICES_CATALOG.put(logicName, physicalName); + public static void registerRelation(Model model, String physicalName) { + LOGIC_INDICES_CATALOG.put(model.getName(), physicalName); + Map columns = PHYSICAL_INDICES_COLUMNS.computeIfAbsent( + physicalName, v -> new ConcurrentHashMap<>()); + model.getColumns().forEach(modelColumn -> { + String columnName = modelColumn.getColumnName().getName(); + if (columns.containsKey(columnName)) { + checkModelColumnConflicts(columns.get(columnName), modelColumn, physicalName); + } else { + columns.put(columnName, modelColumn); + } + }); } - public static boolean isMetricTable(String logicName) { + public static boolean isPhysicalTable(String logicName) { return !getPhysicalTableName(logicName).equals(logicName); } + + public static List getPhysicalTableColumns(Model model) { + String tableName = getPhysicalTableName(model.getName()); + return new ArrayList<>(PHYSICAL_INDICES_COLUMNS.get(tableName).values()); + } + + private static void checkModelColumnConflicts(ModelColumn mc1, ModelColumn mc2, String physicalName) { + if (!(mc1.isIndexOnly() == mc2.isIndexOnly())) { + throw new IllegalArgumentException(mc1.getColumnName() + " and " + mc2.getColumnName() + " isIndexOnly conflict in index: " + physicalName); + } + if (!(mc1.isStorageOnly() == mc2.isStorageOnly())) { + throw new IllegalArgumentException(mc1.getColumnName() + " and " + mc2.getColumnName() + " isStorageOnly conflict in index: " + physicalName); + } + if (!mc1.getType().equals(mc2.getType())) { + throw new IllegalArgumentException(mc1.getColumnName() + " and " + mc2.getColumnName() + " Class type conflict in index: " + physicalName); + } + if (!(mc1.getElasticSearchExtension().needMatchQuery() == mc2.getElasticSearchExtension().needMatchQuery())) { + throw new IllegalArgumentException(mc1.getColumnName() + " and " + mc2.getColumnName() + " needMatchQuery conflict in index: " + physicalName); + } + } } } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java index 1256f3fea3..22f60003ed 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java @@ -44,6 +44,7 @@ public class StorageEsInstaller extends ModelInstaller { private final StorageModuleElasticsearchConfig config; protected final ColumnTypeEsMapping columnTypeEsMapping; + /** * The mappings of the template . */ @@ -66,7 +67,7 @@ public class StorageEsInstaller extends ModelInstaller { protected boolean isExists(Model model) { ElasticSearchClient esClient = (ElasticSearchClient) client; String tableName = IndexController.INSTANCE.getTableName(model); - IndexController.LogicIndicesRegister.registerRelation(model.getName(), tableName); + IndexController.LogicIndicesRegister.registerRelation(model, tableName); if (!model.isTimeSeries()) { boolean exist = esClient.isExistsIndex(tableName); if (exist) { @@ -151,7 +152,7 @@ public class StorageEsInstaller extends ModelInstaller { tableName, settings, structures.getMapping(tableName), config.getIndexTemplateOrder()); log.info("create {} index template finished, isAcknowledged: {}", tableName, isAcknowledged); if (!isAcknowledged) { - throw new IOException("create " + tableName + " index template failure, "); + throw new IOException("create " + tableName + " index template failure"); } } @@ -203,7 +204,8 @@ public class StorageEsInstaller extends ModelInstaller { indexRefreshInterval = 5; } setting.put("index.refresh_interval", indexRefreshInterval + "s"); - setting.put("analysis", getAnalyzerSetting(model.getColumns())); + List columns = IndexController.LogicIndicesRegister.getPhysicalTableColumns(model); + setting.put("analysis", getAnalyzerSetting(columns)); if (!StringUtil.isEmpty(config.getAdvanced())) { Map advancedSettings = gson.fromJson(config.getAdvanced(), Map.class); setting.putAll(advancedSettings); @@ -258,7 +260,8 @@ public class StorageEsInstaller extends ModelInstaller { } } - if (IndexController.INSTANCE.isMetricModel(model)) { + if ((IndexController.INSTANCE.isMetricModel(model) && !config.isLogicSharding()) + || (config.isLogicSharding() && IndexController.INSTANCE.isFunctionMetric(model))) { Map column = new HashMap<>(); column.put("type", "keyword"); properties.put(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, column); diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/cache/NetworkAddressAliasEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/cache/NetworkAddressAliasEsDAO.java index e81ef85479..69b713a91f 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/cache/NetworkAddressAliasEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/cache/NetworkAddressAliasEsDAO.java @@ -23,6 +23,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder; import org.apache.skywalking.library.elasticsearch.requests.search.Query; import org.apache.skywalking.library.elasticsearch.requests.search.Search; import org.apache.skywalking.library.elasticsearch.requests.search.SearchParams; @@ -34,6 +35,7 @@ import org.apache.skywalking.oap.server.core.storage.type.HashMapConverter; import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.StorageModuleElasticsearchConfig; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; @Slf4j public class NetworkAddressAliasEsDAO extends EsDAO implements INetworkAddressAliasDAO { @@ -50,20 +52,24 @@ public class NetworkAddressAliasEsDAO extends EsDAO implements INetworkAddressAl @Override public List loadLastUpdate(long timeBucketInMinute) { List networkAddressAliases = new ArrayList<>(); - + final String index = + IndexController.LogicIndicesRegister.getPhysicalTableName(NetworkAddressAlias.INDEX_NAME); try { final int batchSize = Math.min(resultWindowMaxSize, scrollingBatchSize); - final Search search = - Search.builder().query( - Query.range(NetworkAddressAlias.LAST_UPDATE_TIME_BUCKET) - .gte(timeBucketInMinute)) - .size(batchSize) - .build(); + final BoolQueryBuilder query = Query.bool(); + if (IndexController.LogicIndicesRegister.isPhysicalTable(NetworkAddressAlias.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, NetworkAddressAlias.INDEX_NAME)); + } + query.must(Query.range(NetworkAddressAlias.LAST_UPDATE_TIME_BUCKET) + .gte(timeBucketInMinute)); + + final Search search = Search.builder().query(query).size(batchSize).build(); + final SearchParams params = new SearchParams().scroll(SCROLL_CONTEXT_RETENTION); final NetworkAddressAlias.Builder builder = new NetworkAddressAlias.Builder(); SearchResponse results = - getClient().search(NetworkAddressAlias.INDEX_NAME, search, params); + getClient().search(index, search, params); final Set scrollIds = new HashSet<>(); try { while (true) { diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AggregationQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AggregationQueryEsDAO.java index d99d4b0d8d..77cafcaf6d 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AggregationQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AggregationQueryEsDAO.java @@ -62,7 +62,7 @@ public class AggregationQueryEsDAO extends EsDAO implements IAggregationQueryDAO final boolean asc = condition.getOrder().equals(Order.ASC); if (CollectionUtils.isEmpty(additionalConditions) - && IndexController.LogicIndicesRegister.isMetricTable(condition.getName())) { + && IndexController.LogicIndicesRegister.isPhysicalTable(condition.getName())) { final BoolQueryBuilder boolQuery = Query.bool() .must(basicQuery) @@ -74,7 +74,7 @@ public class AggregationQueryEsDAO extends EsDAO implements IAggregationQueryDAO } else if (CollectionUtils.isEmpty(additionalConditions)) { search.query(basicQuery); } else if (CollectionUtils.isNotEmpty(additionalConditions) - && IndexController.LogicIndicesRegister.isMetricTable(condition.getName())) { + && IndexController.LogicIndicesRegister.isPhysicalTable(condition.getName())) { final BoolQueryBuilder boolQuery = Query.bool() .must(Query.term( diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/EBPFProfilingScheduleEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/EBPFProfilingScheduleEsDAO.java index af81702166..62f9cd69e4 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/EBPFProfilingScheduleEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/EBPFProfilingScheduleEsDAO.java @@ -50,7 +50,9 @@ public class EBPFProfilingScheduleEsDAO extends EsDAO implements IEBPFProfilingS final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(EBPFProfilingScheduleRecord.INDEX_NAME); final BoolQueryBuilder query = Query.bool(); - + if (IndexController.LogicIndicesRegister.isPhysicalTable(EBPFProfilingScheduleRecord.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, EBPFProfilingScheduleRecord.INDEX_NAME)); + } query.must(Query.term(EBPFProfilingScheduleRecord.TASK_ID, taskId)); query.must(Query.range(EBPFProfilingScheduleRecord.START_TIME)); final SearchBuilder search = Search.builder().query(query) @@ -63,11 +65,11 @@ public class EBPFProfilingScheduleEsDAO extends EsDAO implements IEBPFProfilingS private EBPFProfilingSchedule parseSchedule(final SearchHit hit) { final EBPFProfilingSchedule schedule = new EBPFProfilingSchedule(); - schedule.setScheduleId(hit.getId()); + schedule.setScheduleId((String) hit.getSource().get(EBPFProfilingScheduleRecord.EBPF_PROFILING_SCHEDULE_ID)); schedule.setTaskId((String) hit.getSource().get(EBPFProfilingScheduleRecord.TASK_ID)); schedule.setProcessId((String) hit.getSource().get(EBPFProfilingScheduleRecord.PROCESS_ID)); schedule.setStartTime(((Number) hit.getSource().get(EBPFProfilingScheduleRecord.START_TIME)).longValue()); schedule.setEndTime(((Number) hit.getSource().get(EBPFProfilingScheduleRecord.END_TIME)).longValue()); return schedule; } -} \ No newline at end of file +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryEsDAO.java index 5db92975fa..7265987077 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryEsDAO.java @@ -91,6 +91,9 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO { if (StringUtil.isNotEmpty(group)) { query.must(Query.term(ServiceTraffic.GROUP, group)); } + if (IndexController.LogicIndicesRegister.isPhysicalTable(ServiceTraffic.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ServiceTraffic.INDEX_NAME)); + } final SearchParams params = new SearchParams().scroll(SCROLL_CONTEXT_RETENTION); final List services = new ArrayList<>(); @@ -128,6 +131,9 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO { final BoolQueryBuilder query = Query.bool() .must(Query.term(ServiceTraffic.SERVICE_ID, serviceId)); + if (IndexController.LogicIndicesRegister.isPhysicalTable(ServiceTraffic.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ServiceTraffic.INDEX_NAME)); + } final SearchBuilder search = Search.builder().query(query).size(queryMaxSize); final SearchResponse response = getClient().search(index, search.build()); @@ -145,6 +151,9 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO { Query.bool() .must(Query.range(InstanceTraffic.LAST_PING_TIME_BUCKET).gte(minuteTimeBucket)) .must(Query.term(InstanceTraffic.SERVICE_ID, serviceId)); + if (IndexController.LogicIndicesRegister.isPhysicalTable(InstanceTraffic.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, InstanceTraffic.INDEX_NAME)); + } final int batchSize = Math.min(queryMaxSize, scrollingBatchSize); final SearchBuilder search = Search.builder().query(query).size(batchSize); @@ -168,9 +177,13 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO { public ServiceInstance getInstance(final String instanceId) throws IOException { final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(InstanceTraffic.INDEX_NAME); + String id = instanceId; + if (IndexController.LogicIndicesRegister.isPhysicalTable(InstanceTraffic.INDEX_NAME)) { + id = IndexController.INSTANCE.generateDocId(InstanceTraffic.INDEX_NAME, instanceId); + } final BoolQueryBuilder query = Query.bool() - .must(Query.term("_id", instanceId)); + .must(Query.term("_id", id)); final SearchBuilder search = Search.builder().query(query).size(1); final SearchResponse response = getClient().search(index, search.build()); @@ -193,6 +206,10 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO { query.must(Query.match(matchCName, keyword)); } + if (IndexController.LogicIndicesRegister.isPhysicalTable(EndpointTraffic.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, EndpointTraffic.INDEX_NAME)); + } + final SearchBuilder search = Search.builder().query(query).size(limit); final SearchResponse response = getClient().search(index, search.build()); @@ -219,6 +236,9 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO { IndexController.LogicIndicesRegister.getPhysicalTableName(ProcessTraffic.INDEX_NAME); final BoolQueryBuilder query = Query.bool(); + if (IndexController.LogicIndicesRegister.isPhysicalTable(ProcessTraffic.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ProcessTraffic.INDEX_NAME)); + } final SearchBuilder search = Search.builder().query(query).size(queryMaxSize); appendProcessWhereQuery(query, serviceId, null, null, supportStatus, lastPingStartTimeBucket, lastPingEndTimeBucket); final SearchResponse results = getClient().search(index, search.build()); @@ -232,6 +252,9 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO { IndexController.LogicIndicesRegister.getPhysicalTableName(ProcessTraffic.INDEX_NAME); final BoolQueryBuilder query = Query.bool(); + if (IndexController.LogicIndicesRegister.isPhysicalTable(ProcessTraffic.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ProcessTraffic.INDEX_NAME)); + } final SearchBuilder search = Search.builder().query(query).size(queryMaxSize); appendProcessWhereQuery(query, null, serviceInstanceId, null, null, lastPingStartTimeBucket, lastPingEndTimeBucket); final SearchResponse results = getClient().search(index, search.build()); @@ -245,6 +268,9 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO { IndexController.LogicIndicesRegister.getPhysicalTableName(ProcessTraffic.INDEX_NAME); final BoolQueryBuilder query = Query.bool(); + if (IndexController.LogicIndicesRegister.isPhysicalTable(ProcessTraffic.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ProcessTraffic.INDEX_NAME)); + } final SearchBuilder search = Search.builder().query(query).size(queryMaxSize); appendProcessWhereQuery(query, null, null, agentId, null, 0, 0); final SearchResponse results = getClient().search(index, search.build()); @@ -258,6 +284,9 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO { IndexController.LogicIndicesRegister.getPhysicalTableName(ProcessTraffic.INDEX_NAME); final BoolQueryBuilder query = Query.bool(); + if (IndexController.LogicIndicesRegister.isPhysicalTable(ProcessTraffic.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ProcessTraffic.INDEX_NAME)); + } final SearchBuilder search = Search.builder().query(query).size(0); appendProcessWhereQuery(query, serviceId, null, null, profilingSupportStatus, lastPingStartTimeBucket, lastPingEndTimeBucket); @@ -272,6 +301,9 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO { IndexController.LogicIndicesRegister.getPhysicalTableName(ProcessTraffic.INDEX_NAME); final BoolQueryBuilder query = Query.bool(); + if (IndexController.LogicIndicesRegister.isPhysicalTable(ProcessTraffic.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ProcessTraffic.INDEX_NAME)); + } final SearchBuilder search = Search.builder().query(query).size(0); appendProcessWhereQuery(query, null, instanceId, null, null, 0, 0); final SearchResponse results = getClient().search(index, search.build()); @@ -313,6 +345,9 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO { IndexController.LogicIndicesRegister.getPhysicalTableName(ProcessTraffic.INDEX_NAME); final BoolQueryBuilder query = Query.bool() .must(Query.term("_id", processId)); + if (IndexController.LogicIndicesRegister.isPhysicalTable(ProcessTraffic.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ProcessTraffic.INDEX_NAME)); + } final SearchBuilder search = Search.builder().query(query).size(queryMaxSize); final SearchResponse response = getClient().search(index, search.build()); diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java index 31cb215e76..6d194ee2ee 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java @@ -110,7 +110,7 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO { final List ids = pointOfTimes.stream().map(pointOfTime -> { String id = pointOfTime.id(condition.getEntity().buildId()); - if (IndexController.LogicIndicesRegister.isMetricTable(condition.getName())) { + if (IndexController.LogicIndicesRegister.isPhysicalTable(condition.getName())) { id = IndexController.INSTANCE.generateDocId(condition.getName(), id); } String indexName = TimeSeriesUtils.queryIndexName( @@ -255,7 +255,7 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO { final String entityId = condition.getEntity().buildId(); if (entityId == null && - IndexController.LogicIndicesRegister.isMetricTable(condition.getName())) { + IndexController.LogicIndicesRegister.isPhysicalTable(condition.getName())) { sourceBuilder.query( Query.bool() .must(rangeQueryBuilder) @@ -266,7 +266,7 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO { ); } else if (entityId == null) { sourceBuilder.query(rangeQueryBuilder); - } else if (IndexController.LogicIndicesRegister.isMetricTable(condition.getName())) { + } else if (IndexController.LogicIndicesRegister.isPhysicalTable(condition.getName())) { sourceBuilder.query( Query.bool() .must(rangeQueryBuilder) diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ServiceLabelEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ServiceLabelEsDAO.java index a024b396a7..0cb7ee5827 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ServiceLabelEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ServiceLabelEsDAO.java @@ -46,7 +46,9 @@ public class ServiceLabelEsDAO extends EsDAO implements IServiceLabelDAO { final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(ServiceLabelRecord.INDEX_NAME); final BoolQueryBuilder query = Query.bool(); - + if (IndexController.LogicIndicesRegister.isPhysicalTable(ServiceLabelRecord.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ServiceLabelRecord.INDEX_NAME)); + } query.must(Query.term(ServiceLabelRecord.SERVICE_ID, serviceId)); final SearchBuilder search = Search.builder().query(query).size(maxSize); @@ -57,4 +59,4 @@ public class ServiceLabelEsDAO extends EsDAO implements IServiceLabelDAO { private String parseLabel(final SearchHit hit) { return (String) hit.getSource().get(ServiceLabelRecord.LABEL); } -} \ No newline at end of file +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java index b9872cac41..c87e1b6ff1 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java @@ -52,6 +52,9 @@ public class TagAutoCompleteQueryDAO extends EsDAO implements ITagAutoCompleteQu final long endSecondTB) throws IOException { BoolQueryBuilder query = Query.bool(); query.must(Query.term(TagAutocompleteData.TAG_TYPE, tagType.name())); + if (IndexController.LogicIndicesRegister.isPhysicalTable(TagAutocompleteData.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, TagAutocompleteData.INDEX_NAME)); + } final SearchBuilder search = Search.builder().query(query); search.aggregation(Aggregation.terms(TagAutocompleteData.TAG_KEY) .field(TagAutocompleteData.TAG_KEY) @@ -85,6 +88,9 @@ public class TagAutoCompleteQueryDAO extends EsDAO implements ITagAutoCompleteQu final long endSecondTB) throws IOException { BoolQueryBuilder query = Query.bool().must(Query.term(TagAutocompleteData.TAG_KEY, tagKey)); query.must(Query.term(TagAutocompleteData.TAG_TYPE, tagType.name())); + if (IndexController.LogicIndicesRegister.isPhysicalTable(TagAutocompleteData.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, TagAutocompleteData.INDEX_NAME)); + } final SearchBuilder search = Search.builder().query(query).size(limit); final SearchResponse response = getClient().search( diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TopologyQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TopologyQueryEsDAO.java index 7e3880151a..a2d5fd7320 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TopologyQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TopologyQueryEsDAO.java @@ -60,7 +60,7 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO { } final SearchBuilder sourceBuilder = Search.builder().size(0); - setQueryCondition(sourceBuilder, startTB, endTB, serviceIds); + setQueryCondition(sourceBuilder, startTB, endTB, serviceIds, ServiceRelationServerSideMetrics.INDEX_NAME); return buildServiceRelation( sourceBuilder, ServiceRelationServerSideMetrics.INDEX_NAME, DetectPoint.SERVER); @@ -75,7 +75,7 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO { } final SearchBuilder sourceBuilder = Search.builder().size(0); - setQueryCondition(sourceBuilder, startTB, endTB, serviceIds); + setQueryCondition(sourceBuilder, startTB, endTB, serviceIds, ServiceRelationClientSideMetrics.INDEX_NAME); return buildServiceRelation( sourceBuilder, ServiceRelationClientSideMetrics.INDEX_NAME, DetectPoint.CLIENT); @@ -85,10 +85,14 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO { public List loadServiceRelationsDetectedAtServerSide(long startTB, long endTB) { SearchBuilder sourceBuilder = Search.builder(); - sourceBuilder.query(Query.range(ServiceRelationServerSideMetrics.TIME_BUCKET) - .gte(startTB) - .lte(endTB)) - .size(0); + final BoolQueryBuilder query = Query.bool() + .must(Query.range(ServiceRelationServerSideMetrics.TIME_BUCKET) + .gte(startTB) + .lte(endTB)); + if (IndexController.LogicIndicesRegister.isPhysicalTable(ServiceRelationServerSideMetrics.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ServiceRelationServerSideMetrics.INDEX_NAME)); + } + sourceBuilder.query(query).size(0); return buildServiceRelation( sourceBuilder, ServiceRelationServerSideMetrics.INDEX_NAME, DetectPoint.SERVER); @@ -98,10 +102,14 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO { public List loadServiceRelationDetectedAtClientSide(long startTB, long endTB) { SearchBuilder sourceBuilder = Search.builder(); - sourceBuilder.query(Query.range(ServiceRelationServerSideMetrics.TIME_BUCKET) - .gte(startTB) - .lte(endTB)) - .size(0); + final BoolQueryBuilder query = Query.bool() + .must(Query.range(ServiceRelationClientSideMetrics.TIME_BUCKET) + .gte(startTB) + .lte(endTB)); + if (IndexController.LogicIndicesRegister.isPhysicalTable(ServiceRelationClientSideMetrics.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ServiceRelationClientSideMetrics.INDEX_NAME)); + } + sourceBuilder.query(query).size(0); return buildServiceRelation( sourceBuilder, ServiceRelationClientSideMetrics.INDEX_NAME, DetectPoint.CLIENT); @@ -113,7 +121,7 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO { long startTB, long endTB) { final SearchBuilder search = Search.builder().size(0); - setInstanceQueryCondition(search, startTB, endTB, clientServiceId, serverServiceId); + setInstanceQueryCondition(search, startTB, endTB, clientServiceId, serverServiceId, ServiceInstanceRelationServerSideMetrics.INDEX_NAME); return buildInstanceRelation( search, ServiceInstanceRelationServerSideMetrics.INDEX_NAME, DetectPoint.SERVER); @@ -125,14 +133,14 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO { long startTB, long endTB) { final SearchBuilder search = Search.builder().size(0); - setInstanceQueryCondition(search, startTB, endTB, clientServiceId, serverServiceId); + setInstanceQueryCondition(search, startTB, endTB, clientServiceId, serverServiceId, ServiceInstanceRelationClientSideMetrics.INDEX_NAME); return buildInstanceRelation( search, ServiceInstanceRelationClientSideMetrics.INDEX_NAME, DetectPoint.CLIENT); } private void setInstanceQueryCondition(SearchBuilder search, long startTB, long endTB, - String clientServiceId, String serverServiceId) { + String clientServiceId, String serverServiceId, String indexName) { final BoolQueryBuilder serverRelationBoolQuery = Query.bool() .must( @@ -170,7 +178,9 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO { .gte(startTB) .lte(endTB)) .must(serviceIdBoolQuery); - + if (IndexController.LogicIndicesRegister.isPhysicalTable(indexName)) { + boolQuery.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, indexName)); + } search.query(boolQuery); } @@ -195,7 +205,9 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO { Query.term( EndpointRelationServerSideMetrics.DEST_ENDPOINT, destEndpointId )); - + if (IndexController.LogicIndicesRegister.isPhysicalTable(EndpointRelationServerSideMetrics.INDEX_NAME)) { + boolQuery.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, EndpointRelationServerSideMetrics.INDEX_NAME)); + } sourceBuilder.query(boolQuery); return loadEndpoint( @@ -214,11 +226,12 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO { private List buildProcessRelation(String serviceInstanceId, long startTB, long endTB, DetectPoint detectPoint) throws IOException { final SearchBuilder sourceBuilder = Search.builder().size(0); - sourceBuilder.query(Query.bool() - .must(Query.term(ProcessRelationServerSideMetrics.SERVICE_INSTANCE_ID, serviceInstanceId)) - .must(Query.range(EndpointRelationServerSideMetrics.TIME_BUCKET) - .gte(startTB) - .lte(endTB))); + final BoolQueryBuilder query = Query.bool() + .must(Query.term(ProcessRelationServerSideMetrics.SERVICE_INSTANCE_ID, serviceInstanceId)) + .must(Query.range(EndpointRelationServerSideMetrics.TIME_BUCKET) + .gte(startTB) + .lte(endTB)); + sourceBuilder.query(query); sourceBuilder.aggregation( Aggregation .terms(Metrics.ENTITY_ID).field(Metrics.ENTITY_ID) @@ -226,9 +239,13 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO { .collectMode(TermsAggregationBuilder.CollectMode.BREADTH_FIRST) .size(1000)); - final String index = - IndexController.LogicIndicesRegister.getPhysicalTableName(detectPoint == DetectPoint.SERVER ? - ProcessRelationServerSideMetrics.INDEX_NAME : ProcessRelationClientSideMetrics.INDEX_NAME); + String indexName = detectPoint == DetectPoint.SERVER ? + ProcessRelationServerSideMetrics.INDEX_NAME : ProcessRelationClientSideMetrics.INDEX_NAME; + + final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(indexName); + if (IndexController.LogicIndicesRegister.isPhysicalTable(indexName)) { + query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, indexName)); + } final SearchResponse response = getClient().search(index, sourceBuilder.build()); final List calls = new ArrayList<>(); @@ -353,7 +370,7 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO { } private void setQueryCondition(SearchBuilder search, long startTB, long endTB, - List serviceIds) { + List serviceIds, String indexName) { final BoolQueryBuilder query = Query.bool() .must(Query.range(ServiceRelationServerSideMetrics.TIME_BUCKET) @@ -363,7 +380,9 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO { final BoolQueryBuilder serviceIdBoolQuery = Query.bool(); query.must(serviceIdBoolQuery); - + if (IndexController.LogicIndicesRegister.isPhysicalTable(indexName)) { + query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, indexName)); + } if (serviceIds.size() == 1) { serviceIdBoolQuery.should( Query.term( diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAO.java index 8c8f2f196c..ec3ac6d6a3 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAO.java @@ -66,6 +66,9 @@ public class ZipkinQueryEsDAO extends EsDAO implements IZipkinQueryDAO { final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(ZipkinServiceTraffic.INDEX_NAME); final BoolQueryBuilder query = Query.bool(); + if (IndexController.LogicIndicesRegister.isPhysicalTable(ZipkinServiceTraffic.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ZipkinServiceTraffic.INDEX_NAME)); + } final SearchBuilder search = Search.builder().query(query).size(SCROLLING_BATCH_SIZE); final SearchParams params = new SearchParams().scroll(SCROLL_CONTEXT_RETENTION); final List services = new ArrayList<>(); @@ -98,6 +101,9 @@ public class ZipkinQueryEsDAO extends EsDAO implements IZipkinQueryDAO { String index = IndexController.LogicIndicesRegister.getPhysicalTableName( ZipkinServiceRelationTraffic.INDEX_NAME); BoolQueryBuilder query = Query.bool().must(Query.term(ZipkinServiceRelationTraffic.SERVICE_NAME, serviceName)); + if (IndexController.LogicIndicesRegister.isPhysicalTable(ZipkinServiceRelationTraffic.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ZipkinServiceRelationTraffic.INDEX_NAME)); + } SearchBuilder search = Search.builder().query(query).size(NAME_QUERY_MAX_SIZE); SearchResponse response = getClient().search(index, search.build()); List remoteServices = new ArrayList<>(); @@ -114,6 +120,9 @@ public class ZipkinQueryEsDAO extends EsDAO implements IZipkinQueryDAO { public List getSpanNames(final String serviceName) { String index = IndexController.LogicIndicesRegister.getPhysicalTableName(ZipkinServiceSpanTraffic.INDEX_NAME); BoolQueryBuilder query = Query.bool().must(Query.term(ZipkinServiceSpanTraffic.SERVICE_NAME, serviceName)); + if (IndexController.LogicIndicesRegister.isPhysicalTable(ZipkinServiceSpanTraffic.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ZipkinServiceSpanTraffic.INDEX_NAME)); + } SearchBuilder search = Search.builder().query(query).size(NAME_QUERY_MAX_SIZE); SearchResponse response = getClient().search(index, search.build()); List spanNames = new ArrayList<>(); diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/TimeSeriesUtilsTest.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/TimeSeriesUtilsTest.java index 25a0d735d5..43ef3de019 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/TimeSeriesUtilsTest.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/TimeSeriesUtilsTest.java @@ -20,6 +20,8 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base; import com.google.common.collect.Lists; import org.apache.skywalking.oap.server.core.analysis.DownSampling; +import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; +import org.apache.skywalking.oap.server.core.analysis.record.Record; import org.apache.skywalking.oap.server.core.query.enumeration.Step; import org.apache.skywalking.oap.server.core.storage.model.Model; import org.apache.skywalking.oap.server.core.storage.model.SQLDatabaseModelExtension; @@ -39,13 +41,13 @@ public class TimeSeriesUtilsTest { @Before public void prepare() { superDatasetModel = new Model("superDatasetModel", Lists.newArrayList(), - 0, DownSampling.Second, true, true, "", true, new SQLDatabaseModelExtension() + 0, DownSampling.Second, true, true, Record.class, true, new SQLDatabaseModelExtension() ); normalRecordModel = new Model("normalRecordModel", Lists.newArrayList(), - 0, DownSampling.Second, true, false, "", true, new SQLDatabaseModelExtension() + 0, DownSampling.Second, true, false, Record.class, true, new SQLDatabaseModelExtension() ); normalMetricsModel = new Model("normalMetricsModel", Lists.newArrayList(), - 0, DownSampling.Minute, false, false, "", true, new SQLDatabaseModelExtension() + 0, DownSampling.Minute, false, false, Metrics.class, true, new SQLDatabaseModelExtension() ); TimeSeriesUtils.setSUPER_DATASET_DAY_STEP(1); TimeSeriesUtils.setDAY_STEP(3); @@ -75,7 +77,7 @@ public class TimeSeriesUtilsTest { writeIndexName(normalRecordModel, secondTimeBucket) ); Assert.assertEquals( - "normalMetricsModel-20200807", + "metrics-all-20200807", writeIndexName(normalMetricsModel, minuteTimeBucket) ); secondTimeBucket += 1000000; @@ -89,7 +91,7 @@ public class TimeSeriesUtilsTest { writeIndexName(normalRecordModel, secondTimeBucket) ); Assert.assertEquals( - "normalMetricsModel-20200810", + "metrics-all-20200810", writeIndexName(normalMetricsModel, minuteTimeBucket) ); } diff --git a/test/e2e-v2/cases/profiling/ebpf/oncpu/opensearch/docker-compose.yml b/test/e2e-v2/cases/profiling/ebpf/oncpu/opensearch/docker-compose.yml index 73f0017c5d..f2408e2d5a 100644 --- a/test/e2e-v2/cases/profiling/ebpf/oncpu/opensearch/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/ebpf/oncpu/opensearch/docker-compose.yml @@ -43,6 +43,7 @@ services: SW_STORAGE_ES_CLUSTER_NODES: opensearch:9200 SW_ES_USER: admin SW_ES_PASSWORD: admin + SW_STORAGE_ES_LOGIC_SHARDING: "true" depends_on: opensearch: condition: service_healthy @@ -67,4 +68,4 @@ services: condition: service_healthy networks: - e2e: \ No newline at end of file + e2e: diff --git a/test/e2e-v2/cases/storage/banyandb/e2e.yaml b/test/e2e-v2/cases/storage/banyandb/e2e.yaml index 12b86b8efc..833e1957dc 100644 --- a/test/e2e-v2/cases/storage/banyandb/e2e.yaml +++ b/test/e2e-v2/cases/storage/banyandb/e2e.yaml @@ -32,8 +32,11 @@ trigger: action: http interval: 3s times: 10 - url: http://${consumer_host}:${consumer_9092}/info + url: http://${consumer_host}:${consumer_9092}/users method: POST + body: '{"id":"123","name":"skywalking"}' + headers: + "Content-Type": "application/json" verify: # verify with retry strategy diff --git a/test/e2e-v2/cases/storage/es/e2e.yaml b/test/e2e-v2/cases/storage/es/e2e.yaml index 12b86b8efc..833e1957dc 100644 --- a/test/e2e-v2/cases/storage/es/e2e.yaml +++ b/test/e2e-v2/cases/storage/es/e2e.yaml @@ -32,8 +32,11 @@ trigger: action: http interval: 3s times: 10 - url: http://${consumer_host}:${consumer_9092}/info + url: http://${consumer_host}:${consumer_9092}/users method: POST + body: '{"id":"123","name":"skywalking"}' + headers: + "Content-Type": "application/json" verify: # verify with retry strategy diff --git a/test/e2e-v2/cases/storage/es/es-sharding/docker-compose.yml b/test/e2e-v2/cases/storage/es/es-sharding/docker-compose.yml new file mode 100644 index 0000000000..f96e6ae06b --- /dev/null +++ b/test/e2e-v2/cases/storage/es/es-sharding/docker-compose.yml @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + es: + image: elastic/elasticsearch:7.15.0 + expose: + - 9200 + networks: + - e2e + environment: + - discovery.type=single-node + - cluster.routing.allocation.disk.threshold_enabled=false + - xpack.security.enabled=false + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"] + interval: 5s + timeout: 60s + retries: 120 + + oap: + extends: + file: ../../../../script/docker-compose/base-compose.yml + service: oap + environment: + SW_STORAGE: elasticsearch + SW_STORAGE_ES_CLUSTER_NODES: es:9200 + SW_STORAGE_ES_LOGIC_SHARDING: "true" + ports: + - 12800 + depends_on: + es: + condition: service_healthy + networks: + - e2e + + provider: + extends: + file: ../../../../script/docker-compose/base-compose.yml + service: provider + ports: + - 9090 + depends_on: + oap: + condition: service_healthy + networks: + - e2e + + consumer: + extends: + file: ../../../../script/docker-compose/base-compose.yml + service: consumer + ports: + - 9092 + depends_on: + oap: + condition: service_healthy + provider: + condition: service_healthy + +networks: + e2e: diff --git a/test/e2e-v2/cases/storage/es/es-sharding/e2e.yaml b/test/e2e-v2/cases/storage/es/es-sharding/e2e.yaml new file mode 100644 index 0000000000..214270be44 --- /dev/null +++ b/test/e2e-v2/cases/storage/es/es-sharding/e2e.yaml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 20m + init-system-environment: ../../../../script/env + steps: + - name: set PATH + command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + +trigger: + action: http + interval: 3s + times: 10 + url: http://${consumer_host}:${consumer_9092}/users + method: POST + body: '{"id":"123","name":"skywalking"}' + headers: + "Content-Type": "application/json" + +verify: + # verify with retry strategy + retry: + # max retry count + count: 20 + # the interval between two retries, in millisecond. + interval: 10s + cases: + - includes: + - ../../storage-cases.yaml diff --git a/test/e2e-v2/cases/storage/expected/dependency-endpoint-consumer.yml b/test/e2e-v2/cases/storage/expected/dependency-endpoint-consumer.yml new file mode 100644 index 0000000000..8f5a8b26b4 --- /dev/null +++ b/test/e2e-v2/cases/storage/expected/dependency-endpoint-consumer.yml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nodes: +{{- contains .nodes }} +- id: {{ b64enc "User" }}.0_{{ b64enc "User" }} + name: User + serviceid: {{ b64enc "User" }}.0 + servicename: User + type: "" + isreal: false +- id: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "POST:/users" }} + name: POST:/users + serviceid: {{ b64enc "e2e-service-consumer" }}.1 + servicename: e2e-service-consumer + type: "" + isreal: true +- id: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "POST:/users" }} + name: POST:/users + serviceid: {{ b64enc "e2e-service-provider" }}.1 + servicename: e2e-service-provider + type: "" + isreal: true +{{- end }} +calls: +{{- contains .calls }} +- source: {{ b64enc "User" }}.0_{{ b64enc "User" }} + sourcecomponents: [] + target: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "POST:/users" }} + targetcomponents: [] + id: {{ b64enc "User" }}.0-{{ b64enc "User" }}-{{ b64enc "e2e-service-consumer" }}.1-{{ b64enc "POST:/users" }} + detectpoints: + - SERVER +- source: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "POST:/users" }} + sourcecomponents: [] + target: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "POST:/users" }} + targetcomponents: [] + id: {{ b64enc "e2e-service-consumer" }}.1-{{ b64enc "POST:/users" }}-{{ b64enc "e2e-service-provider" }}.1-{{ b64enc "POST:/users" }} + detectpoints: + - SERVER +{{- end }} diff --git a/test/e2e-v2/cases/storage/expected/dependency-endpoint-provider.yml b/test/e2e-v2/cases/storage/expected/dependency-endpoint-provider.yml new file mode 100644 index 0000000000..3835336b8b --- /dev/null +++ b/test/e2e-v2/cases/storage/expected/dependency-endpoint-provider.yml @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nodes: +{{- contains .nodes }} +- id: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "POST:/users" }} + name: POST:/users + serviceid: {{ b64enc "e2e-service-consumer" }}.1 + servicename: e2e-service-consumer + type: "" + isreal: true +- id: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "POST:/users" }} + name: POST:/users + serviceid: {{ b64enc "e2e-service-provider" }}.1 + servicename: e2e-service-provider + type: "" + isreal: true +{{- end }} +calls: +{{- contains .calls }} +- source: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "POST:/users" }} + sourcecomponents: [] + target: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "POST:/users" }} + targetcomponents: [] + id: {{ b64enc "e2e-service-consumer" }}.1-{{ b64enc "POST:/users" }}-{{ b64enc "e2e-service-provider" }}.1-{{ b64enc "POST:/users" }} + detectpoints: + - SERVER +{{- end }} diff --git a/test/e2e-v2/cases/storage/expected/dependency-instance.yml b/test/e2e-v2/cases/storage/expected/dependency-instance.yml new file mode 100644 index 0000000000..d3b9547c60 --- /dev/null +++ b/test/e2e-v2/cases/storage/expected/dependency-instance.yml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nodes: +{{- contains .nodes }} +- id: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "consumer1" }} + name: consumer1 + serviceid: {{ b64enc "e2e-service-consumer" }}.1 + servicename: e2e-service-consumer + type: "" + isreal: true +- id: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} + name: provider1 + serviceid: {{ b64enc "e2e-service-provider" }}.1 + servicename: e2e-service-provider + type: "Tomcat" + isreal: true +{{- end }} +calls: +{{- contains .calls }} +- source: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "consumer1" }} + sourcecomponents: [] + target: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} + targetcomponents: [] + id: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "consumer1" }}-{{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} + detectpoints: + - CLIENT + - SERVER +{{- end }} diff --git a/test/e2e-v2/cases/storage/expected/dependency-services.yml b/test/e2e-v2/cases/storage/expected/dependency-services-consumer.yml similarity index 82% rename from test/e2e-v2/cases/storage/expected/dependency-services.yml rename to test/e2e-v2/cases/storage/expected/dependency-services-consumer.yml index aa6e2fc5fd..11ba084ec2 100644 --- a/test/e2e-v2/cases/storage/expected/dependency-services.yml +++ b/test/e2e-v2/cases/storage/expected/dependency-services-consumer.yml @@ -15,14 +15,18 @@ nodes: {{- contains .nodes }} -- id: {{ b64enc "e2e-service-provider"}}.1 - name: e2e-service-provider - type: Tomcat - isreal: true +- id: {{ b64enc "User"}}.0 + name: User + type: USER + isreal: false - id: {{ b64enc "e2e-service-consumer"}}.1 name: e2e-service-consumer type: Tomcat isreal: true +- id: {{ b64enc "e2e-service-provider"}}.1 + name: e2e-service-provider + type: Tomcat + isreal: true {{- end }} calls: {{- contains .calls }} @@ -34,4 +38,11 @@ calls: detectpoints: - CLIENT - SERVER +- source: {{ b64enc "User" }}.0 + sourcecomponents: [] + target: {{ b64enc "e2e-service-consumer"}}.1 + targetcomponents: [] + id: {{ b64enc "User" }}.0-{{ b64enc "e2e-service-consumer"}}.1 + detectpoints: + - SERVER {{- end }} diff --git a/test/e2e-v2/cases/storage/expected/dependency-services-provider.yml b/test/e2e-v2/cases/storage/expected/dependency-services-provider.yml new file mode 100644 index 0000000000..600dd6dd99 --- /dev/null +++ b/test/e2e-v2/cases/storage/expected/dependency-services-provider.yml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nodes: +{{- contains .nodes }} +- id: {{ b64enc "e2e-service-provider"}}.1 + name: e2e-service-provider + type: Tomcat + isreal: true +- id: {{ b64enc "e2e-service-consumer"}}.1 + name: e2e-service-consumer + type: Tomcat + isreal: true +- id: {{ b64enc "localhost:-1" }}.0 + name: localhost:-1 + type: H2 + isreal: false +{{- end }} +calls: +{{- contains .calls }} +- source: {{ b64enc "e2e-service-consumer"}}.1 + sourcecomponents: [] + target: {{ b64enc "e2e-service-provider"}}.1 + targetcomponents: [] + id: {{ b64enc "e2e-service-consumer"}}.1-{{ b64enc "e2e-service-provider"}}.1 + detectpoints: + - CLIENT + - SERVER +- source: {{ b64enc "e2e-service-provider" }}.1 + sourcecomponents: [] + target: {{ b64enc "localhost:-1"}}.0 + targetcomponents: [] + id: {{ b64enc "e2e-service-provider" }}.1-{{ b64enc "localhost:-1"}}.0 + detectpoints: + - CLIENT +{{- end }} diff --git a/test/e2e-v2/cases/storage/expected/metrics-has-value-percentile.yml b/test/e2e-v2/cases/storage/expected/metrics-has-value-percentile.yml new file mode 100644 index 0000000000..ef17490250 --- /dev/null +++ b/test/e2e-v2/cases/storage/expected/metrics-has-value-percentile.yml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- contains . }} +- key: 0 + value: + {{- contains .value }} + - key: {{ notEmpty .key }} + value: {{ ge .value 1 }} + {{- end }} +- key: 1 + value: + {{- contains .value }} + - key: {{ notEmpty .key }} + value: {{ ge .value 1 }} + {{- end }} +- key: 2 + value: + {{- contains .value }} + - key: {{ notEmpty .key }} + value: {{ ge .value 1 }} + {{- end }} +- key: 3 + value: + {{- contains .value }} + - key: {{ notEmpty .key }} + value: {{ ge .value 1 }} + {{- end }} +- key: 4 + value: + {{- contains .value }} + - key: {{ notEmpty .key }} + value: {{ ge .value 1 }} + {{- end }} +{{- end }} diff --git a/test/e2e-v2/cases/storage/expected/metrics-has-value.yml b/test/e2e-v2/cases/storage/expected/metrics-has-value.yml index 5359e6d315..d9c49854c9 100644 --- a/test/e2e-v2/cases/storage/expected/metrics-has-value.yml +++ b/test/e2e-v2/cases/storage/expected/metrics-has-value.yml @@ -16,4 +16,4 @@ {{- contains . }} - key: {{ notEmpty .key }} value: {{ ge .value 1 }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/test/e2e-v2/cases/storage/expected/service-endpoint.yml b/test/e2e-v2/cases/storage/expected/service-endpoint-consumer.yml similarity index 87% rename from test/e2e-v2/cases/storage/expected/service-endpoint.yml rename to test/e2e-v2/cases/storage/expected/service-endpoint-consumer.yml index d2afb4575b..cedf79a14e 100644 --- a/test/e2e-v2/cases/storage/expected/service-endpoint.yml +++ b/test/e2e-v2/cases/storage/expected/service-endpoint-consumer.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -{{- contains .}} -- id: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "POST:/info" }} - name: POST:/info +{{- contains . }} +- id: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "POST:/users" }} + name: POST:/users {{- end}} diff --git a/test/e2e-v2/cases/storage/expected/service-endpoint-provider.yml b/test/e2e-v2/cases/storage/expected/service-endpoint-provider.yml new file mode 100644 index 0000000000..0303f51790 --- /dev/null +++ b/test/e2e-v2/cases/storage/expected/service-endpoint-provider.yml @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- contains . }} +- id: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "POST:/users" }} + name: POST:/users +{{- end}} \ No newline at end of file diff --git a/test/e2e-v2/cases/storage/expected/service-instance-consumer.yml b/test/e2e-v2/cases/storage/expected/service-instance-consumer.yml new file mode 100644 index 0000000000..dc2ac11dbd --- /dev/null +++ b/test/e2e-v2/cases/storage/expected/service-instance-consumer.yml @@ -0,0 +1,40 @@ +# Licensed to Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Apache Software Foundation (ASF) licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{{- contains . }} +- id: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "consumer1" }} + name: consumer1 + attributes: + {{- contains .attributes }} + - name: OS Name + value: Linux + - name: hostname + value: {{ notEmpty .value }} + - name: Process No. + value: {{ notEmpty .value }} + - name: Start Time + value: {{ notEmpty .value }} + - name: JVM Arguments + value: '{{ notEmpty .value }}' + - name: Jar Dependencies + value: '{{ notEmpty .value }}' + - name: ipv4s + value: {{ notEmpty .value }} + {{- end}} + language: JAVA + instanceuuid: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "consumer1" }} +{{- end}} diff --git a/test/e2e-v2/cases/storage/expected/service-instance.yml b/test/e2e-v2/cases/storage/expected/service-instance-provider.yml similarity index 95% rename from test/e2e-v2/cases/storage/expected/service-instance.yml rename to test/e2e-v2/cases/storage/expected/service-instance-provider.yml index 5f03bc18e7..f32cc2b0f1 100644 --- a/test/e2e-v2/cases/storage/expected/service-instance.yml +++ b/test/e2e-v2/cases/storage/expected/service-instance-provider.yml @@ -15,9 +15,9 @@ # specific language governing permissions and limitations # under the License. -{{- contains .}} +{{- contains . }} - id: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} - name: {{ notEmpty .name }} + name: provider1 attributes: {{- contains .attributes }} - name: OS Name @@ -25,7 +25,7 @@ - name: hostname value: {{ notEmpty .value }} - name: Process No. - value: "1" + value: {{ notEmpty .value }} - name: Start Time value: {{ notEmpty .value }} - name: JVM Arguments diff --git a/test/e2e-v2/cases/storage/expected/trace-info-detail.yml b/test/e2e-v2/cases/storage/expected/trace-info-detail.yml deleted file mode 100644 index 95b9bd98a5..0000000000 --- a/test/e2e-v2/cases/storage/expected/trace-info-detail.yml +++ /dev/null @@ -1,68 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -spans: - {{- contains .spans }} - - traceid: {{ notEmpty .traceid }} - segmentid: {{ notEmpty .segmentid }} - spanid: {{ .spanid }} - parentspanid: {{ .parentspanid }} - refs: - {{- if eq .servicecode "e2e-service-provider" }} - {{- contains .refs }} - - traceid: {{ notEmpty .traceid }} - parentsegmentid: {{ notEmpty .parentsegmentid }} - parentspanid: 1 - type: CROSS_PROCESS - {{- end }} - {{- end }} - {{- if eq .servicecode "e2e-service-consumer" }} - [] - {{- end }} - servicecode: {{ notEmpty .servicecode }} - serviceinstancename: {{ notEmpty .serviceinstancename }} - starttime: {{ gt .starttime 0 }} - endtime: {{ gt .endtime 0 }} - endpointname: - {{- if eq .type "Exit" }} - /info - {{ else }} - POST:/info - {{- end }} - type: {{ notEmpty .type }} - peer: - {{- if eq .type "Exit" }} - provider:9090 - {{ else }} - "" - {{- end }} - component: - {{- if eq .type "Exit" }} - SpringRestTemplate - {{- end }} - {{- if eq .type "Entry" }} - Tomcat - {{- end }} - iserror: false - layer: Http - tags: - {{- contains .tags }} - - key: http.method - value: POST - - key: url - value: {{ notEmpty .value }} - {{- end }} - logs: [] - {{- end }} diff --git a/test/e2e-v2/cases/storage/expected/trace-users-detail.yml b/test/e2e-v2/cases/storage/expected/trace-users-detail.yml index 4dfdd6ec41..0905893c03 100644 --- a/test/e2e-v2/cases/storage/expected/trace-users-detail.yml +++ b/test/e2e-v2/cases/storage/expected/trace-users-detail.yml @@ -14,12 +14,64 @@ # limitations under the License. spans: -{{- contains .spans }} + {{- contains .spans }} - traceid: {{ .traceid }} segmentid: {{ .segmentid }} spanid: {{ .spanid }} parentspanid: {{ .parentspanid }} refs: [] + servicecode: e2e-service-consumer + serviceinstancename: consumer1 + starttime: {{ gt .starttime 0 }} + endtime: {{ gt .endtime 0 }} + endpointname: POST:/users + type: Entry + peer: "" + component: Tomcat + iserror: false + layer: Http + tags: + {{- contains .tags }} + - key: http.method + value: POST + - key: url + value: {{ notEmpty .value }} + {{- end }} + logs: [] + - traceid: {{ notEmpty .traceid }} + segmentid: {{ .segmentid }} + spanid: {{ .spanid }} + parentspanid: {{ .parentspanid }} + refs: [] + servicecode: e2e-service-consumer + serviceinstancename: consumer1 + starttime: {{ gt .starttime 0 }} + endtime: {{ gt .endtime 0 }} + endpointname: /users + type: Exit + peer: provider:9090 + component: SpringRestTemplate + iserror: false + layer: Http + tags: + {{- contains .tags }} + - key: http.method + value: POST + - key: url + value: {{ notEmpty .value }} + {{- end }} + logs: [] + - traceid: {{ notEmpty .traceid }} + segmentid: {{ .segmentid }} + spanid: {{ .spanid }} + parentspanid: {{ .parentspanid }} + refs: + {{- contains .refs }} + - traceid: {{ notEmpty .traceid }} + parentsegmentid: {{ .parentsegmentid }} + parentspanid: 1 + type: CROSS_PROCESS + {{- end }} servicecode: e2e-service-provider serviceinstancename: provider1 starttime: {{ gt .starttime 0 }} diff --git a/test/e2e-v2/cases/storage/expected/traces-list.yml b/test/e2e-v2/cases/storage/expected/traces-list.yml index 9ff69011e8..4d5a1e5fec 100644 --- a/test/e2e-v2/cases/storage/expected/traces-list.yml +++ b/test/e2e-v2/cases/storage/expected/traces-list.yml @@ -17,12 +17,10 @@ traces: {{- contains .traces }} - segmentid: {{ notEmpty .segmentid }} endpointnames: - {{- contains .endpointnames }} - - POST:/info - {{- end }} + - POST:/users duration: {{ ge .duration 0 }} start: "{{ notEmpty .start}}" iserror: false traceids: - - {{ (index .traceids 0) }} + - {{ index .traceids 0 }} {{- end }} diff --git a/test/e2e-v2/cases/storage/h2/e2e.yaml b/test/e2e-v2/cases/storage/h2/e2e.yaml index 12b86b8efc..833e1957dc 100644 --- a/test/e2e-v2/cases/storage/h2/e2e.yaml +++ b/test/e2e-v2/cases/storage/h2/e2e.yaml @@ -32,8 +32,11 @@ trigger: action: http interval: 3s times: 10 - url: http://${consumer_host}:${consumer_9092}/info + url: http://${consumer_host}:${consumer_9092}/users method: POST + body: '{"id":"123","name":"skywalking"}' + headers: + "Content-Type": "application/json" verify: # verify with retry strategy diff --git a/test/e2e-v2/cases/storage/mysql/e2e.yaml b/test/e2e-v2/cases/storage/mysql/e2e.yaml index 12b86b8efc..833e1957dc 100644 --- a/test/e2e-v2/cases/storage/mysql/e2e.yaml +++ b/test/e2e-v2/cases/storage/mysql/e2e.yaml @@ -32,8 +32,11 @@ trigger: action: http interval: 3s times: 10 - url: http://${consumer_host}:${consumer_9092}/info + url: http://${consumer_host}:${consumer_9092}/users method: POST + body: '{"id":"123","name":"skywalking"}' + headers: + "Content-Type": "application/json" verify: # verify with retry strategy diff --git a/test/e2e-v2/cases/storage/opensearch/e2e.yaml b/test/e2e-v2/cases/storage/opensearch/e2e.yaml index 12b86b8efc..833e1957dc 100644 --- a/test/e2e-v2/cases/storage/opensearch/e2e.yaml +++ b/test/e2e-v2/cases/storage/opensearch/e2e.yaml @@ -32,8 +32,11 @@ trigger: action: http interval: 3s times: 10 - url: http://${consumer_host}:${consumer_9092}/info + url: http://${consumer_host}:${consumer_9092}/users method: POST + body: '{"id":"123","name":"skywalking"}' + headers: + "Content-Type": "application/json" verify: # verify with retry strategy diff --git a/test/e2e-v2/cases/storage/postgres/e2e.yaml b/test/e2e-v2/cases/storage/postgres/e2e.yaml index 12b86b8efc..833e1957dc 100644 --- a/test/e2e-v2/cases/storage/postgres/e2e.yaml +++ b/test/e2e-v2/cases/storage/postgres/e2e.yaml @@ -32,8 +32,11 @@ trigger: action: http interval: 3s times: 10 - url: http://${consumer_host}:${consumer_9092}/info + url: http://${consumer_host}:${consumer_9092}/users method: POST + body: '{"id":"123","name":"skywalking"}' + headers: + "Content-Type": "application/json" verify: # verify with retry strategy diff --git a/test/e2e-v2/cases/storage/storage-cases.yaml b/test/e2e-v2/cases/storage/storage-cases.yaml index ec30bf5573..b1ac0b299f 100644 --- a/test/e2e-v2/cases/storage/storage-cases.yaml +++ b/test/e2e-v2/cases/storage/storage-cases.yaml @@ -20,40 +20,123 @@ cases: # service list - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service layer GENERAL expected: expected/service.yml - # service metrics - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name service_sla --service-name=e2e-service-provider | yq e 'to_entries' - - expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics top --name service_sla 5 - expected: expected/metrics-top-service-sla.yml - # service endpoint - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=info --service-name=e2e-service-provider - expected: expected/service-endpoint.yml - # service endpoint metrics - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name endpoint_cpm --endpoint-name=POST:/info --service-name=e2e-service-provider |yq e 'to_entries' - - expected: expected/metrics-has-value.yml - # dependency service - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency service --service-name="e2e-service-provider" - expected: expected/dependency-services.yml # service instance list - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider - expected: expected/service-instance.yml - # service instance jvm metrics - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name instance_jvm_thread_live_count --instance-name=provider1 --service-name=e2e-service-provider | yq e 'to_entries' - - expected: expected/metrics-has-value.yml - + expected: expected/service-instance-provider.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-consumer + expected: expected/service-instance-consumer.yml + # service endpoint + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-provider + expected: expected/service-endpoint-provider.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-consumer + expected: expected/service-endpoint-consumer.yml + # dependency service + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency service --service-name=e2e-service-provider + expected: expected/dependency-services-provider.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency service --service-name=e2e-service-consumer + expected: expected/dependency-services-consumer.yml + # dependency instance + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency instance --service-name=e2e-service-consumer --dest-service-name=e2e-service-provider + expected: expected/dependency-instance.yml + # dependency endpoint + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency endpoint --service-name=e2e-service-provider --endpoint-name=POST:/users + expected: expected/dependency-endpoint-provider.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency endpoint --service-name=e2e-service-consumer --endpoint-name=POST:/users + expected: expected/dependency-endpoint-consumer.yml # trace segment list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls --tags http.method=POST + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls expected: expected/traces-list.yml # negative tags search: relationship should be logical AND instead of logical OR - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls --tags http.method=POST,http.status_code=200 expected: expected/empty-traces-list.yml - # native tracing: trace detail + # trace detail - query: | swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace $( \ swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls \ - | yq e '.traces | select(.[].endpointnames[0]=="POST:/info") | .[0].traceids[0]' - \ + | yq e '.traces | select(.[].endpointnames[0]=="POST:/users") | .[0].traceids[0]' - ) - expected: expected/trace-info-detail.yml + expected: expected/trace-users-detail.yml + # service metrics + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics top --name service_sla 5 + expected: expected/metrics-top-service-sla.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_sla --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_cpm --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_resp_time --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_apdex --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_sla --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_cpm --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_resp_time --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_apdex --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + # service instance metrics + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_instance_resp_time --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_instance_cpm --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_instance_sla --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + # service instance JVM metrics + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_memory_heap --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_memory_heap_max --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_memory_noheap --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_live_count --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_daemon_count --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_peak_count --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_runnable_state_thread_count --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_class_loaded_class_count --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_class_total_loaded_class_count --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_memory_heap --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_memory_heap_max --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_memory_noheap --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_live_count --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_daemon_count --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_peak_count --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_runnable_state_thread_count --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_class_loaded_class_count --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_class_total_loaded_class_count --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + # service endpoint metrics + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=endpoint_cpm --endpoint-name=POST:/users --service-name=e2e-service-provider | yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=endpoint_resp_time --endpoint-name=POST:/users --service-name=e2e-service-provider | yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=endpoint_sla --endpoint-name=POST:/users --service-name=e2e-service-provider | yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=endpoint_cpm --endpoint-name=POST:/users --service-name=e2e-service-consumer | yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=endpoint_resp_time --endpoint-name=POST:/users --service-name=e2e-service-consumer | yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=endpoint_sla --endpoint-name=POST:/users --service-name=e2e-service-consumer | yq e 'to_entries' - + expected: expected/metrics-has-value.yml + # service endpoint metrics percentile + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics multiple-linear --name=endpoint_percentile --endpoint-name=POST:/users --service-name=e2e-service-consumer |yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' - + expected: expected/metrics-has-value-percentile.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics multiple-linear --name=endpoint_percentile --endpoint-name=POST:/users --service-name=e2e-service-consumer |yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' - + expected: expected/metrics-has-value-percentile.yml # native event: event list - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql event list diff --git a/test/e2e-v2/cases/storage/tidb/e2e.yaml b/test/e2e-v2/cases/storage/tidb/e2e.yaml index 12b86b8efc..833e1957dc 100644 --- a/test/e2e-v2/cases/storage/tidb/e2e.yaml +++ b/test/e2e-v2/cases/storage/tidb/e2e.yaml @@ -32,8 +32,11 @@ trigger: action: http interval: 3s times: 10 - url: http://${consumer_host}:${consumer_9092}/info + url: http://${consumer_host}:${consumer_9092}/users method: POST + body: '{"id":"123","name":"skywalking"}' + headers: + "Content-Type": "application/json" verify: # verify with retry strategy