diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index 0381110a05..989404e372 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -459,6 +459,9 @@ jobs: - name: eBPF Profiling Network config: test/e2e-v2/cases/profiling/ebpf/network/e2e.yaml env: ISTIO_VERSION=1.13.1 + - name: eBPF Profiling Network ES Sharding + config: test/e2e-v2/cases/profiling/ebpf/network/es-sharding/e2e.yaml + env: ISTIO_VERSION=1.13.1 - name: Kafka Basic config: test/e2e-v2/cases/kafka/simple-so11y/e2e.yaml diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index be0142aac1..3184bffdc6 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -34,6 +34,7 @@ In the current one index mode, users still could choose to adjust ElasticSearch's shard number(`SW_STORAGE_ES_INDEX_SHARDS_NUMBER`) to scale out. * [Breaking Change] Many columns of metrics and records model names are changed, The H2/Mysql/Tidb/Postgres storage users are required to remove all metrics-related and records-related tables for OAP to re-create or use a new database instance. * Fix Zipkin trace query the max size of spans. +* Add `tls` and `https` component IDs for Network Profiling. #### UI diff --git a/docs/en/concepts-and-designs/mal.md b/docs/en/concepts-and-designs/mal.md index e92ceef7cb..2cced6af58 100644 --- a/docs/en/concepts-and-designs/mal.md +++ b/docs/en/concepts-and-designs/mal.md @@ -249,7 +249,7 @@ They extract level relevant labels from metric labels, then informs the meter-sy extracts endpoint level labels from the second array argument, extracts layer from `Layer` argument. - `serviceRelation(DetectPoint, [source_svc_label1...], [dest_svc_label1...], Layer)` DetectPoint including `DetectPoint.CLIENT` and `DetectPoint.SERVER`, extracts `sourceService` labels from the first array argument, extracts `destService` labels from the second array argument, extracts layer from `Layer` argument. - - `processRelation(detect_point_label, [service_label1...], [instance_label1...], source_process_id_label, dest_process_id_label)` extracts `DetectPoint` labels from first argument, the label value should be `client` or `server`. + - `processRelation(detect_point_label, [service_label1...], [instance_label1...], source_process_id_label, dest_process_id_label, component_label)` extracts `DetectPoint` labels from first argument, the label value should be `client` or `server`. extracts `Service` labels from the first array argument, extracts `Instance` labels from the second array argument, extracts `ProcessID` labels from the fourth and fifth arguments of the source and destination. ## More Examples diff --git a/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/Analyzer.java b/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/Analyzer.java index 28c2c487f4..cf406a375e 100644 --- a/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/Analyzer.java +++ b/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/Analyzer.java @@ -357,6 +357,7 @@ public class Analyzer { metrics.setSourceProcessId(entity.getSourceProcessId()); metrics.setDestProcessId(entity.getDestProcessId()); metrics.setEntityId(entity.id()); + metrics.setComponentId(entity.getComponentId()); MetricsStreamProcessor.getInstance().in(metrics); } @@ -367,6 +368,7 @@ public class Analyzer { metrics.setSourceProcessId(entity.getSourceProcessId()); metrics.setDestProcessId(entity.getDestProcessId()); metrics.setEntityId(entity.id()); + metrics.setComponentId(entity.getComponentId()); MetricsStreamProcessor.getInstance().in(metrics); } diff --git a/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/dsl/EntityDescription/ProcessRelationEntityDescription.java b/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/dsl/EntityDescription/ProcessRelationEntityDescription.java index 8851bbf9b5..1e3533e306 100644 --- a/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/dsl/EntityDescription/ProcessRelationEntityDescription.java +++ b/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/dsl/EntityDescription/ProcessRelationEntityDescription.java @@ -36,6 +36,7 @@ public class ProcessRelationEntityDescription implements EntityDescription { private final String sourceProcessIdKey; private final String destProcessIdKey; private final String detectPointKey; + private final String componentKey; private final String delimiter; @Override @@ -43,6 +44,6 @@ public class ProcessRelationEntityDescription implements EntityDescription { return ImmutableList.builder() .addAll(serviceKeys) .addAll(instanceKeys) - .add(detectPointKey, sourceProcessIdKey, destProcessIdKey).build(); + .add(detectPointKey, sourceProcessIdKey, destProcessIdKey, componentKey).build(); } } diff --git a/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/dsl/SampleFamily.java b/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/dsl/SampleFamily.java index acc7e9c489..383653f73f 100644 --- a/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/dsl/SampleFamily.java +++ b/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/dsl/SampleFamily.java @@ -513,7 +513,7 @@ public class SampleFamily { }).toArray(Sample[]::new)); } - public SampleFamily processRelation(String detectPointKey, List serviceKeys, List instanceKeys, String sourceProcessIdKey, String destProcessIdKey) { + public SampleFamily processRelation(String detectPointKey, List serviceKeys, List instanceKeys, String sourceProcessIdKey, String destProcessIdKey, String componentKey) { Preconditions.checkArgument(serviceKeys.size() > 0); Preconditions.checkArgument(instanceKeys.size() > 0); Preconditions.checkArgument(StringUtil.isNotEmpty(sourceProcessIdKey)); @@ -525,11 +525,12 @@ public class SampleFamily { ctx.scopeLabels.add(detectPointKey); ctx.scopeLabels.add(sourceProcessIdKey); ctx.scopeLabels.add(destProcessIdKey); + ctx.scopeLabels.add(componentKey); }); if (this == EMPTY) { return EMPTY; } - return createMeterSamples(new ProcessRelationEntityDescription(serviceKeys, instanceKeys, sourceProcessIdKey, destProcessIdKey, detectPointKey, Const.POINT)); + return createMeterSamples(new ProcessRelationEntityDescription(serviceKeys, instanceKeys, sourceProcessIdKey, destProcessIdKey, detectPointKey, componentKey, Const.POINT)); } private SampleFamily createMeterSamples(EntityDescription entityDescription) { @@ -690,11 +691,14 @@ public class SampleFamily { final ProcessRelationEntityDescription processRelationEntityDescription = (ProcessRelationEntityDescription) entityDescription; final String detectPointValue = InternalOps.dim(samples, Collections.singletonList(processRelationEntityDescription.getDetectPointKey()), processRelationEntityDescription.getDelimiter()); DetectPoint point = StringUtils.equalsAnyIgnoreCase(detectPointValue, "server") ? DetectPoint.SERVER : DetectPoint.CLIENT; + final String componentValue = InternalOps.dim(samples, Collections.singletonList(processRelationEntityDescription.getComponentKey()), processRelationEntityDescription.getDelimiter()); + final int componentId = StringUtil.isNotEmpty(componentValue) ? Integer.parseInt(componentValue) : 0; return MeterEntity.newProcessRelation( InternalOps.dim(samples, processRelationEntityDescription.getServiceKeys(), processRelationEntityDescription.getDelimiter()), InternalOps.dim(samples, processRelationEntityDescription.getInstanceKeys(), processRelationEntityDescription.getDelimiter()), InternalOps.dim(samples, Collections.singletonList(processRelationEntityDescription.getSourceProcessIdKey()), processRelationEntityDescription.getDelimiter()), InternalOps.dim(samples, Collections.singletonList(processRelationEntityDescription.getDestProcessIdKey()), processRelationEntityDescription.getDelimiter()), + componentId, point ); default: diff --git a/oap-server/analyzer/meter-analyzer/src/test/java/org/apache/skywalking/oap/meter/analyzer/dsl/ScopeTest.java b/oap-server/analyzer/meter-analyzer/src/test/java/org/apache/skywalking/oap/meter/analyzer/dsl/ScopeTest.java index 24882ecaa8..3796ee6ec2 100644 --- a/oap-server/analyzer/meter-analyzer/src/test/java/org/apache/skywalking/oap/meter/analyzer/dsl/ScopeTest.java +++ b/oap-server/analyzer/meter-analyzer/src/test/java/org/apache/skywalking/oap/meter/analyzer/dsl/ScopeTest.java @@ -537,23 +537,23 @@ public class ScopeTest { "sum_process_relation", of("rover_network_profiling_process_write_bytes", SampleFamilyBuilder.newBuilder( Sample.builder() - .labels(of("service", "test", "instance", "test-instance", "side", "server", "client_process_id", "abc", "server_process_id", "def")) + .labels(of("service", "test", "instance", "test-instance", "side", "server", "client_process_id", "abc", "server_process_id", "def", "component", "1")) .value(11) .name("rover_network_profiling_process_write_bytes") .build(), Sample.builder() - .labels(of("service", "test", "instance", "test-instance", "side", "client", "client_process_id", "abc", "server_process_id", "def")) + .labels(of("service", "test", "instance", "test-instance", "side", "client", "client_process_id", "abc", "server_process_id", "def", "component", "2")) .value(12) .name("rover_network_profiling_process_write_bytes") .build() ).build()), - "rover_network_profiling_process_write_bytes.sum(['service' ,'instance', 'side', 'client_process_id', 'server_process_id'])" + - ".processRelation('side', ['service'], ['instance'], 'client_process_id', 'server_process_id')", + "rover_network_profiling_process_write_bytes.sum(['service' ,'instance', 'side', 'client_process_id', 'server_process_id', 'component'])" + + ".processRelation('side', ['service'], ['instance'], 'client_process_id', 'server_process_id', 'component')", false, new HashMap() { { put( - MeterEntity.newProcessRelation("test", "test-instance", "abc", "def", DetectPoint.SERVER), + MeterEntity.newProcessRelation("test", "test-instance", "abc", "def", 1, DetectPoint.SERVER), new Sample[] { Sample.builder() .labels(of()) @@ -562,7 +562,7 @@ public class ScopeTest { } ); put( - MeterEntity.newProcessRelation("test", "test-instance", "abc", "def", DetectPoint.CLIENT), + MeterEntity.newProcessRelation("test", "test-instance", "abc", "def", 2, DetectPoint.CLIENT), new Sample[] { Sample.builder() .labels(of()) diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/process/ProcessRelationClientSideMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/process/ProcessRelationClientSideMetrics.java index 49eebfe6ba..7d9e47ac1c 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/process/ProcessRelationClientSideMetrics.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/process/ProcessRelationClientSideMetrics.java @@ -45,6 +45,7 @@ public class ProcessRelationClientSideMetrics extends Metrics { public static final String SERVICE_INSTANCE_ID = "service_instance_id"; public static final String SOURCE_PROCESS_ID = "source_process_id"; public static final String DEST_PROCESS_ID = "dest_process_id"; + public static final String COMPONENT_ID = "component_id"; @Setter @Getter @@ -62,6 +63,10 @@ public class ProcessRelationClientSideMetrics extends Metrics { @Getter @Column(columnName = ENTITY_ID, length = 512) private String entityId; + @Setter + @Getter + @Column(columnName = COMPONENT_ID, storageOnly = true) + private int componentId; @Override protected String id0() { @@ -97,6 +102,7 @@ public class ProcessRelationClientSideMetrics extends Metrics { setDestProcessId(remoteData.getDataStrings(2)); setEntityId(remoteData.getDataStrings(3)); setTimeBucket(remoteData.getDataLongs(0)); + setComponentId(remoteData.getDataIntegers(0)); } @Override @@ -107,6 +113,7 @@ public class ProcessRelationClientSideMetrics extends Metrics { builder.addDataStrings(getDestProcessId()); builder.addDataStrings(getEntityId()); builder.addDataLongs(getTimeBucket()); + builder.addDataIntegers(getComponentId()); return builder; } @@ -124,6 +131,7 @@ public class ProcessRelationClientSideMetrics extends Metrics { metrics.setDestProcessId((String) converter.get(DEST_PROCESS_ID)); metrics.setTimeBucket(((Number) converter.get(TIME_BUCKET)).longValue()); metrics.setEntityId((String) converter.get(ENTITY_ID)); + metrics.setComponentId(((Number) converter.get(COMPONENT_ID)).intValue()); return metrics; } @@ -135,6 +143,7 @@ public class ProcessRelationClientSideMetrics extends Metrics { converter.accept(SOURCE_PROCESS_ID, storageData.getSourceProcessId()); converter.accept(DEST_PROCESS_ID, storageData.getDestProcessId()); converter.accept(ENTITY_ID, storageData.getEntityId()); + converter.accept(COMPONENT_ID, storageData.getComponentId()); } } } \ No newline at end of file diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/process/ProcessRelationDispatcher.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/process/ProcessRelationDispatcher.java index ccf5ff34f6..34cb161e8a 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/process/ProcessRelationDispatcher.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/process/ProcessRelationDispatcher.java @@ -43,6 +43,7 @@ public class ProcessRelationDispatcher implements SourceDispatcher clientCalls, @@ -84,7 +90,6 @@ public class ProcessTopologyBuilder { .map(t -> (ProcessTraffic) t) .collect(Collectors.toMap(Metrics::id, this::buildNode)); - // adding client side call for (Call.CallDetail clientCall : clientCalls) { if (!callMap.containsKey(clientCall.getId())) { Call call = new Call(); @@ -94,25 +99,25 @@ public class ProcessTopologyBuilder { call.setTarget(clientCall.getTarget()); call.setId(clientCall.getId()); call.addDetectPoint(DetectPoint.CLIENT); + call.addSourceComponent(componentLibraryCatalogService.getComponentName(clientCall.getComponentId())); calls.add(call); } } // adding server side call for (Call.CallDetail serverCall : serverCalls) { - if (!callMap.containsKey(serverCall.getId())) { - Call call = new Call(); + Call call = callMap.get(serverCall.getId()); + if (call == null) { + call = new Call(); callMap.put(serverCall.getId(), call); call.setSource(serverCall.getSource()); call.setTarget(serverCall.getTarget()); call.setId(serverCall.getId()); - call.addDetectPoint(DetectPoint.SERVER); calls.add(call); - } else { - Call call = callMap.get(serverCall.getId()); - call.addDetectPoint(DetectPoint.SERVER); } + call.addDetectPoint(DetectPoint.SERVER); + call.addTargetComponent(componentLibraryCatalogService.getComponentName(serverCall.getComponentId())); } ProcessTopology topology = new ProcessTopology(); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Call.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Call.java index f8cd7ca37f..49caff4857 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Call.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Call.java @@ -122,14 +122,14 @@ public class Call { this.setComponentId(0); } - public void buildProcessRelation(String entityId, DetectPoint detectPoint) { + public void buildProcessRelation(String entityId, int componentId, DetectPoint detectPoint) { this.setId(entityId); final IDManager.ProcessID.ProcessRelationDefine processRelationDefine = IDManager.ProcessID.analysisRelationId(entityId); this.setDetectPoint(detectPoint); this.setSource(processRelationDefine.getSourceId()); this.setTarget(processRelationDefine.getDestId()); - this.setComponentId(0); + this.setComponentId(componentId); } } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ProcessRelation.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ProcessRelation.java index 191858bd97..128514c607 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ProcessRelation.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ProcessRelation.java @@ -61,4 +61,7 @@ public class ProcessRelation extends Source { @Getter @Setter private DetectPoint detectPoint; + @Setter + @Getter + private int componentId; } diff --git a/oap-server/server-starter/src/main/resources/component-libraries.yml b/oap-server/server-starter/src/main/resources/component-libraries.yml index 7435ed290b..beea316938 100644 --- a/oap-server/server-starter/src/main/resources/component-libraries.yml +++ b/oap-server/server-starter/src/main/resources/component-libraries.yml @@ -416,6 +416,12 @@ Apache-ShenYu: Hutool: id: 128 languages: Java +https: + id: 129 + languages: ebpf +tls: + id: 130 + languages: ebpf # .NET/.NET Core components # [3000, 4000) for C#/.NET only diff --git a/oap-server/server-starter/src/main/resources/meter-analyzer-config/network-profiling.yaml b/oap-server/server-starter/src/main/resources/meter-analyzer-config/network-profiling.yaml index 326bcfb3ad..c52b2d246f 100644 --- a/oap-server/server-starter/src/main/resources/meter-analyzer-config/network-profiling.yaml +++ b/oap-server/server-starter/src/main/resources/meter-analyzer-config/network-profiling.yaml @@ -15,7 +15,7 @@ initExp: Kubernetes.startMetadataListener() expSuffix: |- - processRelation('side', ['service'], ['instance'], 'client_process_id', 'server_process_id') + processRelation('side', ['service'], ['instance'], 'client_process_id', 'server_process_id', 'component') expPrefix: |- forEach(['client', 'server'], { prefix, tags -> if (tags[prefix + '_process_id'] != null) { @@ -27,35 +27,51 @@ expPrefix: |- } tags[prefix + '_process_id'] = ProcessRegistry.generateVirtualRemoteProcess(tags.service, tags.instance, tags[prefix + '_address']) }) + .forEach(['component'], { key, tags -> + String result = "" + // protocol are defined in the component-libraries.yml + String protocol = tags['protocol'] + String ssl = tags['is_ssl'] + if (protocol == 'http' && ssl == 'true') { + result = '129' // https + } else if (protocol == 'http') { + result = '49' // http + } else if (ssl == 'true') { + result = '130' // tls + } else { + result = '110' // tcp + } + tags[key] = result + }) metricPrefix: process_relation metricsRules: # client side - name: client_write_cpm - exp: rover_net_p_client_write_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']).downsampling(SUM_PER_MIN) + exp: rover_net_p_client_write_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']).downsampling(SUM_PER_MIN) - name: client_write_total_bytes - exp: rover_net_p_client_write_bytes_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']).downsampling(SUM_PER_MIN) + exp: rover_net_p_client_write_bytes_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']).downsampling(SUM_PER_MIN) - name: client_write_avg_exe_time - exp: rover_net_p_client_write_exe_time_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']) + exp: rover_net_p_client_write_exe_time_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']) - name: client_read_cpm - exp: rover_net_p_client_read_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']).downsampling(SUM_PER_MIN) + exp: rover_net_p_client_read_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']).downsampling(SUM_PER_MIN) - name: client_read_total_bytes - exp: rover_net_p_client_read_bytes_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']).downsampling(SUM_PER_MIN) + exp: rover_net_p_client_read_bytes_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']).downsampling(SUM_PER_MIN) - name: client_read_avg_exe_time - exp: rover_net_p_client_read_exe_time_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']) + exp: rover_net_p_client_read_exe_time_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']) - name: client_write_avg_rtt_time - exp: rover_net_p_client_write_rtt_exe_time_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']) + exp: rover_net_p_client_write_rtt_exe_time_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']) - name: client_connect_cpm - exp: rover_net_p_client_connect_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']).downsampling(SUM_PER_MIN) + exp: rover_net_p_client_connect_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']).downsampling(SUM_PER_MIN) - name: client_connect_exe_time - exp: rover_net_p_client_connect_exe_time_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']) + exp: rover_net_p_client_connect_exe_time_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']) - name: client_close_cpm - exp: rover_net_p_client_close_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']).downsampling(SUM_PER_MIN) + exp: rover_net_p_client_close_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']).downsampling(SUM_PER_MIN) - name: client_close_avg_exe_time - exp: rover_net_p_client_close_exe_time_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']) + exp: rover_net_p_client_close_exe_time_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']) - name: client_retransmit_cpm - exp: rover_net_p_client_retransmit_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']).downsampling(SUM_PER_MIN) + exp: rover_net_p_client_retransmit_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']).downsampling(SUM_PER_MIN) - name: client_drop_cpm - exp: rover_net_p_client_drop_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']).downsampling(SUM_PER_MIN) + exp: rover_net_p_client_drop_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']).downsampling(SUM_PER_MIN) - name: client_write_rtt_time_percentile exp: rover_net_p_client_write_rtt_histogram.histogram().histogram_percentile([50,70,90,99]).downsampling(SUM) - name: client_write_exe_time_percentile @@ -65,31 +81,31 @@ metricsRules: # server side - name: server_write_cpm - exp: rover_net_p_server_write_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']).downsampling(SUM_PER_MIN) + exp: rover_net_p_server_write_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']).downsampling(SUM_PER_MIN) - name: server_write_total_bytes - exp: rover_net_p_server_write_bytes_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']).downsampling(SUM_PER_MIN) + exp: rover_net_p_server_write_bytes_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']).downsampling(SUM_PER_MIN) - name: server_write_avg_exe_time - exp: rover_net_p_server_write_exe_time_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']) + exp: rover_net_p_server_write_exe_time_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']) - name: server_read_cpm - exp: rover_net_p_server_read_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']).downsampling(SUM_PER_MIN) + exp: rover_net_p_server_read_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']).downsampling(SUM_PER_MIN) - name: server_read_total_bytes - exp: rover_net_p_server_read_bytes_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']).downsampling(SUM_PER_MIN) + exp: rover_net_p_server_read_bytes_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']).downsampling(SUM_PER_MIN) - name: server_read_avg_exe_time - exp: rover_net_p_server_read_exe_time_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']) + exp: rover_net_p_server_read_exe_time_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']) - name: server_write_avg_rtt_time - exp: rover_net_p_server_write_rtt_exe_time_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']) + exp: rover_net_p_server_write_rtt_exe_time_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']) - name: server_connect_cpm - exp: rover_net_p_server_connect_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']).downsampling(SUM_PER_MIN) + exp: rover_net_p_server_connect_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']).downsampling(SUM_PER_MIN) - name: server_connect_avg_exe_time - exp: rover_net_p_server_connect_exe_time_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']) + exp: rover_net_p_server_connect_exe_time_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']) - name: server_close_cpm - exp: rover_net_p_server_close_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']).downsampling(SUM_PER_MIN) + exp: rover_net_p_server_close_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']).downsampling(SUM_PER_MIN) - name: server_close_avg_exe_time - exp: rover_net_p_server_close_exe_time_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']) + exp: rover_net_p_server_close_exe_time_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']) - name: server_retransmit_cpm - exp: rover_net_p_server_retransmit_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']).downsampling(SUM_PER_MIN) + exp: rover_net_p_server_retransmit_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']).downsampling(SUM_PER_MIN) - name: server_drop_cpm - exp: rover_net_p_server_drop_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id']).downsampling(SUM_PER_MIN) + exp: rover_net_p_server_drop_counts_counter.sum(['service', 'instance', 'side', 'client_process_id', 'server_process_id', 'component']).downsampling(SUM_PER_MIN) - name: server_write_rtt_time_percentile exp: rover_net_p_server_write_rtt_histogram.histogram().histogram_percentile([50,70,90,99]).downsampling(SUM) - name: server_write_exe_time_percentile diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTopologyQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTopologyQueryDAO.java index 7afc5c80e2..71eb23c601 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTopologyQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTopologyQueryDAO.java @@ -257,7 +257,7 @@ public class BanyanDBTopologyQueryDAO extends AbstractBanyanDBDAO implements ITo ProcessRelationClientSideMetrics.INDEX_NAME; final Map callMap = new HashMap<>(); MeasureQueryResponse resp = query(modelName, - ImmutableSet.of(Metrics.ENTITY_ID), + ImmutableSet.of(Metrics.ENTITY_ID, ProcessRelationClientSideMetrics.COMPONENT_ID), Collections.emptySet(), timestampRange, new QueryBuilder() { @Override protected void apply(MeasureQuery query) { @@ -266,7 +266,8 @@ public class BanyanDBTopologyQueryDAO extends AbstractBanyanDBDAO implements ITo }); final Call.CallDetail call = new Call.CallDetail(); final String entityId = resp.getDataPoints().get(0).getTagValue(Metrics.ENTITY_ID); - call.buildProcessRelation(entityId, detectPoint); + final int componentId = ((Number) resp.getDataPoints().get(0).getTagValue(ProcessRelationClientSideMetrics.COMPONENT_ID)).intValue(); + call.buildProcessRelation(entityId, componentId, detectPoint); callMap.putIfAbsent(entityId, call); return new ArrayList<>(callMap.values()); } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TopologyQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TopologyQueryEsDAO.java index a2d5fd7320..1bd7fdb8bd 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TopologyQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TopologyQueryEsDAO.java @@ -235,6 +235,11 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO { sourceBuilder.aggregation( Aggregation .terms(Metrics.ENTITY_ID).field(Metrics.ENTITY_ID) + .subAggregation( + Aggregation.terms(ProcessRelationServerSideMetrics.COMPONENT_ID) + .field(ProcessRelationServerSideMetrics.COMPONENT_ID) + .executionHint(TermsAggregationBuilder.ExecutionHint.MAP) + .collectMode(TermsAggregationBuilder.CollectMode.BREADTH_FIRST)) .executionHint(TermsAggregationBuilder.ExecutionHint.MAP) .collectMode(TermsAggregationBuilder.CollectMode.BREADTH_FIRST) .size(1000)); @@ -255,9 +260,15 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO { (List>) entityTerms.get("buckets"); for (final Map entityBucket : buckets) { String entityId = (String) entityBucket.get("key"); + final Map componentTerms = + (Map) entityBucket.get( + ServiceRelationServerSideMetrics.COMPONENT_ID); + final List> subAgg = + (List>) componentTerms.get("buckets"); + final int componentId = ((Number) subAgg.iterator().next().get("key")).intValue(); Call.CallDetail call = new Call.CallDetail(); - call.buildProcessRelation(entityId, detectPoint); + call.buildProcessRelation(entityId, componentId, detectPoint); calls.add(call); } return calls; diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TopologyQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TopologyQueryDAO.java index 8d4e1369c8..a7d0f71380 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TopologyQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TopologyQueryDAO.java @@ -264,11 +264,11 @@ public class H2TopologyQueryDAO implements ITopologyQueryDAO { try (Connection connection = h2Client.getConnection()) { try (ResultSet resultSet = h2Client.executeQuery( connection, - "select " + Metrics.ENTITY_ID + " from " + (detectPoint == DetectPoint.SERVER ? - ProcessRelationServerSideMetrics.INDEX_NAME : ProcessRelationClientSideMetrics.INDEX_NAME) + "select " + Metrics.ENTITY_ID + ", " + ProcessRelationServerSideMetrics.COMPONENT_ID + + " from " + (detectPoint == DetectPoint.SERVER ? ProcessRelationServerSideMetrics.INDEX_NAME : ProcessRelationClientSideMetrics.INDEX_NAME) + " where " + Metrics.TIME_BUCKET + ">= ? and " + Metrics.TIME_BUCKET + "<=? and " + ProcessRelationClientSideMetrics.SERVICE_INSTANCE_ID + "=?" - + " group by " + Metrics.ENTITY_ID, + + " group by " + Metrics.ENTITY_ID + ", " + ProcessRelationServerSideMetrics.COMPONENT_ID, conditions )) { buildProcessCalls(resultSet, calls, detectPoint); @@ -316,7 +316,8 @@ public class H2TopologyQueryDAO implements ITopologyQueryDAO { while (resultSet.next()) { Call.CallDetail call = new Call.CallDetail(); String entityId = resultSet.getString(Metrics.ENTITY_ID); - call.buildProcessRelation(entityId, detectPoint); + int componentId = resultSet.getInt(ProcessRelationServerSideMetrics.COMPONENT_ID); + call.buildProcessRelation(entityId, componentId, detectPoint); calls.add(call); } } diff --git a/test/e2e-v2/cases/profiling/ebpf/network/e2e.yaml b/test/e2e-v2/cases/profiling/ebpf/network/e2e.yaml index 2eed39d541..7e7e6f294e 100644 --- a/test/e2e-v2/cases/profiling/ebpf/network/e2e.yaml +++ b/test/e2e-v2/cases/profiling/ebpf/network/e2e.yaml @@ -98,87 +98,5 @@ verify: count: 20 interval: 10s cases: - # service list - - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql service ls - expected: expected/service.yml - # service instance list - - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance list --service-name=default::productpage - expected: expected/service-instance.yml - - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql process list --service-name=default::productpage --instance-name=productpage - expected: expected/process.yml - - # create network profiling task - - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql profiling ebpf create network --service-name=default::productpage --instance-name=productpage - expected: expected/profiling-create.yml - - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql dep process --service-name default::productpage --instance-name=productpage - expected: expected/dependency-processs.yml - - # single value of process relation, client/server side read/write - - query: | - swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics linear \ - --service-name default::productpage --instance-name productpage --process-name envoy \ - --dest-service-name default::productpage --dest-instance-name productpage --dest-process-name /usr/local/bin/python \ - --name process_relation_client_write_cpm|yq e 'to_entries' - - expected: expected/metrics-has-value.yml - - query: | - swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics linear \ - --service-name default::productpage --instance-name productpage --process-name envoy \ - --dest-service-name default::productpage --dest-instance-name productpage --dest-process-name /usr/local/bin/python \ - --name process_relation_client_write_total_bytes|yq e 'to_entries' - - expected: expected/metrics-has-value.yml - - query: | - swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics linear \ - --service-name default::productpage --instance-name productpage --process-name envoy \ - --dest-service-name default::productpage --dest-instance-name productpage --dest-process-name /usr/local/bin/python \ - --name process_relation_client_write_avg_exe_time|yq e 'to_entries' - - expected: expected/metrics-has-value.yml - - query: | - swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics linear \ - --service-name default::productpage --instance-name productpage --process-name envoy \ - --dest-service-name default::productpage --dest-instance-name productpage --dest-process-name /usr/local/bin/python \ - --name process_relation_client_read_cpm|yq e 'to_entries' - - expected: expected/metrics-has-value.yml - - query: | - swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics linear \ - --service-name default::productpage --instance-name productpage --process-name envoy \ - --dest-service-name default::productpage --dest-instance-name productpage --dest-process-name /usr/local/bin/python \ - --name process_relation_server_write_cpm|yq e 'to_entries' - - expected: expected/metrics-has-value.yml - - # histogram value of process relation, client side write_rtt/write_exe/read time - - query: | - swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \ - --service-name default::productpage --instance-name productpage --process-name envoy \ - --dest-service-name default::productpage --dest-instance-name productpage --dest-process-name /usr/local/bin/python \ - --name=process_relation_client_write_rtt_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' - - expected: expected/metrics-has-value-label.yml - - query: | - swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \ - --service-name default::productpage --instance-name productpage --process-name envoy \ - --dest-service-name default::productpage --dest-instance-name productpage --dest-process-name /usr/local/bin/python \ - --name=process_relation_client_write_exe_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' - - expected: expected/metrics-has-value-label.yml - - query: | - swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \ - --service-name default::productpage --instance-name productpage --process-name envoy \ - --dest-service-name default::productpage --dest-instance-name productpage --dest-process-name /usr/local/bin/python \ - --name=process_relation_client_read_exe_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' - - expected: expected/metrics-has-value-label.yml - - query: | - swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \ - --service-name default::productpage --instance-name productpage --process-name envoy \ - --dest-service-name default::productpage --dest-instance-name productpage --dest-process-name /usr/local/bin/python \ - --name=process_relation_server_write_rtt_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' - - expected: expected/metrics-has-value-label.yml - - query: | - swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \ - --service-name default::productpage --instance-name productpage --process-name envoy \ - --dest-service-name default::productpage --dest-instance-name productpage --dest-process-name /usr/local/bin/python \ - --name=process_relation_server_write_exe_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' - - expected: expected/metrics-has-value-label.yml - - query: | - swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \ - --service-name default::productpage --instance-name productpage --process-name envoy \ - --dest-service-name default::productpage --dest-instance-name productpage --dest-process-name /usr/local/bin/python \ - --name=process_relation_server_write_exe_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' - - expected: expected/metrics-has-value-label.yml + - includes: + - ./network-cases.yaml \ No newline at end of file diff --git a/test/e2e-v2/cases/profiling/ebpf/network/es-sharding/e2e.yaml b/test/e2e-v2/cases/profiling/ebpf/network/es-sharding/e2e.yaml new file mode 100644 index 0000000000..eaeb123192 --- /dev/null +++ b/test/e2e-v2/cases/profiling/ebpf/network/es-sharding/e2e.yaml @@ -0,0 +1,103 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: kind + file: ../kind.yaml + init-system-environment: ../../../../../script/env + kind: + import-images: + - skywalking/ui:latest + - skywalking/oap:latest + expose-ports: + - namespace: istio-system + resource: service/skywalking-ui + port: 80 + steps: + - name: set PATH + command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + - name: install kubectl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh kubectl + - name: install istio + command: | + bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh istioctl + istioctl install -y --set profile=demo + kubectl label namespace default istio-injection=enabled + - name: Install helm + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh helm + - name: Install kubectl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh kubectl + - name: Install SkyWalking + command: | + rm -rf skywalking-kubernetes && git clone https://github.com/apache/skywalking-kubernetes.git + cd skywalking-kubernetes + cd chart + helm dep up skywalking + helm -n istio-system install skywalking skywalking \ + --set fullnameOverride=skywalking \ + --set elasticsearch.replicas=1 \ + --set elasticsearch.minimumMasterNodes=1 \ + --set elasticsearch.imageTag=7.5.1 \ + --set oap.env.SW_METER_ANALYZER_ACTIVE_FILES='network-profiling' \ + --set oap.env.SW_STORAGE_ES_LOGIC_SHARDING='true' \ + --set oap.envoy.als.enabled=true \ + --set oap.replicas=1 \ + --set ui.image.repository=skywalking/ui \ + --set ui.image.tag=latest \ + --set oap.image.tag=latest \ + --set oap.image.repository=skywalking/oap \ + --set oap.storageType=elasticsearch + wait: + - namespace: istio-system + resource: deployments/skywalking-oap + for: condition=available + - name: Deploy demo services + command: | + kubectl apply -f https://raw.githubusercontent.com/istio/istio/$ISTIO_VERSION/samples/bookinfo/platform/kube/bookinfo.yaml + kubectl apply -f https://raw.githubusercontent.com/istio/istio/$ISTIO_VERSION/samples/bookinfo/networking/bookinfo-gateway.yaml + kubectl apply -f https://raw.githubusercontent.com/istio/istio/$ISTIO_VERSION/samples/bookinfo/networking/destination-rule-all.yaml + kubectl apply -f https://raw.githubusercontent.com/istio/istio/$ISTIO_VERSION/samples/bookinfo/networking/virtual-service-all-v1.yaml + wait: + - namespace: default + resource: pod + for: condition=Ready + - name: Install SkyWalking Rover + command: | + envsubst < test/e2e-v2/cases/profiling/ebpf/network/rover.yaml | kubectl apply -f - + wait: + - namespace: default + resource: pod + for: condition=Ready + - name: Generate traffic + path: ../traffic-gen.yaml + wait: + - namespace: default + resource: pod + for: condition=Ready + timeout: 25m + +verify: + retry: + count: 20 + interval: 10s + cases: + - includes: + - ../network-cases.yaml \ No newline at end of file diff --git a/test/e2e-v2/cases/profiling/ebpf/network/expected/dependency-processs.yml b/test/e2e-v2/cases/profiling/ebpf/network/expected/dependency-processs.yml index 6f27cef41c..3ec39aedd3 100644 --- a/test/e2e-v2/cases/profiling/ebpf/network/expected/dependency-processs.yml +++ b/test/e2e-v2/cases/profiling/ebpf/network/expected/dependency-processs.yml @@ -23,6 +23,14 @@ {{- end }} {{- end}} +# ingress pod name +{{ $ingressProcessName := "" }} +{{ range .nodes }} +{{ if hasPrefix .name "istio-ingressgateway" }} +{{ $ingressProcessName = .name }} +{{ end }} +{{ end }} + nodes: {{- contains .nodes }} - id: {{template "genProcessId" "/usr/local/bin/python"}} @@ -39,23 +47,57 @@ nodes: serviceinstanceid: {{ b64enc "default::productpage" }}.1_{{ b64enc "productpage" }} serviceinstancename: productpage isreal: true +- id: {{template "genProcessId" $ingressProcessName }} + name: {{$ingressProcessName}} + serviceid: {{ b64enc "default::productpage" }}.1 + servicename: default::productpage + serviceinstanceid: {{ b64enc "default::productpage" }}.1_{{ b64enc "productpage" }} + serviceinstancename: productpage + isreal: false +- id: {{template "genProcessId" "kube-dns.kube-system" }} + name: kube-dns.kube-system + serviceid: {{ b64enc "default::productpage" }}.1 + servicename: default::productpage + serviceinstanceid: {{ b64enc "default::productpage" }}.1_{{ b64enc "productpage" }} + serviceinstancename: productpage + isreal: false {{- end }} calls: {{- contains .calls }} - source: {{template "genProcessId" "/usr/local/bin/python"}} - sourcecomponents: [] + sourcecomponents: + - http target: {{template "genProcessId" "envoy"}} - targetcomponents: [] + targetcomponents: + - http id: {{template "genProcessId" "/usr/local/bin/python"}}-{{template "genProcessId" "envoy"}} detectpoints: - CLIENT - SERVER - source: {{template "genProcessId" "envoy"}} - sourcecomponents: [] + sourcecomponents: + - http target: {{template "genProcessId" "/usr/local/bin/python"}} - targetcomponents: [] + targetcomponents: + - http id: {{template "genProcessId" "envoy"}}-{{template "genProcessId" "/usr/local/bin/python"}} detectpoints: - CLIENT - SERVER +- source: {{template "genProcessId" $ingressProcessName}} + sourcecomponents: [] + target: {{template "genProcessId" "envoy"}} + targetcomponents: + - https + id: {{template "genProcessId" $ingressProcessName}}-{{template "genProcessId" "envoy"}} + detectpoints: + - SERVER +- source: {{template "genProcessId" "envoy"}} + sourcecomponents: + - tcp + target: {{template "genProcessId" "kube-dns.kube-system"}} + targetcomponents: [] + id: {{template "genProcessId" "envoy"}}-{{template "genProcessId" "kube-dns.kube-system"}} + detectpoints: + - CLIENT {{- end }} \ No newline at end of file diff --git a/test/e2e-v2/cases/profiling/ebpf/network/network-cases.yaml b/test/e2e-v2/cases/profiling/ebpf/network/network-cases.yaml new file mode 100644 index 0000000000..144a7d921c --- /dev/null +++ b/test/e2e-v2/cases/profiling/ebpf/network/network-cases.yaml @@ -0,0 +1,100 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cases: + # service list + - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql service ls + expected: expected/service.yml + # service instance list + - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance list --service-name=default::productpage + expected: expected/service-instance.yml + - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql process list --service-name=default::productpage --instance-name=productpage + expected: expected/process.yml + + # create network profiling task + - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql profiling ebpf create network --service-name=default::productpage --instance-name=productpage + expected: expected/profiling-create.yml + - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql dep process --service-name default::productpage --instance-name=productpage + expected: expected/dependency-processs.yml + + # single value of process relation, client/server side read/write + - query: | + swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics linear \ + --service-name default::productpage --instance-name productpage --process-name envoy \ + --dest-service-name default::productpage --dest-instance-name productpage --dest-process-name /usr/local/bin/python \ + --name process_relation_client_write_cpm|yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: | + swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics linear \ + --service-name default::productpage --instance-name productpage --process-name envoy \ + --dest-service-name default::productpage --dest-instance-name productpage --dest-process-name /usr/local/bin/python \ + --name process_relation_client_write_total_bytes|yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: | + swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics linear \ + --service-name default::productpage --instance-name productpage --process-name envoy \ + --dest-service-name default::productpage --dest-instance-name productpage --dest-process-name /usr/local/bin/python \ + --name process_relation_client_write_avg_exe_time|yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: | + swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics linear \ + --service-name default::productpage --instance-name productpage --process-name envoy \ + --dest-service-name default::productpage --dest-instance-name productpage --dest-process-name /usr/local/bin/python \ + --name process_relation_client_read_cpm|yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: | + swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics linear \ + --service-name default::productpage --instance-name productpage --process-name envoy \ + --dest-service-name default::productpage --dest-instance-name productpage --dest-process-name /usr/local/bin/python \ + --name process_relation_server_write_cpm|yq e 'to_entries' - + expected: expected/metrics-has-value.yml + + # histogram value of process relation, client side write_rtt/write_exe/read time + - query: | + swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \ + --service-name default::productpage --instance-name productpage --process-name envoy \ + --dest-service-name default::productpage --dest-instance-name productpage --dest-process-name /usr/local/bin/python \ + --name=process_relation_client_write_rtt_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' - + expected: expected/metrics-has-value-label.yml + - query: | + swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \ + --service-name default::productpage --instance-name productpage --process-name envoy \ + --dest-service-name default::productpage --dest-instance-name productpage --dest-process-name /usr/local/bin/python \ + --name=process_relation_client_write_exe_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' - + expected: expected/metrics-has-value-label.yml + - query: | + swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \ + --service-name default::productpage --instance-name productpage --process-name envoy \ + --dest-service-name default::productpage --dest-instance-name productpage --dest-process-name /usr/local/bin/python \ + --name=process_relation_client_read_exe_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' - + expected: expected/metrics-has-value-label.yml + - query: | + swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \ + --service-name default::productpage --instance-name productpage --process-name envoy \ + --dest-service-name default::productpage --dest-instance-name productpage --dest-process-name /usr/local/bin/python \ + --name=process_relation_server_write_rtt_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' - + expected: expected/metrics-has-value-label.yml + - query: | + swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \ + --service-name default::productpage --instance-name productpage --process-name envoy \ + --dest-service-name default::productpage --dest-instance-name productpage --dest-process-name /usr/local/bin/python \ + --name=process_relation_server_write_exe_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' - + expected: expected/metrics-has-value-label.yml + - query: | + swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \ + --service-name default::productpage --instance-name productpage --process-name envoy \ + --dest-service-name default::productpage --dest-instance-name productpage --dest-process-name /usr/local/bin/python \ + --name=process_relation_server_write_exe_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' - + expected: expected/metrics-has-value-label.yml diff --git a/test/e2e-v2/script/env b/test/e2e-v2/script/env index 1d192eb4ba..94bf5980ee 100644 --- a/test/e2e-v2/script/env +++ b/test/e2e-v2/script/env @@ -22,7 +22,7 @@ SW_AGENT_PYTHON_COMMIT=c76a6ec51a478ac91abb20ec8f22a99b8d4d6a58 SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449 SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016 SW_KUBERNETES_COMMIT_SHA=0f3ec68e5a7e1608cec8688716b848ed15e971e5 -SW_ROVER_COMMIT=96bfd2a51e582f34ccbbedfd4bceb7407559de2a +SW_ROVER_COMMIT=d956eaede57b62108b78bca48045bd09ba88e653 SW_BANYANDB_COMMIT=5a326d7e36a008c5ea10e3ae506309cb29733c53 -SW_CTL_COMMIT=a73f65367b335e08412c0471e85ad8f15e92d411 +SW_CTL_COMMIT=9f750c8ffe4d4dcea63e811dc1f5f857756a443c