Self observability of OAP backend (#3140)

* First commit

* Fix e2e test and some nits

* Documenting so11y and setting default values of it in dev env

docker/oap/docker-entrypoint.sh

@@ -256,6 +256,8 @@ receiver-jvm:
   default:
 receiver-clr:
   default:
+receiver-so11y:
+  default:
 service-mesh:
   default:
     bufferPath: \${SW_SERVICE_MESH_BUFFER_PATH:../mesh-buffer/}  # Path to trace buffer files, suggest to use absolute path
@@ -270,9 +272,10 @@ query:
 alarm:
   default:
 telemetry:
-  prometheus:
-    host: \${SW_TELEMETRY_PROMETHEUS_HOST:0.0.0.0}
-    port: \${SW_TELEMETRY_PROMETHEUS_PORT:1234}
+  so11y:
+    prometheusExporterEnabled: \${SW_TELEMETRY_SO11Y_PROMETHEUS_ENABLED:true}
+    prometheusExporterHost: \${SW_TELEMETRY_PROMETHEUS_HOST:0.0.0.0}
+    prometheusExporterPort: \${SW_TELEMETRY_PROMETHEUS_PORT:1234}
 EOT
     # generate configuration
     case ${SW_CONFIGURATION} in

docs/en/setup/backend/backend-telemetry.md

@@ -27,4 +27,30 @@ telemetry:
 Provide two grafana dashboard settings.
 1. Use [SkyWalking trace-mode dashboard](telemetry/trace-mode-grafana.json) when SkyWalking is used with tracing agent.
 1. Use [SkyWalking mesh-mode dashboard](telemetry/mesh-mode-grafana.json) when SkyWalking is used with service mesh
-telemetry, including istio, envoy. 
+telemetry, including istio, envoy. 
+
+## Self Observability
+
+SkyWalking supports to collect telemetry data into OAP backend directly. Users could check them out through UI or
+GraphQL API then.
+
+Adding following configuration to enable `so11y`(self-observability) related modules.
+
+```yaml
+receiver-so11y:
+  default:
+telemetry:
+  so11y:
+```
+
+Another example represents how to combine `promethues` and `so11y`. Adding some items in `so11y` to make it happen.
+
+```yaml
+telemetry:
+  so11y:
+    prometheusExporterEnabled: true
+    prometheusExporterHost: 0.0.0.0
+    prometheusExporterPort: 1234
+```
+
+Then prometheus exporter is listening on `0.0.0.0:1234`.

oap-server/server-receiver-plugin/pom.xml

@@ -39,6 +39,7 @@
         <module>skywalking-clr-receiver-plugin</module>
         <module>jaeger-receiver-plugin</module>
         <module>receiver-proto</module>
+        <module>skywalking-so11y-receiver-plugin</module>
     </modules>
 
     <dependencies>

oap-server/server-receiver-plugin/skywalking-so11y-receiver-plugin/pom.xml

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>server-receiver-plugin</artifactId>
+        <groupId>org.apache.skywalking</groupId>
+        <version>6.3.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>skywalking-so11y-receiver-plugin</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.skywalking</groupId>
+            <artifactId>telemetry-api</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+    </dependencies>
+
+
+</project>

oap-server/server-receiver-plugin/skywalking-so11y-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/so11y/So11yReceiverConfig.java

@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.receiver.so11y;
+
+import org.apache.skywalking.oap.server.library.module.ModuleConfig;
+
+/**
+ * Self observability receiver config.
+ */
+public class So11yReceiverConfig extends ModuleConfig {
+}

oap-server/server-receiver-plugin/skywalking-so11y-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/so11y/So11yReceiverModule.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.receiver.so11y;
+
+import org.apache.skywalking.oap.server.library.module.ModuleDefine;
+
+/**
+ * Self observability receiver module.
+ *
+ * @author gaohongtao
+ */
+public class So11yReceiverModule extends ModuleDefine {
+    public static final String NAME = "receiver-so11y";
+
+    public So11yReceiverModule() {
+        super(NAME);
+    }
+
+    @Override
+    public Class[] services() {
+        return new Class[0];
+    }
+}

oap-server/server-receiver-plugin/skywalking-so11y-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/so11y/So11yReceiverModuleProvider.java

@@ -0,0 +1,381 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.receiver.so11y;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.ToString;
+import org.apache.skywalking.oap.server.core.Const;
+import org.apache.skywalking.oap.server.core.CoreModule;
+import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
+import org.apache.skywalking.oap.server.core.register.service.IServiceInstanceInventoryRegister;
+import org.apache.skywalking.oap.server.core.register.service.IServiceInventoryRegister;
+import org.apache.skywalking.oap.server.core.source.*;
+import org.apache.skywalking.oap.server.library.module.*;
+import org.apache.skywalking.oap.server.telemetry.TelemetryModule;
+import org.apache.skywalking.oap.server.telemetry.api.MetricFamily;
+import org.apache.skywalking.oap.server.telemetry.api.MetricsCollector;
+import org.apache.skywalking.oap.server.telemetry.api.TelemetryRelatedContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
+
+import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.summingDouble;
+
+/**
+ * Self observability receiver provider.
+ *
+ * @author gaohongtao
+ */
+public class So11yReceiverModuleProvider extends ModuleProvider {
+
+    private static final Logger logger = LoggerFactory.getLogger(So11yReceiverModuleProvider.class);
+
+    private static final String SERVICE_NAME = "SkyWalking";
+
+    private static final int RUN_RATE_SECONDS = 5;
+
+    private final long[] lastNewGc = new long[]{0L, 0L};
+
+    private final long[] lastOldGc = new long[]{0L, 0L};
+
+    private int serviceId;
+
+    private int serviceInstanceId;
+
+    private String serviceInstanceName;
+
+    private double lastCpuSeconds = -1;
+
+    private IServiceInventoryRegister serviceInventoryRegister;
+
+    private IServiceInstanceInventoryRegister serviceInstanceInventoryRegister;
+
+    private SourceReceiver sourceReceiver;
+
+
+    @Override
+    public String name() {
+        return "default";
+    }
+
+    @Override
+    public Class<? extends ModuleDefine> module() {
+        return So11yReceiverModule.class;
+    }
+
+    @Override
+    public ModuleConfig createConfigBeanIfAbsent() {
+        return new So11yReceiverConfig();
+    }
+
+    @Override
+    public void prepare() throws ServiceNotProvidedException, ModuleStartException {
+    }
+
+    @Override
+    public void start() throws ServiceNotProvidedException, ModuleStartException {
+        serviceInventoryRegister = getManager().find(CoreModule.NAME).provider().getService(IServiceInventoryRegister.class);
+        serviceInstanceInventoryRegister = getManager().find(CoreModule.NAME).provider().getService(IServiceInstanceInventoryRegister.class);
+        sourceReceiver = getManager().find(CoreModule.NAME).provider().getService(SourceReceiver.class);
+        MetricsCollector collector = getManager().find(TelemetryModule.NAME).provider().getService(MetricsCollector.class);
+        Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder()
+                .setDaemon(true).setNameFormat("So11y-receiver-%s").build()).scheduleAtFixedRate(() -> {
+                    if (register()) {
+                        heartbeat();
+                    } else {
+                        return;
+                    }
+                    Iterable<MetricFamily> mfs = collector.collect();
+                    Map<String, MetricFamily> metricsIndex = new HashMap<>();
+                    for (MetricFamily each : mfs) {
+                        if (each.samples.size() < 1) {
+                            continue;
+                        }
+                        metricsIndex.put(each.name, each);
+                    }
+                    writeCpuUsage(metricsIndex);
+                    writeJvmMemory(metricsIndex);
+                    writeJvmMemoryPool(metricsIndex);
+                    writeGC(metricsIndex);
+                }, RUN_RATE_SECONDS, RUN_RATE_SECONDS, TimeUnit.SECONDS);
+    }
+
+
+
+    @Override
+    public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException {
+
+    }
+
+    @Override
+    public String[] requiredModules() {
+        return new String[] {TelemetryModule.NAME, CoreModule.NAME};
+    }
+
+    private void writeGC(Map<String, MetricFamily> metricsIndex) {
+        if (!metricsIndex.containsKey("jvm_gc_collection_seconds")) {
+            return;
+        }
+        List<String> newGC = ImmutableList.of("PS Scavenge", "ParNew", "G1 Young Generation", "Copy");
+        List<String> oldGC = ImmutableList.of("PS MarkSweep", "ConcurrentMarkSweep", "G1 Old Generation", "MarkSweepCompact");
+        metricsIndex.get("jvm_gc_collection_seconds").samples.stream()
+                .map(sample -> {
+                    int index = Iterables.indexOf(sample.labelNames, i -> Objects.equals(i, "gc"));
+                    if (index < 0) {
+                        return null;
+                    }
+                    String gcPhrase = sample.labelValues.get(index);
+                    GCMetricType type = sample.name.contains("sum") ? GCMetricType.SUM : GCMetricType.COUNT;
+                    double value = type == GCMetricType.SUM ? sample.value * 1000 : sample.value;
+                    if (newGC.contains(gcPhrase)) {
+                        return new GCMetric(GCPhrase.NEW, type, value);
+                    } else if (oldGC.contains(gcPhrase)) {
+                        return new GCMetric(GCPhrase.OLD, type, value);
+                    }
+                    throw new RuntimeException(String.format("Unsupported gc phrase %s", gcPhrase));
+                })
+                .filter(Objects::nonNull)
+                .collect(groupingBy(GCMetric::getPhrase))
+                .forEach((gcPhrase, gcMetrics) -> {
+                    ServiceInstanceJVMGC gc = new ServiceInstanceJVMGC();
+                    gc.setId(serviceInstanceId);
+                    gc.setName(serviceInstanceName);
+                    gc.setServiceId(serviceId);
+                    gc.setServiceName(SERVICE_NAME);
+                    gc.setPhrase(gcPhrase);
+                    long[] lastGc = gcPhrase == GCPhrase.NEW ? lastNewGc : lastOldGc;
+                    gcMetrics.stream().filter(m -> m.type.equals(GCMetricType.COUNT)).findFirst().ifPresent(m -> {
+                        gc.setCount(m.getValue().longValue() - lastGc[0]);
+                        lastGc[0] = m.getValue().longValue();
+                    });
+                    gcMetrics.stream().filter(m -> m.type.equals(GCMetricType.SUM)).findFirst().ifPresent(m -> {
+                        gc.setTime(m.getValue().longValue() - lastGc[1]);
+                        lastGc[1] = m.getValue().longValue();
+                    });
+                    gc.setTimeBucket(TimeBucket.getMinuteTimeBucket(System.currentTimeMillis()));
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("Write {} {}counts {}ms to {}", gc.getPhrase(), gc.getCount(), gc.getTime(), gc.getName());
+                    }
+                    sourceReceiver.receive(gc);
+                });
+    }
+
+    private void writeJvmMemoryPool(Map<String, MetricFamily> metricsIndex) {
+        List<MetricSetter<ServiceInstanceJVMMemoryPool>> setterList = ImmutableList.of(
+                new MetricSetter<>("jvm_memory_pool_bytes_used", (m, v) -> m.setUsed(v.longValue())),
+                new MetricSetter<>("jvm_memory_pool_bytes_committed", (m, v) -> m.setCommitted(v.longValue())),
+                new MetricSetter<>("jvm_memory_pool_bytes_max", (m, v) -> m.setMax(v.longValue())),
+                new MetricSetter<>("jvm_memory_pool_bytes_init", (m, v) -> m.setInit(v.longValue())));
+        if (setterList.stream().anyMatch(i -> !metricsIndex.containsKey(i.name))) {
+            return;
+        }
+        Map<MemoryPoolType, ServiceInstanceJVMMemoryPool> poolMap = new HashMap<>();
+        setterList.forEach(setter -> metricsIndex.get(setter.name).samples.stream()
+                .map(sample -> {
+                    int index = Iterables.indexOf(sample.labelNames, i -> Objects.equals(i, "pool"));
+                    if (index < 0) {
+                        return null;
+                    }
+                    String poolType = sample.labelValues.get(index);
+                    if (poolType.contains("Code")) {
+                        return new PoolMetric(MemoryPoolType.CODE_CACHE_USAGE, sample.value);
+                    } else if (poolType.contains("Eden")) {
+                        return new PoolMetric(MemoryPoolType.NEWGEN_USAGE, sample.value);
+                    } else if (poolType.contains("Survivor")) {
+                        return new PoolMetric(MemoryPoolType.SURVIVOR_USAGE, sample.value);
+                    } else if (poolType.contains("Old")) {
+                        return new PoolMetric(MemoryPoolType.OLDGEN_USAGE, sample.value);
+                    } else if (poolType.contains("Metaspace")) {
+                        return new PoolMetric(MemoryPoolType.METASPACE_USAGE, sample.value);
+                    } else if (poolType.contains("Perm") || poolType.contains("Compressed Class Space")) {
+                        return new PoolMetric(MemoryPoolType.PERMGEN_USAGE, sample.value);
+                    }
+                    throw new RuntimeException(String.format("Unknown pool type %s", poolType));
+                })
+                .filter(Objects::nonNull)
+                .collect(groupingBy(PoolMetric::getType, summingDouble(PoolMetric::getValue)))
+                .forEach((memoryPoolType, value) -> {
+                    if (!poolMap.containsKey(memoryPoolType)) {
+                        ServiceInstanceJVMMemoryPool pool = new ServiceInstanceJVMMemoryPool();
+                        pool.setId(serviceInstanceId);
+                        pool.setName(serviceInstanceName);
+                        pool.setServiceId(serviceId);
+                        pool.setServiceName(SERVICE_NAME);
+                        pool.setPoolType(memoryPoolType);
+                        pool.setTimeBucket(TimeBucket.getMinuteTimeBucket(System.currentTimeMillis()));
+                        poolMap.put(memoryPoolType, pool);
+                    }
+                    ServiceInstanceJVMMemoryPool pool = poolMap.get(memoryPoolType);
+                    setter.delegated.accept(pool, value);
+                }));
+        poolMap.values().forEach(p -> {
+            if (logger.isDebugEnabled()) {
+                logger.debug("Write {} {}-{}-{}-{} to {}", p.getPoolType(),
+                        humanReadableByteCount(p.getInit(), false),
+                        humanReadableByteCount(p.getUsed(), false),
+                        humanReadableByteCount(p.getCommitted(), false),
+                        humanReadableByteCount(p.getMax(), false), p.getName());
+            }
+            sourceReceiver.receive(p);
+        });
+    }
+
+    private void writeJvmMemory(final Map<String, MetricFamily> metricsIndex) {
+        List<MetricSetter<ServiceInstanceJVMMemory>> setterList = ImmutableList.of(
+                new MetricSetter<>("jvm_memory_bytes_used", (m, v) -> m.setUsed(v.longValue())),
+                new MetricSetter<>("jvm_memory_bytes_committed", (m, v) -> m.setCommitted(v.longValue())),
+                new MetricSetter<>("jvm_memory_bytes_max", (m, v) -> m.setMax(v.longValue())),
+                new MetricSetter<>("jvm_memory_bytes_init", (m, v) -> m.setInit(v.longValue())));
+        if (setterList.stream().anyMatch(i -> !metricsIndex.containsKey(i.name))) {
+            return;
+        }
+        ImmutableList.of(createJVMMemory(true), createJVMMemory(false))
+                .forEach(memory -> {
+                    String area = memory.isHeapStatus() ? "heap" : "nonheap";
+                    setterList.forEach(setter -> {
+                        metricsIndex.get(setter.name).samples.stream()
+                                .filter(input -> {
+                                    int index = Iterables.indexOf(input.labelNames, i -> Objects.equals(i, "area"));
+                                    if (index < 0) {
+                                        return false;
+                                    }
+                                    return Objects.equals(input.labelValues.get(index), area);
+                                })
+                                .findFirst()
+                                .ifPresent(sample -> setter.delegated.accept(memory, sample.value));
+                    });
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("Write {} {}-{}-{}-{} to {}", area,
+                                humanReadableByteCount(memory.getInit(), false),
+                                humanReadableByteCount(memory.getUsed(), false),
+                                humanReadableByteCount(memory.getCommitted(), false),
+                                humanReadableByteCount(memory.getMax(), false), memory.getName());
+                    }
+                    sourceReceiver.receive(memory);
+                });
+    }
+
+    private ServiceInstanceJVMMemory createJVMMemory(boolean isHeap) {
+        ServiceInstanceJVMMemory memory = new ServiceInstanceJVMMemory();
+        memory.setId(serviceInstanceId);
+        memory.setName(serviceInstanceName);
+        memory.setServiceId(serviceId);
+        memory.setServiceName(SERVICE_NAME);
+        memory.setHeapStatus(isHeap);
+        memory.setTimeBucket(TimeBucket.getMinuteTimeBucket(System.currentTimeMillis()));
+        return memory;
+    }
+
+    private void writeCpuUsage(Map<String, MetricFamily> metricsIndex) {
+        if (!metricsIndex.containsKey("process_cpu_seconds_total")) {
+            return;
+        }
+        double value = metricsIndex.get("process_cpu_seconds_total").samples.get(0).value;
+        if (lastCpuSeconds < 0) {
+            lastCpuSeconds = value;
+            return;
+        }
+        double percentage = (value - lastCpuSeconds) * 100 / (RUN_RATE_SECONDS * Runtime.getRuntime().availableProcessors());
+        lastCpuSeconds = value;
+        ServiceInstanceJVMCPU serviceInstanceJVMCPU = new ServiceInstanceJVMCPU();
+        serviceInstanceJVMCPU.setId(serviceInstanceId);
+        serviceInstanceJVMCPU.setName(serviceInstanceName);
+        serviceInstanceJVMCPU.setServiceId(serviceId);
+        serviceInstanceJVMCPU.setServiceName(SERVICE_NAME);
+        serviceInstanceJVMCPU.setUsePercent(percentage);
+        serviceInstanceJVMCPU.setTimeBucket(TimeBucket.getMinuteTimeBucket(System.currentTimeMillis()));
+        logger.debug("Write so11y cpu usage {} to {}", percentage, serviceInstanceName);
+        sourceReceiver.receive(serviceInstanceJVMCPU);
+    }
+
+
+    private void heartbeat() {
+        long now = System.currentTimeMillis();
+        serviceInventoryRegister.heartbeat(serviceId, now);
+        serviceInstanceInventoryRegister.heartbeat(serviceInstanceId, now);
+    }
+
+    private boolean register() {
+        if (serviceId == Const.NONE) {
+            logger.debug("Register so11y service [{}].", SERVICE_NAME);
+            serviceId = serviceInventoryRegister.getOrCreate(SERVICE_NAME, null);
+        }
+        if (serviceId != Const.NONE && serviceInstanceId == Const.NONE) {
+            serviceInstanceName = TelemetryRelatedContext.INSTANCE.getId();
+            logger.debug("Register so11y service instance [{}].", serviceInstanceName);
+            serviceInstanceId = serviceInstanceInventoryRegister.getOrCreate(serviceId, serviceInstanceName, serviceInstanceName,
+                    System.currentTimeMillis(), null);
+        }
+        return serviceInstanceId != Const.NONE;
+    }
+
+    @RequiredArgsConstructor
+    private class MetricSetter<T> {
+
+        final String name;
+
+        final BiConsumer<T, Double> delegated;
+
+    }
+
+    private static String humanReadableByteCount(long bytes, boolean si) {
+        int unit = si ? 1000 : 1024;
+        if (bytes < unit) return bytes + " B";
+        int exp = (int) (Math.log(bytes) / Math.log(unit));
+        String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp - 1) + (si ? "" : "i");
+        return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre);
+    }
+
+    @RequiredArgsConstructor
+    @EqualsAndHashCode(of = "type")
+    @ToString
+    @Getter
+    private class PoolMetric {
+        private final MemoryPoolType type;
+        private final Double value;
+    }
+
+    @RequiredArgsConstructor
+    @EqualsAndHashCode(of = "phrase")
+    @ToString
+    @Getter
+    private class GCMetric {
+        private final GCPhrase phrase;
+        private final GCMetricType type;
+        private final Double value;
+    }
+
+    private enum GCMetricType {
+        SUM, COUNT
+    }
+
+}

oap-server/server-receiver-plugin/skywalking-so11y-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine

@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+org.apache.skywalking.oap.server.receiver.so11y.So11yReceiverModule

oap-server/server-receiver-plugin/skywalking-so11y-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider

@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+org.apache.skywalking.oap.server.receiver.so11y.So11yReceiverModuleProvider

oap-server/server-starter/pom.xml

@@ -117,6 +117,11 @@
             <artifactId>skywalking-clr-receiver-plugin</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.skywalking</groupId>
+            <artifactId>skywalking-so11y-receiver-plugin</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <!-- receiver module -->
 
         <!-- storage module -->
@@ -163,6 +168,11 @@
             <artifactId>telemetry-prometheus</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.skywalking</groupId>
+            <artifactId>telemetry-so11y</artifactId>
+            <version>${project.version}</version>
+        </dependency>
 
         <!-- exporter -->
         <dependency>

oap-server/server-starter/src/main/resources/application.yml

@@ -106,6 +106,8 @@ receiver-jvm:
   default:
 receiver-clr:
   default:
+#receiver-so11y:
+#  default:
 service-mesh:
   default:
     bufferPath: ${SW_SERVICE_MESH_BUFFER_PATH:../mesh-buffer/}  # Path to trace buffer files, suggest to use absolute path
@@ -132,9 +134,14 @@ query:
 alarm:
   default:
 telemetry:
-  prometheus:
-    host: ${SW_TELEMETRY_PROMETHEUS_HOST:0.0.0.0}
-    port: ${SW_TELEMETRY_PROMETHEUS_PORT:1234}
+  none:
+#  prometheus:
+#    host: ${SW_TELEMETRY_PROMETHEUS_HOST:0.0.0.0}
+#    port: ${SW_TELEMETRY_PROMETHEUS_PORT:1234}
+#  so11y:
+#    prometheusExporterEnabled: ${SW_TELEMETRY_SO11Y_PROMETHEUS_ENABLED:true}
+#    prometheusExporterHost: ${SW_TELEMETRY_PROMETHEUS_HOST:0.0.0.0}
+#    prometheusExporterPort: ${SW_TELEMETRY_PROMETHEUS_PORT:1234}
 configuration:
   none:
 #  apollo:

oap-server/server-starter/src/main/resources/log4j2.xml

@@ -31,12 +31,12 @@
         <logger name="io.grpc.netty" level="INFO"/>
         <logger name="io.netty" level="INFO"/>
         <logger name="org.apache.http" level="INFO"/>
-        <logger name="org.apache.skywalking.oap.server.receiver.istio.telemetry" level="DEBUG"/>
         <logger name="org.apache.skywalking.oap.server.core.alarm.AlarmStandardPersistence" level="DEBUG"/>
         <logger name="org.apache.skywalking.oap.server.core" level="INFO"/>
         <logger name="org.apache.skywalking.oap.server.core.remote.client" level="DEBUG"/>
         <logger name="org.apache.skywalking.oap.server.library.buffer" level="INFO"/>
-        <Root level="DEBUG">
+        <logger name="org.apache.skywalking.oap.server.receiver.so11y" level="DEBUG" />
+        <Root level="ERROR">
             <AppenderRef ref="Console"/>
         </Root>
     </Loggers>

oap-server/server-telemetry/pom.xml

@@ -30,5 +30,6 @@
     <modules>
         <module>telemetry-prometheus</module>
         <module>telemetry-api</module>
+        <module>telemetry-so11y</module>
     </modules>
 </project>

oap-server/server-telemetry/telemetry-api/src/main/java/org/apache/skywalking/oap/server/telemetry/TelemetryModule.java

@@ -19,6 +19,7 @@
 package org.apache.skywalking.oap.server.telemetry;
 
 import org.apache.skywalking.oap.server.library.module.ModuleDefine;
+import org.apache.skywalking.oap.server.telemetry.api.MetricsCollector;
 import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator;
 
 /**
@@ -34,6 +35,6 @@ public class TelemetryModule extends ModuleDefine {
     }
 
     @Override public Class[] services() {
-        return new Class[] {MetricsCreator.class};
+        return new Class[] {MetricsCreator.class, MetricsCollector.class};
     }
 }

oap-server/server-telemetry/telemetry-api/src/main/java/org/apache/skywalking/oap/server/telemetry/api/MetricFamily.java

@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.telemetry.api;
+
+import lombok.AllArgsConstructor;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
+
+import java.util.List;
+
+/**
+ * MetricFamily define a metric and all its samples.
+ *
+ * @author gaohongtao
+ */
+@AllArgsConstructor
+@EqualsAndHashCode
+@ToString
+public class MetricFamily {
+
+    public final String name;
+    public final Type type;
+    public final String help;
+    public final List<Sample> samples;
+
+    public enum Type {
+        COUNTER,
+        GAUGE,
+        SUMMARY,
+        HISTOGRAM,
+        UNTYPED,
+    }
+
+    /**
+     * A single Sample, with a unique name and set of labels.
+     */
+    @AllArgsConstructor
+    @EqualsAndHashCode
+    @ToString
+    public static class Sample {
+        public final String name;
+        public final List<String> labelNames;
+        public final List<String> labelValues;  // Must have same length as labelNames.
+        public final double value;
+        public final Long timestampMs;  // It's an epoch format with milliseconds value included (this field is subject to change).
+
+    }
+}

oap-server/server-telemetry/telemetry-api/src/main/java/org/apache/skywalking/oap/server/telemetry/api/MetricsCollector.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.telemetry.api;
+
+import org.apache.skywalking.oap.server.library.module.Service;
+
+/**
+ * Collect all metrics from telemetry.
+ *
+ * @author gaohongtao
+ */
+public interface MetricsCollector extends Service {
+
+    /**
+     * Get all of metrics.
+     *
+     * @return all metrics
+     */
+    Iterable<MetricFamily> collect();
+}

oap-server/server-telemetry/telemetry-api/src/main/java/org/apache/skywalking/oap/server/telemetry/none/MetricsCollectorNoop.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.telemetry.none;
+
+import org.apache.skywalking.oap.server.telemetry.api.MetricFamily;
+import org.apache.skywalking.oap.server.telemetry.api.MetricsCollector;
+
+import java.util.Collections;
+
+/**
+ * No-op MetricFamily Collector.
+ *
+ * @author gaohongtao
+ */
+public class MetricsCollectorNoop implements MetricsCollector {
+    @Override
+    public Iterable<MetricFamily> collect() {
+        return Collections.emptyList();
+    }
+}

oap-server/server-telemetry/telemetry-api/src/main/java/org/apache/skywalking/oap/server/telemetry/none/NoneTelemetryProvider.java

@@ -20,6 +20,7 @@ package org.apache.skywalking.oap.server.telemetry.none;
 
 import org.apache.skywalking.oap.server.library.module.*;
 import org.apache.skywalking.oap.server.telemetry.TelemetryModule;
+import org.apache.skywalking.oap.server.telemetry.api.MetricsCollector;
 import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator;
 
 /**
@@ -43,6 +44,7 @@ public class NoneTelemetryProvider extends ModuleProvider {
 
     @Override public void prepare() throws ServiceNotProvidedException, ModuleStartException {
         this.registerServiceImplementation(MetricsCreator.class, new MetricsCreatorNoop());
+        this.registerServiceImplementation(MetricsCollector.class, new MetricsCollectorNoop());
     }
 
     @Override public void start() throws ServiceNotProvidedException, ModuleStartException {

oap-server/server-telemetry/telemetry-so11y/pom.xml

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>server-telemetry</artifactId>
+        <groupId>org.apache.skywalking</groupId>
+        <version>6.3.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>telemetry-so11y</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.skywalking</groupId>
+            <artifactId>telemetry-prometheus</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+    </dependencies>
+
+
+</project>

oap-server/server-telemetry/telemetry-so11y/src/main/java/org/apache/skywalking/oap/server/telemetry/so11y/So11yConfig.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.telemetry.so11y;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.skywalking.oap.server.library.module.ModuleConfig;
+
+/**
+ * The module configuration of self observability.
+ */
+@Setter
+@Getter
+public class So11yConfig extends ModuleConfig {
+
+    private boolean prometheusExporterEnabled = false;
+
+    private String prometheusExporterHost = "0.0.0.0";
+
+    private int prometheusExporterPort = 1234;
+
+}

oap-server/server-telemetry/telemetry-so11y/src/main/java/org/apache/skywalking/oap/server/telemetry/so11y/So11yMetricsCollector.java

@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.telemetry.so11y;
+
+import io.prometheus.client.Collector;
+import io.prometheus.client.CollectorRegistry;
+import org.apache.skywalking.oap.server.telemetry.api.MetricFamily;
+import org.apache.skywalking.oap.server.telemetry.api.MetricsCollector;
+
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Implement MetricCollector to generate prometheus metrics.
+ *
+ * @author gaohongtao
+ */
+public class So11yMetricsCollector implements MetricsCollector {
+    @Override
+    public Iterable<MetricFamily> collect() {
+        Enumeration<Collector.MetricFamilySamples> mfs = CollectorRegistry.defaultRegistry.metricFamilySamples();
+        List<MetricFamily> result = new LinkedList<>();
+        while (mfs.hasMoreElements()) {
+            Collector.MetricFamilySamples metricFamilySamples = mfs.nextElement();
+            List<MetricFamily.Sample> samples = new ArrayList<>(metricFamilySamples.samples.size());
+            MetricFamily m = new MetricFamily(metricFamilySamples.name, MetricFamily.Type.valueOf(metricFamilySamples.type.name()),
+                    metricFamilySamples.help, samples);
+            result.add(m);
+            for (Collector.MetricFamilySamples.Sample sample: metricFamilySamples.samples) {
+                samples.add(new MetricFamily.Sample(sample.name, sample.labelNames, sample.labelValues, sample.value, sample.timestampMs));
+            }
+        }
+        return result;
+    }
+}

oap-server/server-telemetry/telemetry-so11y/src/main/java/org/apache/skywalking/oap/server/telemetry/so11y/So11yMetricsCreator.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.telemetry.so11y;
+
+import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics;
+import org.apache.skywalking.oap.server.telemetry.api.GaugeMetrics;
+import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics;
+import org.apache.skywalking.oap.server.telemetry.api.MetricsTag;
+import org.apache.skywalking.oap.server.telemetry.prometheus.PrometheusMetricsCreator;
+
+/**
+ * Delegate prometheus metrics creator.
+ *
+ * @author gaohongtao
+ */
+public class So11yMetricsCreator extends PrometheusMetricsCreator {
+
+    @Override
+    public CounterMetrics createCounter(String name, String tips, MetricsTag.Keys tagKeys, MetricsTag.Values tagValues) {
+        return super.createCounter(name, tips, tagKeys, tagValues);
+    }
+
+    @Override
+    public GaugeMetrics createGauge(String name, String tips, MetricsTag.Keys tagKeys, MetricsTag.Values tagValues) {
+        return super.createGauge(name, tips, tagKeys, tagValues);
+    }
+
+    @Override
+    public HistogramMetrics createHistogramMetric(String name, String tips, MetricsTag.Keys tagKeys, MetricsTag.Values tagValues, double... buckets) {
+        return super.createHistogramMetric(name, tips, tagKeys, tagValues, buckets);
+    }
+}

oap-server/server-telemetry/telemetry-so11y/src/main/java/org/apache/skywalking/oap/server/telemetry/so11y/So11yTelemetryProvider.java

@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.telemetry.so11y;
+
+import io.prometheus.client.exporter.HTTPServer;
+import io.prometheus.client.hotspot.DefaultExports;
+import org.apache.skywalking.oap.server.library.module.*;
+import org.apache.skywalking.oap.server.telemetry.TelemetryModule;
+import org.apache.skywalking.oap.server.telemetry.api.MetricsCollector;
+import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator;
+
+import java.io.IOException;
+
+/**
+ * Self observability telemetry provider.
+ *
+ * @author gaohongtao
+ */
+public class So11yTelemetryProvider extends ModuleProvider {
+    private So11yConfig config;
+
+    public So11yTelemetryProvider() {
+        config = new So11yConfig();
+    }
+
+    @Override public String name() {
+        return "so11y";
+    }
+
+    @Override public Class<? extends ModuleDefine> module() {
+        return TelemetryModule.class;
+    }
+
+    @Override public ModuleConfig createConfigBeanIfAbsent() {
+        return config;
+    }
+
+    @Override public void prepare() throws ServiceNotProvidedException, ModuleStartException {
+        this.registerServiceImplementation(MetricsCreator.class, new So11yMetricsCreator());
+        this.registerServiceImplementation(MetricsCollector.class, new So11yMetricsCollector());
+        if (config.isPrometheusExporterEnabled()) {
+            try {
+                new HTTPServer(config.getPrometheusExporterHost(), config.getPrometheusExporterPort());
+            } catch (IOException e) {
+                throw new ModuleStartException(e.getMessage(), e);
+            }
+        }
+        DefaultExports.initialize();
+    }
+
+    @Override public void start() throws ServiceNotProvidedException, ModuleStartException {
+
+    }
+
+    @Override public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException {
+    }
+
+    @Override public String[] requiredModules() {
+        return new String[0];
+    }
+}

oap-server/server-telemetry/telemetry-so11y/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider

@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+
+org.apache.skywalking.oap.server.telemetry.so11y.So11yTelemetryProvider