diff --git a/.github/workflows/e2e.js.yaml b/.github/workflows/e2e.js.yaml new file mode 100644 index 000000000..d0a87ef7d --- /dev/null +++ b/.github/workflows/e2e.js.yaml @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: E2E + +on: + pull_request: + push: + branches: + - master + tags: + - 'v*' +env: + SKIP_TEST: true + +jobs: + JavaScriptClient: + name: Java Script Client + runs-on: ubuntu-latest + strategy: + matrix: + storage: ['h2', 'mysql', 'es6', 'es7', 'influxdb'] + env: + SW_STORAGE: ${{ matrix.storage }} + steps: + - uses: actions/checkout@v2 + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 + - name: Compile and Build + run: make docker && ES_VERSION=es7 TAG=latest-es7 make docker + - name: Copy dist package + run: cp -R dist test/e2e/ + - name: Browser Receiver ${{ matrix.storage }} + run: ./mvnw --batch-mode -f test/e2e/pom.xml -am -DfailIfNoTests=false verify -Dit.test=org.apache.skywalking.e2e.browser.BrowserE2E + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: logs + path: logs \ No newline at end of file diff --git a/README.md b/README.md index be2bc0b9b..6f88b888e 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ The core features are following. - Distributed tracing and context propagation - Database access metrics. Detect slow database access statements(including SQL statements). - Alarm - +- Browser performance monitoring diff --git a/docs/README.md b/docs/README.md index 1e86a230c..982a920ea 100644 --- a/docs/README.md +++ b/docs/README.md @@ -66,6 +66,8 @@ If you are already familiar with SkyWalking, you could use this catalog to find * [Plugin development guide](en/setup/service-agent/java-agent/README.md#plugin-development-guide). * [Agent plugin tests and performance tests](en/setup/service-agent/java-agent/README.md#test). * [Other language agents](en/setup/README.md#language-agents-in-service) includes Nginx LUA, Python, .NetCore, PHP, NodeJS, Go. + * Browser performance monitoring + * Track the performance of the browser, such as latency of redirect, dns, ttfb. For more information, [click here](https://github.com/apache/skywalking-client-js). * Service Mesh * [SkyWalking on Istio](en/setup/istio/README.md). Introduces how to use Istio Mixer bypass Adapter to work with SkyWalking. * Use [ALS (access log service)](https://www.envoyproxy.io/docs/envoy/latest/api-v2/service/accesslog/v2/als.proto) to observe service mesh, without Mixer. Follow [document](en/setup/envoy/als_setting.md) to open it. diff --git a/docs/en/concepts-and-designs/oal.md b/docs/en/concepts-and-designs/oal.md index 195b5f7fe..c297072df 100644 --- a/docs/en/concepts-and-designs/oal.md +++ b/docs/en/concepts-and-designs/oal.md @@ -53,6 +53,12 @@ In this case, input are request of each ServiceInstanceJVMCPU scope, avg is base > endpoint_percent = from(Endpoint.*).percent(status == true); In this case, all input are requests of each endpoint, condition is `endpoint.status == true`. +- `rate`. The rate expressed as a fraction of 100, for the condition matched input. +> browser_app_error_rate = from(BrowserAppTraffic.*).rate(trafficCategory == BrowserAppTrafficCategory.FIRST_ERROR, trafficCategory == BrowserAppTrafficCategory.NORMAL); + +In this case, all input are requests of each browser app traffic, `numerator` condition is `trafficCategory == BrowserAppTrafficCategory.FIRST_ERROR` and `denominator` condition is `trafficCategory == BrowserAppTrafficCategory.NORMAL`. +The parameter (1) is the `numerator` condition. +The parameter (2) is the `denominator` condition. - `sum`. The sum calls per scope entity. > service_calls_sum = from(Service.*).sum(); diff --git a/docs/en/concepts-and-designs/scope-definitions.md b/docs/en/concepts-and-designs/scope-definitions.md index 0a368aa73..f69d1bf0b 100644 --- a/docs/en/concepts-and-designs/scope-definitions.md +++ b/docs/en/concepts-and-designs/scope-definitions.md @@ -19,8 +19,8 @@ Calculate the metrics data from each request of the service. | Name | Remarks | Group Key | Type | |---|---|---|---| -| id | Represent the unique id of the service | yes | int | | name | Represent the name of the service | | string | +| nodeType | Represent which kind of node of Service or Network address represents to, Such as: Normal, Database, MQ, Cache. | | enum | | serviceInstanceName | Represent the name of the service instance id referred | | string | | endpointName | Represent the name of the endpoint, such a full path of HTTP URI | | string | | latency | Represent how much time of each request. | | int | @@ -34,9 +34,9 @@ Calculate the metrics data from each request of the service instance. | Name | Remarks | Group Key | Type | |---|---|---|---| -| id | Represent the unique id of the service instance, usually a number. | yes | int | -| name | Represent the name of the service instance. Such as `ip:port@Service Name`. **Notice**: current native agent uses `processId@Service name` as instance name, which is useless when you want to setup a filter in aggregation. | | string| +| name | Represent the name of the service instance. Such as `ip:port@Service Name`. **Notice**: current native agent uses `uuid@ipv4` as instance name, which is useless when you want to setup a filter in aggregation. | | string| | serviceName | Represent the name of the service. | | string | +| nodeType | Represent which kind of node of Service or Network address represents to, Such as: Normal, Database, MQ, Cache. | | enum | | endpointName | Represent the name of the endpoint, such a full path of HTTP URI. | | string| | latency | Represent how much time of each request. | | int | | status | Represent whether success or fail of the request. | | bool(true for success) | @@ -51,8 +51,7 @@ Calculate the metrics data if the service instance is a JVM and collected by jav | Name | Remarks | Group Key | Type | |---|---|---|---| -| id | Represent the unique id of the service instance, usually a number. | yes | int | -| name | Represent the name of the service instance. Such as `ip:port@Service Name`. **Notice**: current native agent uses `processId@Service name` as instance name, which is useless when you want to setup a filter in aggregation. | | string| +| name | Represent the name of the service instance. Such as `ip:port@Service Name`. **Notice**: current native agent uses `uuid@ipv4` as instance name, which is useless when you want to setup a filter in aggregation. | | string| | serviceName | Represent the name of the service. | | string | | usePercent | Represent how much percent of cpu time cost| | double| @@ -60,8 +59,7 @@ Calculate the metrics data if the service instance is a JVM and collected by jav | Name | Remarks | Group Key | Type | |---|---|---|---| -| id | Represent the unique id of the service instance, usually a number. | yes | int | -| name | Represent the name of the service instance. Such as `ip:port@Service Name`. **Notice**: current native agent uses `processId@Service name` as instance name, which is useless when you want to setup a filter in aggregation. | | string| +| name | Represent the name of the service instance. Such as `ip:port@Service Name`. **Notice**: current native agent uses `uuid@ipv4` as instance name, which is useless when you want to setup a filter in aggregation. | | string| | serviceName | Represent the name of the service. | | string | | heapStatus | Represent this value the memory metrics values are heap or not | | bool | | init | See JVM document | | long | @@ -73,8 +71,7 @@ Calculate the metrics data if the service instance is a JVM and collected by jav | Name | Remarks | Group Key | Type | |---|---|---|---| -| id | Represent the unique id of the service instance, usually a number. | yes | int | -| name | Represent the name of the service instance. Such as `ip:port@Service Name`. **Notice**: current native agent uses `processId@Service name` as instance name, which is useless when you want to setup a filter in aggregation. | | string| +| name | Represent the name of the service instance. Such as `ip:port@Service Name`. **Notice**: current native agent uses `uuid@ipv4` as instance name, which is useless when you want to setup a filter in aggregation. | | string| | serviceName | Represent the name of the service. | | string | | poolType | Include CODE_CACHE_USAGE, NEWGEN_USAGE, OLDGEN_USAGE, SURVIVOR_USAGE, PERMGEN_USAGE, METASPACE_USAGE based on different version of JVM. | | enum | | init | See JVM document | | long | @@ -86,8 +83,7 @@ Calculate the metrics data if the service instance is a JVM and collected by jav | Name | Remarks | Group Key | Type | |---|---|---|---| -| id | Represent the unique id of the service instance, usually a number. | yes | int | -| name | Represent the name of the service instance. Such as `ip:port@Service Name`. **Notice**: current native agent uses `processId@Service name` as instance name, which is useless when you want to setup a filter in aggregation. | | string| +| name | Represent the name of the service instance. Such as `ip:port@Service Name`. **Notice**: current native agent uses `uuid@ipv4` as instance name, which is useless when you want to setup a filter in aggregation. | | string| | serviceName | Represent the name of the service. | | string | | phrase | Include NEW and OLD | | Enum | | time | GC time cost | | long | @@ -97,8 +93,7 @@ Calculate the metrics data if the service instance is a JVM and collected by jav | Name | Remarks | Group Key | Type | |---|---|---|---| -| id | Represent the unique id of the service instance, usually a number. | yes | int | -| name | Represent the name of the service instance. Such as `ip:port@Service Name`. **Notice**: current native agent uses `processId@Service name` as instance name, which is useless when you want to setup a filter in aggregation. | | string| +| name | Represent the name of the service instance. Such as `ip:port@Service Name`. **Notice**: current native agent uses `uuid@ipv4` as instance name, which is useless when you want to setup a filter in aggregation. | | string| | serviceName | Represent the name of the service. | | string | | liveCount | Represent Current number of live threads | | int | | daemonCount | Represent Current number of daemon threads | | int | @@ -110,9 +105,9 @@ Calculate the metrics data from each request of the endpoint in the service. | Name | Remarks | Group Key | Type | |---|---|---|---| -| id | Represent the unique id of the endpoint, usually a number. | yes | int | | name | Represent the name of the endpoint, such a full path of HTTP URI. | | string | | serviceName | Represent the name of the service. | | string | +| serviceNodeType | Represent which kind of node of Service or Network address represents to, Such as: Normal, Database, MQ, Cache. | | enum | | serviceInstanceName | Represent the name of the service instance id referred. | | string | | latency | Represent how much time of each request. | | int | | status | Represent whether success or fail of the request.| | bool(true for success) | @@ -125,11 +120,11 @@ Calculate the metrics data from each request between one service and the other s | Name | Remarks | Group Key | Type | |---|---|---|---| -| sourceServiceId | Represent the id of the source service. | yes | int | | sourceServiceName | Represent the name of the source service. | | string | +| sourceServiceNodeType | Represent which kind of node of Service or Network address represents to, Such as: Normal, Database, MQ, Cache. | | enum | | sourceServiceInstanceName | Represent the name of the source service instance. | | string | -| destServiceId | Represent the id of the destination service. | yes | string | | destServiceName | Represent the name of the destination service. | | string | +| destServiceNodeType | Represent which kind of node of Service or Network address represents to. | | enum | | destServiceInstanceName | Represent the name of the destination service instance.| | string| | endpoint | Represent the endpoint used in this call. | | string | componentId | Represent the id of component used in this call. | yes | string @@ -146,11 +141,11 @@ Calculate the metrics data from each request between one service instance and th | Name | Remarks | Group Key | Type | |---|---|---|---| -| sourceServiceInstanceId | Represent the id of the source service instance. | yes | int| | sourceServiceName | Represent the name of the source service. | | string | +| sourceServiceNodeType | Represent which kind of node of Service or Network address represents to, Such as: Normal, Database, MQ, Cache. | | enum | | sourceServiceInstanceName | Represent the name of the source service instance. | | string | | destServiceName | Represent the name of the destination service. | | | -| destServiceInstanceId | Represent the id of the destination service instance. | yes | int| +| destServiceNodeType | Represent which kind of node of Service or Network address represents to, Such as: Normal, Database, MQ, Cache. | | string | | destServiceInstanceName | Represent the name of the destination service instance. | | string | | endpoint | Represent the endpoint used in this call. | | string | componentId | Represent the id of component used in this call. | yes | string @@ -169,13 +164,76 @@ including auto instrument agents(like Java, .NET), OpenCensus SkyWalking exporte | Name | Remarks | Group Key | Type | |---|---|---|---| -| endpointId | Represent the id of the endpoint as parent in the dependency. | yes | int | -| endpoint | Represent the endpoint as parent in the dependency.| | string| -| childEndpointId | Represent the id of the endpoint being used by the parent endpoint in row(1) | yes | int| -| childEndpoint| Represent the endpoint being used by the parent endpoint in row(2) | | string | +| endpoint | Represent the endpoint as parent in the dependency.| | string| +| serviceName | Represent the name of the service. | | string | +| serviceNodeType | Represent which kind of node of Service or Network address represents to, Such as: Normal, Database, MQ, Cache. | | enum | +| childEndpoint| Represent the endpoint being used by the parent endpoint in row(1) | | string | +| childServiceName | Represent the endpoint being used by the parent service in row(1) | | string | +| childServiceNodeType | Represent which kind of node of Service or Network address represents to, Such as: Normal, Database, MQ, Cache. | | string | +| childServiceInstanceName | Represent the endpoint being used by the parent service instance in row(1) | | string | | rpcLatency | Represent the latency of the RPC from some codes in the endpoint to the childEndpoint. Exclude the latency caused by the endpoint(1) itself. | componentId | Represent the id of component used in this call. | yes | string | status | Represent whether success or fail of the request.| | bool(true for success) | | responseCode | Represent the response code of HTTP response, if this request is the HTTP call. | | int | | type | Represent the type of each request. Such as: Database, HTTP, RPC, gRPC. | | enum | | detectPoint | Represent where is the relation detected. Values: client, server, proxy. | yes | enum| + + +### SCOPE `BrowserAppTraffic` + +Calculate the metrics data form each request of the browser app (only browser). + +| Name | Remarks | Group Key | Type | +|---|---|---|---| +| name | Represent the browser app name of each request. | | string | +| count | Represents the number of request, fixed at 1. | | int | +| trafficCategory | Represents traffic category, Values: NORMAL, FIRST_ERROR, ERROR | | enum | +| errorCategory | Represents error category, Values: AJAX, RESOURCE, VUE, PROMISE, UNKNOWN | | enum | + +### SCOPE `BrowserAppSingleVersionTraffic` + +Calculate the metrics data form each request of the browser single version in the browser app (only browser). + +| Name | Remarks | Group Key | Type | +|---|---|---|---| +| name | Represent the single version name of each request. | | string | +| serviceName | Represent the name of the browser app. | | string | +| count | Represents the number of request, fixed at 1. | | int | +| trafficCategory | Represents traffic category, Values: NORMAL, FIRST_ERROR, ERROR | | enum | +| errorCategory | Represents error category, Values: AJAX, RESOURCE, VUE, PROMISE, UNKNOWN | | enum | + +### SCOPE `BrowserAppPageTraffic` + +Calculate the metrics data form each request of the page in the browser app (only browser). + +| Name | Remarks | Group Key | Type | +|---|---|---|---| +| name | Represent the page name of each request. | | string | +| serviceName | Represent the name of the browser app. | | string | +| count | Represents the number of request, fixed at 1. | | int | +| trafficCategory | Represents the traffic category, Values: NORMAL, FIRST_ERROR, ERROR | | enum | +| errorCategory | Represents the error category, Values: AJAX, RESOURCE, VUE, PROMISE, UNKNOWN | | enum | + + +### SCOPE `BrowserAppPagePerf` + +Calculate the metrics data form each request of the page in the browser app (only browser). + +| Name | Remarks | Group Key | Type | +|---|---|---|---| +| name | Represent the page name of each request. | | string | +| serviceName | Represent the name of the browser app. | | string | +| redirectTime | Represents the time of redirection. | | int(in ms) | +| dnsTime | Represents the DNS query time. | | int(in ms) | +| ttfbTime | Time to first Byte. | | int(in ms) | +| tcpTime | TCP connection time. | | int(in ms) | +| transTime | Content transfer time. | | int(in ms) | +| domAnalysisTime | Dom parsing time. | | int(in ms) | +| fptTime | First paint time or blank screen time. | | int(in ms) | +| domReadyTime | Dom ready time. | | int(in ms) | +| loadPageTime | Page full load time. | | int(in ms) | +| resTime | Synchronous load resources in the page. | | int(in ms) | +| sslTime | Only valid for HTTPS. | | int(in ms) | +| ttlTime | Time to interact. | | int(in ms) | +| firstPackTime | First pack time. | | int(in ms) | +| fmpTime | First Meaningful Paint. | | int(in ms) | \ No newline at end of file diff --git a/docs/en/guides/storage-extention.md b/docs/en/guides/storage-extention.md index 5436332f1..371b90810 100644 --- a/docs/en/guides/storage-extention.md +++ b/docs/en/guides/storage-extention.md @@ -33,6 +33,7 @@ Here is the list of all DAO interfaces in storage 1. IRegisterDAO 1. ILogQueryDAO 1. ITopNRecordsQueryDAO +1. IBrowserLogQueryDAO ## Register all service implementations In `public void prepare()`, use `this#registerServiceImplementation` method to do register binding your implementation with the above interfaces. diff --git a/docs/en/protocols/Browser-Protocol.md b/docs/en/protocols/Browser-Protocol.md new file mode 100644 index 000000000..679ecaf06 --- /dev/null +++ b/docs/en/protocols/Browser-Protocol.md @@ -0,0 +1,18 @@ +# Browser Protocol + +Browser protocol describes the data format between [skywalking-client-js](https://github.com/apache/skywalking-client-js) and backend. + +## Overview + +Browser protocol is defined and provided in [gRPC format](https://github.com/apache/skywalking-data-collect-protocol/blob/master/browser/BrowserPerf.proto). + +### Send performance data and error log + +You can send performance data and error logs via the following services: + +1. `BrowserPerfService#collectPerfData` for performance data format. +1. `BrowserPerfService#collectErrorLogs` for error log format. + +For error log format, there are some notices + +1. `BrowserErrorLog#uniqueId` should be unique in the whole distributed environments. diff --git a/docs/en/protocols/README.md b/docs/en/protocols/README.md index f94aa5643..1b8a65570 100644 --- a/docs/en/protocols/README.md +++ b/docs/en/protocols/README.md @@ -29,6 +29,12 @@ SkyWalking javaagent begins to support this since 8.0.0. [SkyWalking Trace Data Protocol v3](Trace-Data-Protocol-v3.md) defines the communication way and format between agent and backend. +### Browser probe protocol + +The browser probe, such as [skywalking-client-js](https://github.com/apache/skywalking-client-js) could use this protocol to send to backend. This service provided by gRPC. + +[SkyWalking Browser Protocol](Browser-Protocol.md) define the communication way and format between `skywalking-client-js` and backend. + ### Service Mesh probe protocol The probe in sidecar or proxy could use this protocol to send data to backendEnd. This service provided by gRPC, requires the following key info: diff --git a/docs/en/setup/backend/backend-receivers.md b/docs/en/setup/backend/backend-receivers.md index 8db39ac8d..88aa26dfd 100644 --- a/docs/en/setup/backend/backend-receivers.md +++ b/docs/en/setup/backend/backend-receivers.md @@ -16,6 +16,7 @@ We have following receivers, and `default` implementors are provided in our Apac 1. **receiver_jaeger**. See [details](#jaeger-receiver). 1. **receiver-oc**. See [details](#opencensus-receiver). 1. **receiver-meter**. See [details](backend-meter.md). +1. **receiver-browser**. gRPC services to accept browser performance data and error log. The sample settings of these receivers should be already in default `application.yml`, and also list here ```yaml @@ -60,6 +61,10 @@ receiver_zipkin: receiver-profile: selector: ${SW_RECEIVER_PROFILE:default} default: + +receiver-browser: + selector: ${SW_RECEIVER_BROWSER:default} + default: ``` ## gRPC/HTTP server for receiver diff --git a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/jvm/JVMSourceDispatcher.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/jvm/JVMSourceDispatcher.java index dd059467d..0db68dce2 100644 --- a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/jvm/JVMSourceDispatcher.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/jvm/JVMSourceDispatcher.java @@ -19,6 +19,7 @@ package org.apache.skywalking.oap.server.analyzer.provider.jvm; import java.util.List; +import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.apm.network.common.v3.CPU; import org.apache.skywalking.apm.network.language.agent.v3.GC; import org.apache.skywalking.apm.network.language.agent.v3.JVMMetric; @@ -38,11 +39,9 @@ import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMMemoryPool import org.apache.skywalking.oap.server.core.source.ServiceInstanceJVMThread; import org.apache.skywalking.oap.server.core.source.SourceReceiver; import org.apache.skywalking.oap.server.library.module.ModuleManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +@Slf4j public class JVMSourceDispatcher { - private static final Logger LOGGER = LoggerFactory.getLogger(JVMSourceDispatcher.class); private final SourceReceiver sourceReceiver; public JVMSourceDispatcher(ModuleManager moduleManager) { diff --git a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 index aa598c248..316b5582c 100644 --- a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 +++ b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4 @@ -45,6 +45,14 @@ SRC_SERVICE_INSTANCE_CLR_GC: 'ServiceInstanceCLRGC'; SRC_SERVICE_INSTANCE_CLR_THREAD: 'ServiceInstanceCLRThread'; SRC_ENVOY_INSTANCE_METRIC: 'EnvoyInstanceMetric'; +// Browser keywords +SRC_BROWSER_APP_PERF: 'BrowserAppPerf'; +SRC_BROWSER_APP_PAGE_PERF: 'BrowserAppPagePerf'; +SRC_BROWSER_APP_SINGLE_VERSION_PERF: 'BrowserAppSingleVersionPerf'; +SRC_BROWSER_APP_TRAFFIC: 'BrowserAppTraffic'; +SRC_BROWSER_APP_PAGE_TRAFFIC: 'BrowserAppPageTraffic'; +SRC_BROWSER_APP_SINGLE_VERSION_TRAFFIC: 'BrowserAppSingleVersionTraffic'; + //hard code entities, only used when need to be deactived. //Disable is targeting @Stream#name SRC_SEGMENT: 'segment'; @@ -61,6 +69,8 @@ SRC_PROFILE_TASK: 'profile_task'; SRC_PROFILE_TASK_LOG: 'profile_task_log'; SRC_PROFILE_THREAD_SHANPSHOT: 'profile_task_segment_snapshot'; +SRC_BROWSER_ERROR_LOG: 'browser_error_log'; + // Constructors symbols DOT: '.'; diff --git a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 index cc0fd5085..0296571f5 100644 --- a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 +++ b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 @@ -54,7 +54,9 @@ source SRC_SERVICE_RELATION | SRC_SERVICE_INSTANCE_RELATION | SRC_ENDPOINT_RELATION | SRC_SERVICE_INSTANCE_JVM_CPU | SRC_SERVICE_INSTANCE_JVM_MEMORY | SRC_SERVICE_INSTANCE_JVM_MEMORY_POOL | SRC_SERVICE_INSTANCE_JVM_GC | SRC_SERVICE_INSTANCE_JVM_THREAD |// JVM source of service instance SRC_SERVICE_INSTANCE_CLR_CPU | SRC_SERVICE_INSTANCE_CLR_GC | SRC_SERVICE_INSTANCE_CLR_THREAD | - SRC_ENVOY_INSTANCE_METRIC + SRC_ENVOY_INSTANCE_METRIC | + SRC_BROWSER_APP_PERF | SRC_BROWSER_APP_PAGE_PERF | SRC_BROWSER_APP_SINGLE_VERSION_PERF | + SRC_BROWSER_APP_TRAFFIC | SRC_BROWSER_APP_PAGE_TRAFFIC | SRC_BROWSER_APP_SINGLE_VERSION_TRAFFIC ; disableSource @@ -72,7 +74,7 @@ variable ; aggregateFunction - : functionName LR_BRACKET (funcParamExpression | (literalExpression (COMMA literalExpression)?))? RR_BRACKET + : functionName LR_BRACKET ((funcParamExpression (COMMA funcParamExpression)?) | (literalExpression (COMMA literalExpression)?))? RR_BRACKET ; functionName diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/AnalysisResult.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/AnalysisResult.java index ad4f4e897..06ef512b1 100644 --- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/AnalysisResult.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/AnalysisResult.java @@ -57,6 +57,8 @@ public class AnalysisResult { private List funcConditionExpressions; + private int funcConditionExpressionGetIdx = 0; + private List funcArgs; private int argGetIdx = 0; @@ -82,6 +84,10 @@ public class AnalysisResult { funcConditionExpressions.add(conditionExpression); } + public ConditionExpression getNextFuncConditionExpression() { + return funcConditionExpressions.get(funcConditionExpressionGetIdx++); + } + public void addFilterExpressions(Expression filterExpression) { if (filterExpressions == null) { filterExpressions = new LinkedList<>(); diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/DeepAnalysis.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/DeepAnalysis.java index d1d065aef..e968c3300 100644 --- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/DeepAnalysis.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/DeepAnalysis.java @@ -31,6 +31,8 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.Entranc import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.SourceFrom; import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import static java.util.Objects.isNull; + public class DeepAnalysis { public AnalysisResult analysis(AnalysisResult result) { // 1. Set sub package name by source.metrics @@ -85,16 +87,20 @@ public class DeepAnalysis { Class parameterType = parameter.getType(); Annotation[] parameterAnnotations = parameter.getAnnotations(); if (parameterAnnotations == null || parameterAnnotations.length == 0) { - throw new IllegalArgumentException("Entrance method:" + entranceMethod + " doesn't include the annotation."); + throw new IllegalArgumentException( + "Entrance method:" + entranceMethod + " doesn't include the annotation."); } Annotation annotation = parameterAnnotations[0]; if (annotation instanceof SourceFrom) { - entryMethod.addArg(parameterType, "source." + ClassMethodUtil.toGetMethod(result.getSourceAttribute()) + "()"); + entryMethod.addArg( + parameterType, "source." + ClassMethodUtil.toGetMethod(result.getSourceAttribute()) + "()"); } else if (annotation instanceof ConstOne) { entryMethod.addArg(parameterType, "1"); } else if (annotation instanceof org.apache.skywalking.oap.server.core.analysis.metrics.annotation.Expression) { - if (result.getFuncConditionExpressions().size() == 1) { - final ConditionExpression expression = result.getFuncConditionExpressions().get(0); + if (isNull(result.getFuncConditionExpressions()) || result.getFuncConditionExpressions().isEmpty()) { + throw new IllegalArgumentException("Entrance method:" + entranceMethod + " argument can't find funcParamExpression."); + } else { + ConditionExpression expression = result.getNextFuncConditionExpression(); final FilterMatchers.MatcherInfo matcherInfo = FilterMatchers.INSTANCE.find(expression.getExpressionType()); final String getter = matcherInfo.isBooleanType() @@ -107,13 +113,12 @@ public class DeepAnalysis { argExpression.setLeft("source." + getter + "()"); entryMethod.addArg(argExpression); - } else { - throw new IllegalArgumentException("Entrance method:" + entranceMethod + " argument can't find funcParamExpression."); } } else if (annotation instanceof Arg) { entryMethod.addArg(parameterType, result.getNextFuncArg()); } else { - throw new IllegalArgumentException("Entrance method:" + entranceMethod + " doesn't the expected annotation."); + throw new IllegalArgumentException( + "Entrance method:" + entranceMethod + " doesn't the expected annotation."); } } diff --git a/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/ScriptParserTest.java b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/ScriptParserTest.java index fd91dd2d5..c92703852 100644 --- a/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/ScriptParserTest.java +++ b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/ScriptParserTest.java @@ -167,6 +167,29 @@ public class ScriptParserTest { Assert.assertEquals("lessEqualMatch", booleanMatchExp.getExpressionType()); } + @Test + public void testParse5() throws IOException { + ScriptParser parser = ScriptParser.createFromScriptText( + "service_response_s4_summary = from(Service.latency).rate(param1 == true,param2 == false);", + TEST_SOURCE_PACKAGE + ); + List results = parser.parse().getMetricsStmts(); + Assert.assertEquals(1, results.size()); + AnalysisResult result = results.get(0); + Assert.assertEquals("rate", result.getAggregationFunctionName()); + Assert.assertEquals(2, result.getFuncConditionExpressions().size()); + + ConditionExpression expression1 = result.getFuncConditionExpressions().get(0); + Assert.assertEquals("param1", expression1.getAttribute()); + Assert.assertEquals("booleanMatch", expression1.getExpressionType()); + Assert.assertEquals("true", expression1.getValue()); + + ConditionExpression expression2 = result.getFuncConditionExpressions().get(1); + Assert.assertEquals("param2", expression2.getAttribute()); + Assert.assertEquals("booleanMatch", expression2.getExpressionType()); + Assert.assertEquals("false", expression2.getValue()); + } + @Test public void testParse6() throws IOException { ScriptParser parser = ScriptParser.createFromScriptText( diff --git a/oap-server/server-bootstrap/pom.xml b/oap-server/server-bootstrap/pom.xml index 4e2b9821c..d3a369b0f 100644 --- a/oap-server/server-bootstrap/pom.xml +++ b/oap-server/server-bootstrap/pom.xml @@ -126,6 +126,11 @@ skywalking-meter-receiver-plugin ${project.version} + + org.apache.skywalking + skywalking-browser-receiver-plugin + ${project.version} + @@ -264,4 +269,4 @@ - + \ No newline at end of file diff --git a/oap-server/server-bootstrap/src/main/resources/application.yml b/oap-server/server-bootstrap/src/main/resources/application.yml index a74ee817c..f35e3a2e3 100755 --- a/oap-server/server-bootstrap/src/main/resources/application.yml +++ b/oap-server/server-bootstrap/src/main/resources/application.yml @@ -276,6 +276,10 @@ receiver_jaeger: gRPCHost: ${SW_RECEIVER_JAEGER_HOST:0.0.0.0} gRPCPort: ${SW_RECEIVER_JAEGER_PORT:14250} +receiver-browser: + selector: ${SW_RECEIVER_BROWSER:default} + default: + query: selector: ${SW_QUERY:graphql} graphql: diff --git a/oap-server/server-bootstrap/src/main/resources/oal/browser.oal b/oap-server/server-bootstrap/src/main/resources/oal/browser.oal new file mode 100644 index 000000000..a7a4458a8 --- /dev/null +++ b/oap-server/server-bootstrap/src/main/resources/oal/browser.oal @@ -0,0 +1,63 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ +// browser app +browser_app_pv = from(BrowserAppTraffic.count).filter(trafficCategory == BrowserAppTrafficCategory.NORMAL).sum(); +browser_app_error_rate = from(BrowserAppTraffic.*).rate(trafficCategory == BrowserAppTrafficCategory.FIRST_ERROR,trafficCategory == BrowserAppTrafficCategory.NORMAL); +browser_app_error_sum = from(BrowserAppTraffic.count).filter(trafficCategory != BrowserAppTrafficCategory.NORMAL).sum(); + +// browser app single version +browser_app_single_version_pv = from(BrowserAppSingleVersionTraffic.count).filter(trafficCategory == BrowserAppTrafficCategory.NORMAL).sum(); +browser_app_single_version_error_rate = from(BrowserAppSingleVersionTraffic.trafficCategory).rate(trafficCategory == BrowserAppTrafficCategory.FIRST_ERROR,trafficCategory == BrowserAppTrafficCategory.NORMAL); +browser_app_single_version_error_sum = from(BrowserAppSingleVersionTraffic.count).filter(trafficCategory != BrowserAppTrafficCategory.NORMAL).sum(); + +// browser app page +browser_app_page_pv = from(BrowserAppPageTraffic.count).filter(trafficCategory == BrowserAppTrafficCategory.NORMAL).sum(); +browser_app_page_error_rate = from(BrowserAppPageTraffic.*).rate(trafficCategory == BrowserAppTrafficCategory.FIRST_ERROR,trafficCategory == BrowserAppTrafficCategory.NORMAL); +browser_app_page_error_sum = from(BrowserAppPageTraffic.count).filter(trafficCategory != BrowserAppTrafficCategory.NORMAL).sum(); + +browser_app_page_ajax_error_sum = from(BrowserAppPageTraffic.count).filter(trafficCategory != BrowserAppTrafficCategory.NORMAL).filter(errorCategory == BrowserErrorCategory.AJAX).sum(); +browser_app_page_resource_error_sum = from(BrowserAppPageTraffic.count).filter(trafficCategory != BrowserAppTrafficCategory.NORMAL).filter(errorCategory == BrowserErrorCategory.RESOURCE).sum(); +browser_app_page_js_error_sum = from(BrowserAppPageTraffic.count).filter(trafficCategory != BrowserAppTrafficCategory.NORMAL).filter(errorCategory in [BrowserErrorCategory.JS,BrowserErrorCategory.VUE,BrowserErrorCategory.PROMISE]).sum(); +browser_app_page_unknown_error_sum = from(BrowserAppPageTraffic.count).filter(trafficCategory != BrowserAppTrafficCategory.NORMAL).filter(errorCategory == BrowserErrorCategory.UNKNOWN).sum(); + +// browser performance metrics +browser_app_page_redirect_avg = from(BrowserAppPagePerf.redirectTime).longAvg(); +browser_app_page_dns_avg = from(BrowserAppPagePerf.dnsTime).longAvg(); +browser_app_page_ttfb_avg = from(BrowserAppPagePerf.ttfbTime).longAvg(); +browser_app_page_tcp_avg = from(BrowserAppPagePerf.tcpTime).longAvg(); +browser_app_page_trans_avg = from(BrowserAppPagePerf.transTime).longAvg(); +browser_app_page_dom_analysis_avg = from(BrowserAppPagePerf.domAnalysisTime).longAvg(); +browser_app_page_fpt_avg = from(BrowserAppPagePerf.fptTime).longAvg(); +browser_app_page_dom_ready_avg = from(BrowserAppPagePerf.domReadyTime).longAvg(); +browser_app_page_load_page_avg = from(BrowserAppPagePerf.loadPageTime).longAvg(); +browser_app_page_res_avg = from(BrowserAppPagePerf.resTime).longAvg(); +browser_app_page_ssl_avg = from(BrowserAppPagePerf.sslTime).longAvg(); +browser_app_page_ttl_avg = from(BrowserAppPagePerf.ttlTime).longAvg(); +browser_app_page_first_pack_avg = from(BrowserAppPagePerf.firstPackTime).longAvg(); +browser_app_page_fmp_avg = from(BrowserAppPagePerf.fmpTime).longAvg(); + +browser_app_page_fpt_percentile = from(BrowserAppPagePerf.fptTime).percentile(10); +browser_app_page_ttl_percentile = from(BrowserAppPagePerf.ttlTime).percentile(10); +browser_app_page_dom_ready_percentile = from(BrowserAppPagePerf.domReadyTime).percentile(10); +browser_app_page_load_page_percentile = from(BrowserAppPagePerf.loadPageTime).percentile(10); +browser_app_page_first_pack_percentile = from(BrowserAppPagePerf.firstPackTime).percentile(10); +browser_app_page_fmp_percentile = from(BrowserAppPagePerf.fmpTime).percentile(10); + +// Disable unnecessary hard core stream, targeting @Stream#name +///////// +//disable(browser_error_log); \ No newline at end of file diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java index e796aca8d..358bc28de 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java @@ -33,6 +33,7 @@ import org.apache.skywalking.oap.server.core.oal.rt.OALEngineLoaderService; import org.apache.skywalking.oap.server.core.profile.ProfileTaskMutationService; import org.apache.skywalking.oap.server.core.query.AggregationQueryService; import org.apache.skywalking.oap.server.core.query.AlarmQueryService; +import org.apache.skywalking.oap.server.core.query.BrowserLogQueryService; import org.apache.skywalking.oap.server.core.query.LogQueryService; import org.apache.skywalking.oap.server.core.query.MetadataQueryService; import org.apache.skywalking.oap.server.core.query.MetricsMetadataQueryService; @@ -114,6 +115,7 @@ public class CoreModule extends ModuleDefine { classes.add(AggregationQueryService.class); classes.add(AlarmQueryService.class); classes.add(TopNRecordsQueryService.class); + classes.add(BrowserLogQueryService.class); } private void addServerInterface(List classes) { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java index 80521caad..b5f9bec0c 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java @@ -52,6 +52,7 @@ import org.apache.skywalking.oap.server.core.oal.rt.OALEngineLoaderService; import org.apache.skywalking.oap.server.core.profile.ProfileTaskMutationService; import org.apache.skywalking.oap.server.core.query.AggregationQueryService; import org.apache.skywalking.oap.server.core.query.AlarmQueryService; +import org.apache.skywalking.oap.server.core.query.BrowserLogQueryService; import org.apache.skywalking.oap.server.core.query.LogQueryService; import org.apache.skywalking.oap.server.core.query.MetadataQueryService; import org.apache.skywalking.oap.server.core.query.MetricsMetadataQueryService; @@ -242,6 +243,7 @@ public class CoreModuleProvider extends ModuleProvider { this.registerServiceImplementation(MetricsMetadataQueryService.class, new MetricsMetadataQueryService()); this.registerServiceImplementation(MetricsQueryService.class, new MetricsQueryService(getManager())); this.registerServiceImplementation(TraceQueryService.class, new TraceQueryService(getManager())); + this.registerServiceImplementation(BrowserLogQueryService.class, new BrowserLogQueryService(getManager())); this.registerServiceImplementation(LogQueryService.class, new LogQueryService(getManager())); this.registerServiceImplementation(MetadataQueryService.class, new MetadataQueryService(getManager())); this.registerServiceImplementation(AggregationQueryService.class, new AggregationQueryService(getManager())); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/DispatcherManager.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/DispatcherManager.java index 059d3b04a..9f3f7ee58 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/DispatcherManager.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/DispatcherManager.java @@ -21,6 +21,7 @@ package org.apache.skywalking.oap.server.core.analysis; import com.google.common.collect.ImmutableSet; import com.google.common.reflect.ClassPath; import java.io.IOException; +import java.lang.reflect.Modifier; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; import java.util.ArrayList; @@ -80,7 +81,8 @@ public class DispatcherManager implements DispatcherDetectorListener { @Override public void addIfAsSourceDispatcher(Class aClass) throws IllegalAccessException, InstantiationException { - if (!aClass.isInterface() && SourceDispatcher.class.isAssignableFrom(aClass)) { + if (!aClass.isInterface() && !Modifier.isAbstract( + aClass.getModifiers()) && SourceDispatcher.class.isAssignableFrom(aClass)) { Type[] genericInterfaces = aClass.getGenericInterfaces(); for (Type genericInterface : genericInterfaces) { ParameterizedType anInterface = (ParameterizedType) genericInterface; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/NodeType.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/NodeType.java index 5c409222a..30fe61d97 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/NodeType.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/NodeType.java @@ -89,6 +89,8 @@ public enum NodeType { return MQ; case 5: return Cache; + case 6: + return Browser; case 10: return User; case 11: diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/RateMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/RateMetrics.java new file mode 100644 index 000000000..d4cedb692 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/RateMetrics.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.analysis.metrics; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.Entrance; +import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.Expression; +import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.MetricsFunction; +import org.apache.skywalking.oap.server.core.query.sql.Function; +import org.apache.skywalking.oap.server.core.storage.annotation.Column; + +@MetricsFunction(functionName = "rate") +public abstract class RateMetrics extends Metrics implements IntValueHolder { + protected static final String DENOMINATOR = "denominator"; + protected static final String NUMERATOR = "numerator"; + protected static final String PERCENTAGE = "percentage"; + + @Getter + @Setter + @Column(columnName = DENOMINATOR) + private long denominator; + @Getter + @Setter + @Column(columnName = PERCENTAGE, dataType = Column.ValueDataType.COMMON_VALUE, function = Function.Avg) + private int percentage; + @Getter + @Setter + @Column(columnName = NUMERATOR) + private long numerator; + + @Entrance + public final void combine(@Expression boolean isNumerator, @Expression boolean isDenominator) { + if (isNumerator) { + numerator++; + } + if (isDenominator) { + denominator++; + } + } + + @Override + public final void combine(Metrics metrics) { + denominator += ((RateMetrics) metrics).denominator; + numerator += ((RateMetrics) metrics).numerator; + } + + @Override + public void calculate() { + percentage = (int) (numerator * 10000 / denominator); + } + + @Override + public int getValue() { + return percentage; + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/InMatch.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/InMatch.java index 2873f3d27..897bd000e 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/InMatch.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/expression/InMatch.java @@ -22,7 +22,6 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.FilterM @FilterMatcher public class InMatch { - public boolean match(Object left, Object[] rights) { for (Object right : rights) { if (right instanceof String) { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/BrowserAppTrafficSourceDispatcher.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/BrowserAppTrafficSourceDispatcher.java new file mode 100644 index 000000000..9c3521889 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/BrowserAppTrafficSourceDispatcher.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.browser.manual; + +import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher; +import org.apache.skywalking.oap.server.core.browser.source.BrowserAppTrafficCategory; +import org.apache.skywalking.oap.server.core.browser.source.BrowserAppTrafficSource; + +public abstract class BrowserAppTrafficSourceDispatcher implements SourceDispatcher { + @Override + public void dispatch(final SOURCE source) { + // filter error traffic + if (!source.getTrafficCategory().equals(BrowserAppTrafficCategory.NORMAL)) { + return; + } + dispatchInterval(source); + } + + protected abstract void dispatchInterval(SOURCE source); +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/endpoint/BrowserAppPageTrafficDispatcher.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/endpoint/BrowserAppPageTrafficDispatcher.java new file mode 100644 index 000000000..8df13e6f1 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/endpoint/BrowserAppPageTrafficDispatcher.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.browser.manual.endpoint; + +import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher; +import org.apache.skywalking.oap.server.core.analysis.manual.endpoint.EndpointTraffic; +import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor; +import org.apache.skywalking.oap.server.core.browser.manual.BrowserAppTrafficSourceDispatcher; +import org.apache.skywalking.oap.server.core.browser.source.BrowserAppPageTraffic; + +public class BrowserAppPageTrafficDispatcher extends BrowserAppTrafficSourceDispatcher implements SourceDispatcher { + @Override + protected void dispatchInterval(final BrowserAppPageTraffic source) { + EndpointTraffic traffic = new EndpointTraffic(); + traffic.setTimeBucket(source.getTimeBucket()); + traffic.setName(source.getName()); + traffic.setServiceId(source.getServiceId()); + MetricsStreamProcessor.getInstance().in(traffic); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java new file mode 100644 index 000000000..1515a569d --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.browser.manual.errorlog; + +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.apm.util.StringUtil; +import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.core.analysis.Stream; +import org.apache.skywalking.oap.server.core.analysis.record.Record; +import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; +import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.StorageBuilder; +import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; + +@Stream(name = BrowserErrorLogRecord.INDEX_NAME, scopeId = DefaultScopeDefine.BROWSER_ERROR_LOG, builder = BrowserErrorLogRecord.Builder.class, processor = RecordStreamProcessor.class) +public class BrowserErrorLogRecord extends Record { + public static final String INDEX_NAME = "browser_error_log"; + public static final String UNIQUE_ID = "unique_id"; + public static final String SERVICE_ID = "service_id"; + public static final String SERVICE_VERSION_ID = "service_version_id"; + public static final String PAGE_PATH_ID = "pate_path_id"; + public static final String PAGE_PATH = "page_path"; + public static final String TIMESTAMP = "timestamp"; + public static final String ERROR_CATEGORY = "error_category"; + public static final String DATA_BINARY = "data_binary"; + + @Override + public String id() { + return uniqueId; + } + + @Setter + @Getter + @Column(columnName = UNIQUE_ID) + private String uniqueId; + + @Setter + @Getter + @Column(columnName = SERVICE_ID) + private String serviceId; + + @Setter + @Getter + @Column(columnName = SERVICE_VERSION_ID) + private String serviceVersionId; + + @Setter + @Getter + @Column(columnName = PAGE_PATH_ID) + private String pagePathId; + + @Setter + @Getter + @Column(columnName = PAGE_PATH, matchQuery = true) + private String pagePath; + + @Setter + @Getter + @Column(columnName = TIMESTAMP) + private long timestamp; + + @Setter + @Getter + @Column(columnName = ERROR_CATEGORY) + private int errorCategory; + + @Setter + @Getter + @Column(columnName = DATA_BINARY) + private byte[] dataBinary; + + public static class Builder implements StorageBuilder { + @Override + public BrowserErrorLogRecord map2Data(final Map dbMap) { + BrowserErrorLogRecord record = new BrowserErrorLogRecord(); + record.setUniqueId((String) dbMap.get(UNIQUE_ID)); + record.setServiceId((String) dbMap.get(SERVICE_ID)); + record.setServiceVersionId((String) dbMap.get(SERVICE_VERSION_ID)); + record.setPagePathId((String) dbMap.get(PAGE_PATH_ID)); + record.setPagePath((String) dbMap.get(PAGE_PATH)); + record.setTimestamp(((Number) dbMap.get(TIMESTAMP)).longValue()); + record.setTimeBucket(((Number) dbMap.get(TIME_BUCKET)).longValue()); + record.setErrorCategory(((Number) dbMap.get(ERROR_CATEGORY)).intValue()); + String dataBinary = (String) dbMap.get(DATA_BINARY); + if (StringUtil.isEmpty(dataBinary)) { + record.setDataBinary(new byte[] {}); + } else { + record.setDataBinary(Base64.getDecoder().decode(dataBinary)); + } + return record; + } + + @Override + public Map data2Map(final BrowserErrorLogRecord storageData) { + Map map = new HashMap<>(); + map.put(UNIQUE_ID, storageData.getUniqueId()); + map.put(SERVICE_ID, storageData.getServiceId()); + map.put(SERVICE_VERSION_ID, storageData.getServiceVersionId()); + map.put(PAGE_PATH_ID, storageData.getPagePathId()); + map.put(PAGE_PATH, storageData.getPagePath()); + map.put(TIMESTAMP, storageData.getTimestamp()); + map.put(TIME_BUCKET, storageData.getTimeBucket()); + map.put(ERROR_CATEGORY, storageData.getErrorCategory()); + if (CollectionUtils.isEmpty(storageData.getDataBinary())) { + map.put(DATA_BINARY, Const.EMPTY_STRING); + } else { + map.put(DATA_BINARY, new String(Base64.getEncoder().encode(storageData.getDataBinary()))); + } + return map; + } + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecordDispatcher.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecordDispatcher.java new file mode 100644 index 000000000..e2a2630cf --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecordDispatcher.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.browser.manual.errorlog; + +import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher; +import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; +import org.apache.skywalking.oap.server.core.browser.source.BrowserErrorLog; + +public class BrowserErrorLogRecordDispatcher implements SourceDispatcher { + @Override + public void dispatch(final BrowserErrorLog source) { + BrowserErrorLogRecord record = new BrowserErrorLogRecord(); + record.setUniqueId(source.getUniqueId()); + record.setServiceId(source.getServiceId()); + record.setServiceVersionId(source.getServiceVersionId()); + record.setPagePathId(source.getPagePathId()); + record.setPagePath(source.getPagePath()); + record.setTimestamp(source.getTimestamp()); + record.setTimeBucket(source.getTimeBucket()); + record.setErrorCategory(source.getErrorCategory().getValue()); + record.setDataBinary(source.getDataBinary()); + RecordStreamProcessor.getInstance().in(record); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/instance/BrowserAppSingleVersionTrafficDispatcher.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/instance/BrowserAppSingleVersionTrafficDispatcher.java new file mode 100644 index 000000000..7ea9b3df8 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/instance/BrowserAppSingleVersionTrafficDispatcher.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.browser.manual.instance; + +import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher; +import org.apache.skywalking.oap.server.core.analysis.manual.instance.InstanceTraffic; +import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor; +import org.apache.skywalking.oap.server.core.browser.manual.BrowserAppTrafficSourceDispatcher; +import org.apache.skywalking.oap.server.core.browser.source.BrowserAppSingleVersionTraffic; + +public class BrowserAppSingleVersionTrafficDispatcher extends BrowserAppTrafficSourceDispatcher implements SourceDispatcher { + @Override + protected void dispatchInterval(final BrowserAppSingleVersionTraffic source) { + InstanceTraffic traffic = new InstanceTraffic(); + traffic.setTimeBucket(source.getTimeBucket()); + traffic.setName(source.getName()); + traffic.setServiceId(source.getServiceId()); + traffic.setLastPingTimestamp(source.getTimeBucket()); + MetricsStreamProcessor.getInstance().in(traffic); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/service/BrowserAppTrafficDispatcher.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/service/BrowserAppTrafficDispatcher.java new file mode 100644 index 000000000..f71087201 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/service/BrowserAppTrafficDispatcher.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.browser.manual.service; + +import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher; +import org.apache.skywalking.oap.server.core.analysis.manual.service.ServiceTraffic; +import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor; +import org.apache.skywalking.oap.server.core.browser.manual.BrowserAppTrafficSourceDispatcher; +import org.apache.skywalking.oap.server.core.browser.source.BrowserAppTraffic; + +public class BrowserAppTrafficDispatcher extends BrowserAppTrafficSourceDispatcher implements SourceDispatcher { + @Override + protected void dispatchInterval(final BrowserAppTraffic source) { + ServiceTraffic traffic = new ServiceTraffic(); + traffic.setTimeBucket(source.getTimeBucket()); + traffic.setName(source.getName()); + traffic.setNodeType(source.getNodeType()); + MetricsStreamProcessor.getInstance().in(traffic); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppPagePerf.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppPagePerf.java new file mode 100644 index 000000000..3a1f95c3b --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppPagePerf.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.browser.source; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.source.ScopeDeclaration; +import org.apache.skywalking.oap.server.core.source.ScopeDefaultColumn; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.BROWSER_APP_PAGE_PERF; +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.ENDPOINT_CATALOG_NAME; + +@ScopeDeclaration(id = BROWSER_APP_PAGE_PERF, name = "BrowserAppPagePerf", catalog = ENDPOINT_CATALOG_NAME) +@ScopeDefaultColumn.VirtualColumnDefinition(fieldName = "entityId", columnName = "entity_id", isID = true, type = String.class) +public class BrowserAppPagePerf extends BrowserAppPerfSource { + @Override + public int scope() { + return BROWSER_APP_PAGE_PERF; + } + + @Override + public String getEntityId() { + return IDManager.EndpointID.buildId(serviceId, name); + } + + @Getter + @ScopeDefaultColumn.DefinedByField(columnName = "service_id") + private String serviceId; + @Getter + @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true) + private String serviceName; + + @Override + public void prepare() { + serviceId = IDManager.ServiceID.buildId(serviceName, nodeType); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppPageTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppPageTraffic.java new file mode 100644 index 000000000..b040e2b4e --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppPageTraffic.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.browser.source; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.source.ScopeDeclaration; +import org.apache.skywalking.oap.server.core.source.ScopeDefaultColumn; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.BROWSER_APP_PAGE_TRAFFIC; +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.ENDPOINT_CATALOG_NAME; + +@ScopeDeclaration(id = BROWSER_APP_PAGE_TRAFFIC, name = "BrowserAppPageTraffic", catalog = ENDPOINT_CATALOG_NAME) +@ScopeDefaultColumn.VirtualColumnDefinition(fieldName = "entityId", columnName = "entity_id", isID = true, type = String.class) +public class BrowserAppPageTraffic extends BrowserAppTrafficSource { + @Override + public int scope() { + return BROWSER_APP_PAGE_TRAFFIC; + } + + @Override + public String getEntityId() { + return IDManager.EndpointID.buildId(serviceId, name); + } + + @Getter + @ScopeDefaultColumn.DefinedByField(columnName = "service_id") + private String serviceId; + @Getter + @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true) + private String serviceName; + + @Override + public void prepare() { + serviceId = IDManager.ServiceID.buildId(serviceName, nodeType); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppPerf.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppPerf.java new file mode 100644 index 000000000..12730ad7d --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppPerf.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.browser.source; + +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.source.ScopeDeclaration; +import org.apache.skywalking.oap.server.core.source.ScopeDefaultColumn; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.BROWSER_APP_PERF; +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_CATALOG_NAME; + +@ScopeDeclaration(id = BROWSER_APP_PERF, name = "BrowserAppPerf", catalog = SERVICE_CATALOG_NAME) +@ScopeDefaultColumn.VirtualColumnDefinition(fieldName = "entityId", columnName = "entity_id", isID = true, type = String.class) +public class BrowserAppPerf extends BrowserAppPerfSource { + @Override + public int scope() { + return BROWSER_APP_PERF; + } + + @Override + public String getEntityId() { + return IDManager.ServiceID.buildId(name, nodeType); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppPerfSource.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppPerfSource.java new file mode 100644 index 000000000..c3bbe65f1 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppPerfSource.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.browser.source; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.analysis.NodeType; +import org.apache.skywalking.oap.server.core.source.Source; + +/** + * Browser performance details + */ +@Setter +@Getter +public abstract class BrowserAppPerfSource extends Source { + protected String name; + protected final NodeType nodeType = NodeType.Browser; + private int redirectTime; + private int dnsTime; + private int ttfbTime; + private int tcpTime; + private int transTime; + private int domAnalysisTime; + private int fptTime; + private int domReadyTime; + private int loadPageTime; + private int resTime; + private int sslTime; + private int ttlTime; + private int firstPackTime; + private int fmpTime; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppSingleVersionPerf.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppSingleVersionPerf.java new file mode 100644 index 000000000..bc869991d --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppSingleVersionPerf.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.browser.source; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.source.ScopeDeclaration; +import org.apache.skywalking.oap.server.core.source.ScopeDefaultColumn; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.BROWSER_APP_SINGLE_VERSION_PERF; +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_INSTANCE_CATALOG_NAME; + +@ScopeDeclaration(id = BROWSER_APP_SINGLE_VERSION_PERF, name = "BrowserAppSingleVersionPerf", catalog = SERVICE_INSTANCE_CATALOG_NAME) +@ScopeDefaultColumn.VirtualColumnDefinition(fieldName = "entityId", columnName = "entity_id", isID = true, type = String.class) +public class BrowserAppSingleVersionPerf extends BrowserAppPerfSource { + @Override + public int scope() { + return BROWSER_APP_SINGLE_VERSION_PERF; + } + + @Override + public String getEntityId() { + return IDManager.ServiceInstanceID.buildId(serviceId, name); + } + + @Getter + @ScopeDefaultColumn.DefinedByField(columnName = "service_id") + private String serviceId; + @Getter + @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true) + private String serviceName; + + @Override + public void prepare() { + serviceId = IDManager.ServiceID.buildId(serviceName, nodeType); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppSingleVersionTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppSingleVersionTraffic.java new file mode 100644 index 000000000..9adce36d9 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppSingleVersionTraffic.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.browser.source; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.source.ScopeDeclaration; +import org.apache.skywalking.oap.server.core.source.ScopeDefaultColumn; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.BROWSER_APP_SINGLE_VERSION_TRAFFIC; +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_INSTANCE_CATALOG_NAME; + +@ScopeDeclaration(id = BROWSER_APP_SINGLE_VERSION_TRAFFIC, name = "BrowserAppSingleVersionTraffic", catalog = SERVICE_INSTANCE_CATALOG_NAME) +@ScopeDefaultColumn.VirtualColumnDefinition(fieldName = "entityId", columnName = "entity_id", isID = true, type = String.class) +public class BrowserAppSingleVersionTraffic extends BrowserAppTrafficSource { + @Override + public int scope() { + return BROWSER_APP_SINGLE_VERSION_TRAFFIC; + } + + @Override + public String getEntityId() { + return IDManager.ServiceInstanceID.buildId(serviceId, name); + } + + @Getter + @ScopeDefaultColumn.DefinedByField(columnName = "service_id") + private String serviceId; + @Getter + @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true) + private String serviceName; + + @Override + public void prepare() { + serviceId = IDManager.ServiceID.buildId(serviceName, nodeType); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppTraffic.java new file mode 100644 index 000000000..aa0cb848b --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppTraffic.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.browser.source; + +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.source.ScopeDeclaration; +import org.apache.skywalking.oap.server.core.source.ScopeDefaultColumn; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.BROWSER_APP_TRAFFIC; +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_CATALOG_NAME; + +@ScopeDeclaration(id = BROWSER_APP_TRAFFIC, name = "BrowserAppTraffic", catalog = SERVICE_CATALOG_NAME) +@ScopeDefaultColumn.VirtualColumnDefinition(fieldName = "entityId", columnName = "entity_id", isID = true, type = String.class) +public class BrowserAppTraffic extends BrowserAppTrafficSource { + @Override + public int scope() { + return BROWSER_APP_TRAFFIC; + } + + @Override + public String getEntityId() { + return IDManager.ServiceID.buildId(name, nodeType); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppTrafficCategory.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppTrafficCategory.java new file mode 100644 index 000000000..0918da6a6 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppTrafficCategory.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.browser.source; + +public enum BrowserAppTrafficCategory { + /** + * From the BrowserPerfData. + */ + NORMAL, + /** + * From the BrowserErrorLog and BrowserErrorLog#firstReportedError = true. + */ + FIRST_ERROR, + /** + * From the BrowserErrorLog and BrowserErrorLog#firstReportedError = false. + */ + ERROR +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppTrafficSource.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppTrafficSource.java new file mode 100644 index 000000000..d01b6dffa --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserAppTrafficSource.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.browser.source; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.analysis.NodeType; +import org.apache.skywalking.oap.server.core.source.ScopeDefaultColumn; +import org.apache.skywalking.oap.server.core.source.Source; + +/** + * From js client reported access traffic + */ +@Setter +@Getter +public abstract class BrowserAppTrafficSource extends Source { + @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true) + protected String name; + + protected final NodeType nodeType = NodeType.Browser; + + private final int count = 1; + + private BrowserAppTrafficCategory trafficCategory; + + /** + * if {@link #trafficCategory} is {@link BrowserAppTrafficCategory#NORMAL}, errorCategory is null. + */ + private BrowserErrorCategory errorCategory; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserErrorCategory.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserErrorCategory.java new file mode 100644 index 000000000..442d96bf7 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserErrorCategory.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.browser.source; + +import lombok.Getter; +import org.apache.skywalking.apm.network.language.agent.v3.ErrorCategory; + +public enum BrowserErrorCategory { + AJAX(0), RESOURCE(1), VUE(2), PROMISE(3), JS(4), UNKNOWN(5); + + public static BrowserErrorCategory fromErrorCategory(ErrorCategory category) { + switch (category) { + case ajax: + return BrowserErrorCategory.AJAX; + case resource: + return BrowserErrorCategory.RESOURCE; + case vue: + return BrowserErrorCategory.VUE; + case promise: + return BrowserErrorCategory.PROMISE; + case js: + return BrowserErrorCategory.JS; + default: + return BrowserErrorCategory.UNKNOWN; + } + } + + @Getter + private final int value; + + BrowserErrorCategory(int value) { + this.value = value; + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserErrorLog.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserErrorLog.java new file mode 100644 index 000000000..b8a723f8b --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/source/BrowserErrorLog.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.browser.source; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.source.ScopeDeclaration; +import org.apache.skywalking.oap.server.core.source.Source; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.BROWSER_ERROR_LOG; + +/** + * Browser error log raw data + */ +@ScopeDeclaration(id = BROWSER_ERROR_LOG, name = "BrowserErrorLog") +public class BrowserErrorLog extends Source { + @Override + public int scope() { + return BROWSER_ERROR_LOG; + } + + @Override + public String getEntityId() { + return uniqueId; + } + + @Getter + @Setter + private String uniqueId; + @Getter + @Setter + private String serviceId; + @Getter + @Setter + private String serviceVersionId; + @Getter + @Setter + private String pagePathId; + @Getter + @Setter + private String pagePath; + @Getter + @Setter + private long timestamp; + @Getter + @Setter + private BrowserErrorCategory errorCategory; + @Getter + @Setter + private byte[] dataBinary; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/BrowserLogQueryService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/BrowserLogQueryService.java new file mode 100644 index 000000000..fcbe3bbd2 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/BrowserLogQueryService.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.query; + +import java.io.IOException; +import java.util.Optional; +import lombok.RequiredArgsConstructor; +import org.apache.skywalking.oap.server.core.browser.source.BrowserErrorCategory; +import org.apache.skywalking.oap.server.core.query.type.BrowserErrorLogs; +import org.apache.skywalking.oap.server.core.query.type.ErrorCategory; +import org.apache.skywalking.oap.server.core.query.type.Pagination; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.query.IBrowserLogQueryDAO; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.Service; + +@RequiredArgsConstructor +public class BrowserLogQueryService implements Service { + private final ModuleManager moduleManager; + private IBrowserLogQueryDAO browserLogQueryDAO; + + private IBrowserLogQueryDAO getBrowserLogQueryDAO() { + return Optional.ofNullable(browserLogQueryDAO).orElseGet(() -> { + browserLogQueryDAO = moduleManager.find(StorageModule.NAME) + .provider() + .getService(IBrowserLogQueryDAO.class); + return browserLogQueryDAO; + }); + } + + public BrowserErrorLogs queryBrowserErrorLogs(final String serviceId, + final String serviceVersionId, + final String pagePathId, + final String pagePath, + final ErrorCategory category, + final long startSecondTB, + final long endSecondTB, + final Pagination paging) throws IOException { + PaginationUtils.Page page = PaginationUtils.INSTANCE.exchange(paging); + BrowserErrorCategory errorCategory = Optional.ofNullable(category) + .map(c -> BrowserErrorCategory.valueOf(c.name())) + .orElse(null); + + return getBrowserLogQueryDAO().queryBrowserErrorLogs( + serviceId, serviceVersionId, pagePathId, pagePath, errorCategory, startSecondTB, endSecondTB, + page.getLimit(), + page.getFrom() + ); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/BrowserErrorLogQueryCondition.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/BrowserErrorLogQueryCondition.java new file mode 100644 index 000000000..441538fdc --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/BrowserErrorLogQueryCondition.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.query.input; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.query.type.ErrorCategory; +import org.apache.skywalking.oap.server.core.query.type.Pagination; + +@Setter +@Getter +public class BrowserErrorLogQueryCondition { + private String serviceId; + private String serviceVersionId; + private String pagePathId; + private String pagePath; + private ErrorCategory category; + private Duration queryDuration; + private Pagination paging; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/BrowserErrorLog.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/BrowserErrorLog.java new file mode 100644 index 000000000..5e7353674 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/BrowserErrorLog.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.Getter; +import lombok.Setter; + +@Setter +@Getter +public class BrowserErrorLog { + private String service; + private String serviceVersion; + private Long time; + private String pagePath; + private ErrorCategory category; + private String grade; + private String message; + private Integer line; + private Integer col; + private String stack; + private String errorUrl; + private boolean firstReportedError; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/BrowserErrorLogs.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/BrowserErrorLogs.java new file mode 100644 index 000000000..810880873 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/BrowserErrorLogs.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import java.util.ArrayList; +import java.util.List; +import lombok.Getter; +import lombok.Setter; + +@Getter +public class BrowserErrorLogs { + private final List logs; + @Setter + private int total; + + public BrowserErrorLogs() { + this.logs = new ArrayList<>(); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/ErrorCategory.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/ErrorCategory.java new file mode 100644 index 000000000..7da58f82d --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/ErrorCategory.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.query.type; + +public enum ErrorCategory { + ALL, + AJAX, + RESOURCE, + VUE, + PROMISE, + JS, + UNKNOWN +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java index e753c629d..17bf1bb36 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java @@ -69,6 +69,15 @@ public class DefaultScopeDefine { public static final int UI_TEMPLATE = 32; public static final int SERVICE_INSTANCE_JVM_THREAD = 33; + // browser + public static final int BROWSER_ERROR_LOG = 34; + public static final int BROWSER_APP_PERF = 35; + public static final int BROWSER_APP_PAGE_PERF = 36; + public static final int BROWSER_APP_SINGLE_VERSION_PERF = 37; + public static final int BROWSER_APP_TRAFFIC = 38; + public static final int BROWSER_APP_SINGLE_VERSION_TRAFFIC = 39; + public static final int BROWSER_APP_PAGE_TRAFFIC = 40; + /** * Catalog of scope, the metrics processor could use this to group all generated metrics by oal rt. */ diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java index 1b5b9cd5c..4f72c0ca5 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java @@ -25,6 +25,7 @@ import org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskQueryDA import org.apache.skywalking.oap.server.core.storage.profile.IProfileThreadSnapshotQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IAggregationQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IAlarmQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IBrowserLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.ILogQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IMetadataQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IMetricsQueryDAO; @@ -63,7 +64,8 @@ public class StorageModule extends ModuleDefine { IProfileTaskQueryDAO.class, IProfileTaskLogQueryDAO.class, IProfileThreadSnapshotQueryDAO.class, - UITemplateManagementDAO.class + UITemplateManagementDAO.class, + IBrowserLogQueryDAO.class }; } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IBrowserLogQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IBrowserLogQueryDAO.java new file mode 100644 index 000000000..a96ec59a1 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IBrowserLogQueryDAO.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.core.storage.query; + +import com.google.protobuf.InvalidProtocolBufferException; +import java.io.IOException; +import java.util.Base64; +import org.apache.skywalking.oap.server.core.browser.source.BrowserErrorCategory; +import org.apache.skywalking.oap.server.core.query.type.BrowserErrorLog; +import org.apache.skywalking.oap.server.core.query.type.BrowserErrorLogs; +import org.apache.skywalking.oap.server.core.query.type.ErrorCategory; +import org.apache.skywalking.oap.server.library.module.Service; + +public interface IBrowserLogQueryDAO extends Service { + BrowserErrorLogs queryBrowserErrorLogs(String serviceId, + String serviceVersionId, + String pagePathId, + String pagePath, + BrowserErrorCategory category, + long startSecondTB, + long endSecondTB, + int limit, + int from) throws IOException; + + /** + * Parser the raw error log. + */ + default BrowserErrorLog parserDataBinary( + String dataBinaryBase64) { + try { + BrowserErrorLog log = new BrowserErrorLog(); + org.apache.skywalking.apm.network.language.agent.v3.BrowserErrorLog browserErrorLog = org.apache.skywalking.apm.network.language.agent.v3.BrowserErrorLog + .parseFrom(Base64.getDecoder().decode(dataBinaryBase64)); + + log.setService(browserErrorLog.getService()); + log.setServiceVersion(browserErrorLog.getServiceVersion()); + log.setTime(browserErrorLog.getTime()); + log.setPagePath(browserErrorLog.getPagePath()); + log.setCategory(ErrorCategory.valueOf(browserErrorLog.getCategory().name().toUpperCase())); + log.setGrade(browserErrorLog.getGrade()); + log.setMessage(browserErrorLog.getMessage()); + log.setLine(browserErrorLog.getLine()); + log.setCol(browserErrorLog.getCol()); + log.setStack(browserErrorLog.getStack()); + log.setErrorUrl(browserErrorLog.getErrorUrl()); + log.setFirstReportedError(browserErrorLog.getFirstReportedError()); + + return log; + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + } +} diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/CoreModuleTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/CoreModuleTest.java index ba3facc5a..747536490 100644 --- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/CoreModuleTest.java +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/CoreModuleTest.java @@ -26,6 +26,6 @@ public class CoreModuleTest { public void testOpenServiceList() { CoreModule coreModule = new CoreModule(); - Assert.assertEquals(31, coreModule.services().length); + Assert.assertEquals(32, coreModule.services().length); } } diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java index 145559db1..82c4d8cc3 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java @@ -23,6 +23,7 @@ import graphql.GraphQL; import graphql.schema.GraphQLSchema; import org.apache.skywalking.oap.query.graphql.resolver.AggregationQuery; import org.apache.skywalking.oap.query.graphql.resolver.AlarmQuery; +import org.apache.skywalking.oap.query.graphql.resolver.BrowserLogQuery; import org.apache.skywalking.oap.query.graphql.resolver.HealthQuery; import org.apache.skywalking.oap.query.graphql.resolver.LogQuery; import org.apache.skywalking.oap.query.graphql.resolver.MetadataQuery; @@ -72,48 +73,50 @@ public class GraphQLQueryProvider extends ModuleProvider { @Override public void prepare() throws ServiceNotProvidedException, ModuleStartException { GraphQLSchema schema = SchemaParser.newParser() - .file("query-protocol/common.graphqls") - .resolvers(new Query(), new Mutation(), new HealthQuery(getManager())) - .file("query-protocol/metadata.graphqls") - .resolvers(new MetadataQuery(getManager())) - .file("query-protocol/topology.graphqls") - .resolvers(new TopologyQuery(getManager())) - /** - * Metrics v2 query protocol is an alternative metrics query(s) of original v1, - * defined in the metric.graphql, top-n-records.graphqls, and aggregation.graphqls. - */ - .file("query-protocol/metrics-v2.graphqls") - .resolvers(new MetricsQuery(getManager())) - //////// - //Deprecated Queries - //////// - .file("query-protocol/metric.graphqls") - .resolvers(new MetricQuery(getManager())) - .file("query-protocol/aggregation.graphqls") - .resolvers(new AggregationQuery(getManager())) - .file("query-protocol/top-n-records.graphqls") - .resolvers(new TopNRecordsQuery(getManager())) - //////// - .file("query-protocol/trace.graphqls") - .resolvers(new TraceQuery(getManager())) - .file("query-protocol/alarm.graphqls") - .resolvers(new AlarmQuery(getManager())) - .file("query-protocol/log.graphqls") - .resolvers(new LogQuery(getManager())) - .file("query-protocol/profile.graphqls") - .resolvers(new ProfileQuery(getManager()), new ProfileMutation(getManager())) - .file("query-protocol/ui-configuration.graphqls") - .resolvers(new UIConfigurationManagement(getManager())) - .build() - .makeExecutableSchema(); + .file("query-protocol/common.graphqls") + .resolvers(new Query(), new Mutation(), new HealthQuery(getManager())) + .file("query-protocol/metadata.graphqls") + .resolvers(new MetadataQuery(getManager())) + .file("query-protocol/topology.graphqls") + .resolvers(new TopologyQuery(getManager())) + /** + * Metrics v2 query protocol is an alternative metrics query(s) of original v1, + * defined in the metric.graphql, top-n-records.graphqls, and aggregation.graphqls. + */ + .file("query-protocol/metrics-v2.graphqls") + .resolvers(new MetricsQuery(getManager())) + //////// + //Deprecated Queries + //////// + .file("query-protocol/metric.graphqls") + .resolvers(new MetricQuery(getManager())) + .file("query-protocol/aggregation.graphqls") + .resolvers(new AggregationQuery(getManager())) + .file("query-protocol/top-n-records.graphqls") + .resolvers(new TopNRecordsQuery(getManager())) + //////// + .file("query-protocol/trace.graphqls") + .resolvers(new TraceQuery(getManager())) + .file("query-protocol/alarm.graphqls") + .resolvers(new AlarmQuery(getManager())) + .file("query-protocol/log.graphqls") + .resolvers(new LogQuery(getManager())) + .file("query-protocol/profile.graphqls") + .resolvers(new ProfileQuery(getManager()), new ProfileMutation(getManager())) + .file("query-protocol/ui-configuration.graphqls") + .resolvers(new UIConfigurationManagement(getManager())) + .file("query-protocol/browser-log.graphqls") + .resolvers(new BrowserLogQuery(getManager())) + .build() + .makeExecutableSchema(); this.graphQL = GraphQL.newGraphQL(schema).build(); } @Override public void start() throws ServiceNotProvidedException, ModuleStartException { JettyHandlerRegister service = getManager().find(CoreModule.NAME) - .provider() - .getService(JettyHandlerRegister.class); + .provider() + .getService(JettyHandlerRegister.class); service.addHandler(new GraphQLQueryHandler(config.getPath(), graphQL)); } diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/BrowserLogQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/BrowserLogQuery.java new file mode 100644 index 000000000..3298f12f9 --- /dev/null +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/BrowserLogQuery.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.query.graphql.resolver; + +import com.coxautodev.graphql.tools.GraphQLQueryResolver; +import java.io.IOException; +import java.util.Optional; +import lombok.RequiredArgsConstructor; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.query.BrowserLogQueryService; +import org.apache.skywalking.oap.server.core.query.input.BrowserErrorLogQueryCondition; +import org.apache.skywalking.oap.server.core.query.type.BrowserErrorLogs; +import org.apache.skywalking.oap.server.library.module.ModuleManager; + +import static java.util.Objects.nonNull; + +@RequiredArgsConstructor +public class BrowserLogQuery implements GraphQLQueryResolver { + private final ModuleManager moduleManager; + private BrowserLogQueryService queryService; + + private BrowserLogQueryService getQueryService() { + return Optional.ofNullable(queryService).orElseGet(() -> { + queryService = moduleManager.find(CoreModule.NAME).provider().getService(BrowserLogQueryService.class); + return queryService; + }); + } + + public BrowserErrorLogs queryBrowserErrorLogs(BrowserErrorLogQueryCondition condition) throws IOException { + long startSecondTB = 0, endSecondTB = 0; + if (nonNull(condition.getQueryDuration())) { + startSecondTB = condition.getQueryDuration() + .getStartTimeBucketInSec(); + endSecondTB = condition.getQueryDuration() + .getEndTimeBucketInSec(); + } + + return getQueryService().queryBrowserErrorLogs( + condition.getServiceId(), condition.getServiceVersionId(), condition.getPagePathId(), + condition.getPagePath(), condition.getCategory() + , startSecondTB, endSecondTB, condition.getPaging() + ); + } +} diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol index 5ecefdf2c..92f62f13b 160000 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol @@ -1 +1 @@ -Subproject commit 5ecefdf2c16ca16d4973806bdd26f9eafa3faf1d +Subproject commit 92f62f13be3a2bf3dfd007ee85c68d54af2c6149 diff --git a/oap-server/server-receiver-plugin/pom.xml b/oap-server/server-receiver-plugin/pom.xml index 50b039471..3483578a9 100644 --- a/oap-server/server-receiver-plugin/pom.xml +++ b/oap-server/server-receiver-plugin/pom.xml @@ -42,6 +42,7 @@ skywalking-profile-receiver-plugin opencensus-receiver-plugin skywalking-meter-receiver-plugin + skywalking-browser-receiver-plugin diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/pom.xml b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/pom.xml new file mode 100644 index 000000000..f35587a8b --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/pom.xml @@ -0,0 +1,38 @@ + + + + + + server-receiver-plugin + org.apache.skywalking + 8.2.0-SNAPSHOT + + 4.0.0 + + skywalking-browser-receiver-plugin + + + + org.apache.skywalking + skywalking-sharing-server-plugin + ${project.version} + + + \ No newline at end of file diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/module/BrowserModule.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/module/BrowserModule.java new file mode 100644 index 000000000..7e3d3b8b1 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/module/BrowserModule.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.receiver.browser.module; + +import org.apache.skywalking.oap.server.library.module.ModuleDefine; + +public class BrowserModule extends ModuleDefine { + private static final String NAME = "receiver-browser"; + + public BrowserModule() { + super(NAME); + } + + @Override + public Class[] services() { + return new Class[0]; + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/BrowserModuleProvider.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/BrowserModuleProvider.java new file mode 100644 index 000000000..bc58caf01 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/BrowserModuleProvider.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.receiver.browser.provider; + +import org.apache.skywalking.oap.server.configuration.api.ConfigurationModule; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.oal.rt.OALEngineLoaderService; +import org.apache.skywalking.oap.server.core.server.GRPCHandlerRegister; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; +import org.apache.skywalking.oap.server.library.module.ModuleDefine; +import org.apache.skywalking.oap.server.library.module.ModuleProvider; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; +import org.apache.skywalking.oap.server.receiver.browser.module.BrowserModule; +import org.apache.skywalking.oap.server.receiver.browser.provider.handler.BrowserPerfServiceHandler; +import org.apache.skywalking.oap.server.receiver.browser.provider.parser.errorlog.ErrorLogParserListenerManager; +import org.apache.skywalking.oap.server.receiver.browser.provider.parser.errorlog.listener.ErrorLogRecordListener; +import org.apache.skywalking.oap.server.receiver.browser.provider.parser.errorlog.listener.MultiScopesErrorLogAnalysisListener; +import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.PerfDataParserListenerManager; +import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.listener.MultiScopesPerfDataAnalysisListener; +import org.apache.skywalking.oap.server.receiver.sharing.server.SharingServerModule; +import org.apache.skywalking.oap.server.telemetry.TelemetryModule; + +public class BrowserModuleProvider extends ModuleProvider { + private final BrowserServiceModuleConfig moduleConfig = new BrowserServiceModuleConfig(); + + @Override + public String name() { + return "default"; + } + + @Override + public Class module() { + return BrowserModule.class; + } + + @Override + public ModuleConfig createConfigBeanIfAbsent() { + return moduleConfig; + } + + @Override + public void prepare() throws ServiceNotProvidedException, ModuleStartException { + + } + + @Override + public void start() throws ServiceNotProvidedException, ModuleStartException { + // load browser analysis + getManager().find(CoreModule.NAME) + .provider() + .getService(OALEngineLoaderService.class) + .load(BrowserOALDefine.INSTANCE); + + GRPCHandlerRegister grpcHandlerRegister = getManager().find(SharingServerModule.NAME) + .provider().getService(GRPCHandlerRegister.class); + + grpcHandlerRegister.addHandler( + new BrowserPerfServiceHandler( + getManager(), moduleConfig, perfDataListenerManager(), errorLogListenerManager())); + } + + @Override + public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException { + + } + + @Override + public String[] requiredModules() { + return new String[] { + TelemetryModule.NAME, + CoreModule.NAME, + SharingServerModule.NAME, + ConfigurationModule.NAME + }; + } + + private PerfDataParserListenerManager perfDataListenerManager() { + PerfDataParserListenerManager listenerManager = new PerfDataParserListenerManager(); + listenerManager.add(new MultiScopesPerfDataAnalysisListener.Factory(getManager(), moduleConfig)); + + return listenerManager; + } + + private ErrorLogParserListenerManager errorLogListenerManager() { + ErrorLogParserListenerManager listenerManager = new ErrorLogParserListenerManager(); + listenerManager.add(new MultiScopesErrorLogAnalysisListener.Factory(getManager(), moduleConfig)); + listenerManager.add(new ErrorLogRecordListener.Factory(getManager(), moduleConfig)); + + return listenerManager; + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/BrowserOALDefine.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/BrowserOALDefine.java new file mode 100644 index 000000000..aebc9d42e --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/BrowserOALDefine.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.receiver.browser.provider; + +import org.apache.skywalking.oap.server.core.oal.rt.OALDefine; + +/** + * Browser OAL script includes the metrics related to browser only. + */ +public class BrowserOALDefine extends OALDefine { + public static final BrowserOALDefine INSTANCE = new BrowserOALDefine(); + + private BrowserOALDefine() { + super( + "oal/browser.oal", + "org.apache.skywalking.oap.server.core.browser.source" + ); + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/BrowserServiceModuleConfig.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/BrowserServiceModuleConfig.java new file mode 100644 index 000000000..7d9974f3d --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/BrowserServiceModuleConfig.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.receiver.browser.provider; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; + +public class BrowserServiceModuleConfig extends ModuleConfig { + /** + * The sample rate precision is 1/10000. 10000 means 100% sample in default. + */ + @Setter + @Getter + private int sampleRate = 10000; +} diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/handler/BrowserPerfServiceHandler.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/handler/BrowserPerfServiceHandler.java new file mode 100644 index 000000000..3b1f6331f --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/handler/BrowserPerfServiceHandler.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.receiver.browser.provider.handler; + +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.apm.network.common.v3.Commands; +import org.apache.skywalking.apm.network.language.agent.v3.BrowserErrorLog; +import org.apache.skywalking.apm.network.language.agent.v3.BrowserPerfData; +import org.apache.skywalking.apm.network.language.agent.v3.BrowserPerfServiceGrpc; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.server.grpc.GRPCHandler; +import org.apache.skywalking.oap.server.receiver.browser.provider.BrowserServiceModuleConfig; +import org.apache.skywalking.oap.server.receiver.browser.provider.parser.errorlog.ErrorLogAnalyzer; +import org.apache.skywalking.oap.server.receiver.browser.provider.parser.errorlog.ErrorLogParserListenerManager; +import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.PerfDataAnalyzer; +import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.PerfDataParserListenerManager; +import org.apache.skywalking.oap.server.telemetry.TelemetryModule; +import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics; +import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics; +import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator; +import org.apache.skywalking.oap.server.telemetry.api.MetricsTag; + +/** + * Collect and process the performance and error log. + */ +@Slf4j +public class BrowserPerfServiceHandler extends BrowserPerfServiceGrpc.BrowserPerfServiceImplBase implements GRPCHandler { + private final ModuleManager moduleManager; + private final BrowserServiceModuleConfig config; + private final PerfDataParserListenerManager perfDataListenerManager; + private final ErrorLogParserListenerManager errorLogListenerManager; + + // performance + private final HistogramMetrics perfHistogram; + private final CounterMetrics perfErrorCounter; + + // error log + private final HistogramMetrics errorLogHistogram; + private final CounterMetrics logErrorCounter; + + public BrowserPerfServiceHandler(ModuleManager moduleManager, + BrowserServiceModuleConfig config, + PerfDataParserListenerManager perfDataListenerManager, + ErrorLogParserListenerManager errorLogListenerManager) { + this.moduleManager = moduleManager; + this.config = config; + this.perfDataListenerManager = perfDataListenerManager; + this.errorLogListenerManager = errorLogListenerManager; + + MetricsCreator metricsCreator = moduleManager.find(TelemetryModule.NAME) + .provider() + .getService(MetricsCreator.class); + + // performance + perfHistogram = metricsCreator.createHistogramMetric( + "browser_perf_data_in_latency", "The process latency of browser performance data", + new MetricsTag.Keys("protocol"), new MetricsTag.Values("grpc") + ); + perfErrorCounter = metricsCreator.createCounter( + "browser_perf_data_analysis_error_count", "The error number of browser performance data analysis", + new MetricsTag.Keys("protocol"), new MetricsTag.Values("grpc") + ); + + // error log + errorLogHistogram = metricsCreator.createHistogramMetric( + "browser_error_log_in_latency", "The process latency of browser error log", new MetricsTag.Keys("protocol"), + new MetricsTag.Values("grpc") + ); + logErrorCounter = metricsCreator.createCounter( + "browser_error_log_analysis_error_count", "The error number of browser error log analysis", + new MetricsTag.Keys("protocol"), new MetricsTag.Values("grpc") + ); + } + + @Override + public void collectPerfData(final BrowserPerfData request, final StreamObserver responseObserver) { + if (log.isDebugEnabled()) { + log.debug("receive browser performance data"); + } + HistogramMetrics.Timer timer = perfHistogram.createTimer(); + try { + PerfDataAnalyzer analyzer = new PerfDataAnalyzer(moduleManager, perfDataListenerManager, config); + analyzer.doAnalysis(request); + } catch (Throwable e) { + log.error(e.getMessage(), e); + perfErrorCounter.inc(); + } finally { + timer.finish(); + responseObserver.onNext(Commands.newBuilder().build()); + responseObserver.onCompleted(); + } + } + + @Override + public StreamObserver collectErrorLogs(final StreamObserver responseObserver) { + return new StreamObserver() { + @Override + public void onNext(final BrowserErrorLog browserErrorLog) { + if (log.isDebugEnabled()) { + log.debug("receive browser error log"); + } + + HistogramMetrics.Timer timer = errorLogHistogram.createTimer(); + try { + ErrorLogAnalyzer analyzer = new ErrorLogAnalyzer(moduleManager, errorLogListenerManager, config); + analyzer.doAnalysis(browserErrorLog); + } catch (Throwable e) { + log.error(e.getMessage(), e); + logErrorCounter.inc(); + } finally { + timer.finish(); + } + } + + @Override + public void onError(final Throwable throwable) { + log.error(throwable.getMessage(), throwable); + responseObserver.onCompleted(); + } + + @Override + public void onCompleted() { + responseObserver.onNext(Commands.newBuilder().build()); + responseObserver.onCompleted(); + } + }; + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/BrowserErrorLogDecorator.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/BrowserErrorLogDecorator.java new file mode 100644 index 000000000..56785fd46 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/BrowserErrorLogDecorator.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.receiver.browser.provider.parser.errorlog; + +import lombok.RequiredArgsConstructor; +import org.apache.skywalking.apm.network.language.agent.v3.BrowserErrorLog; +import org.apache.skywalking.apm.network.language.agent.v3.ErrorCategory; + +@RequiredArgsConstructor +public class BrowserErrorLogDecorator { + private boolean isOrigin = true; + private final BrowserErrorLog errorLog; + private BrowserErrorLog.Builder builder; + + public String getUniqueId() { + return isOrigin ? errorLog.getUniqueId() : builder.getUniqueId(); + } + + public String getService() { + return isOrigin ? errorLog.getService() : builder.getService(); + } + + public String getServiceVersion() { + return isOrigin ? errorLog.getServiceVersion() : builder.getServiceVersion(); + } + + public long getTime() { + return isOrigin ? errorLog.getTime() : builder.getTime(); + } + + public String getPagePath() { + return isOrigin ? errorLog.getPagePath() : builder.getPagePath(); + } + + public ErrorCategory getCategory() { + return isOrigin ? errorLog.getCategory() : builder.getCategory(); + } + + public String getGrade() { + return isOrigin ? errorLog.getGrade() : builder.getGrade(); + } + + public String getMessage() { + return isOrigin ? errorLog.getMessage() : builder.getMessage(); + } + + public int getLine() { + return isOrigin ? errorLog.getLine() : builder.getLine(); + } + + public int getCol() { + return isOrigin ? errorLog.getCol() : builder.getCol(); + } + + public String getStack() { + return isOrigin ? errorLog.getStack() : builder.getStack(); + } + + public String getErrorUrl() { + return isOrigin ? errorLog.getErrorUrl() : builder.getErrorUrl(); + } + + public boolean isFirstReportedError() { + return isOrigin ? errorLog.getFirstReportedError() : builder.getFirstReportedError(); + } + + public byte[] toByteArray() { + return isOrigin ? errorLog.toByteArray() : builder.build().toByteArray(); + } + + public void setTime(long time) { + if (isOrigin) { + toBuilder(); + } + builder.setTime(time); + } + + void toBuilder() { + if (isOrigin) { + this.isOrigin = false; + this.builder = errorLog.toBuilder(); + } + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/ErrorLogAnalyzer.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/ErrorLogAnalyzer.java new file mode 100644 index 000000000..d56763ff9 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/ErrorLogAnalyzer.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.receiver.browser.provider.parser.errorlog; + +import java.util.LinkedList; +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.apm.network.language.agent.v3.BrowserErrorLog; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.receiver.browser.provider.BrowserServiceModuleConfig; +import org.apache.skywalking.oap.server.receiver.browser.provider.parser.errorlog.listener.ErrorLogAnalysisListener; + +@Slf4j +@RequiredArgsConstructor +public class ErrorLogAnalyzer { + private final ModuleManager moduleManager; + private final ErrorLogParserListenerManager listenerManager; + private final BrowserServiceModuleConfig moduleConfig; + + private final List analysisListeners = new LinkedList<>(); + + public void doAnalysis(BrowserErrorLog errorLog) { + createAnalysisListeners(); + + try { + BrowserErrorLogDecorator decorator = new BrowserErrorLogDecorator(errorLog); + // Use the server side current time. + long nowMillis = System.currentTimeMillis(); + decorator.setTime(nowMillis); + + notifyListener(decorator); + + notifyListenerToBuild(); + } catch (Throwable e) { + log.error(e.getMessage(), e); + } + } + + private void notifyListener(BrowserErrorLogDecorator decorator) { + analysisListeners.forEach(listener -> listener.parse(decorator)); + } + + private void notifyListenerToBuild() { + analysisListeners.forEach(ErrorLogAnalysisListener::build); + } + + private void createAnalysisListeners() { + listenerManager.getErrorLogAnalysisListeners() + .forEach(factory -> analysisListeners.add(factory.create(moduleManager, moduleConfig))); + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/ErrorLogParserListenerManager.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/ErrorLogParserListenerManager.java new file mode 100644 index 000000000..f7aea0f50 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/ErrorLogParserListenerManager.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.receiver.browser.provider.parser.errorlog; + +import java.util.LinkedList; +import java.util.List; +import lombok.Getter; +import org.apache.skywalking.oap.server.receiver.browser.provider.parser.errorlog.listener.ErrorLogListenerFactory; + +public class ErrorLogParserListenerManager { + @Getter + private final List errorLogAnalysisListeners = new LinkedList<>(); + + public void add(final ErrorLogListenerFactory factory) { + errorLogAnalysisListeners.add(factory); + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/listener/ErrorLogAnalysisListener.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/listener/ErrorLogAnalysisListener.java new file mode 100644 index 000000000..4fdd98a9d --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/listener/ErrorLogAnalysisListener.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.receiver.browser.provider.parser.errorlog.listener; + +import org.apache.skywalking.oap.server.receiver.browser.provider.parser.errorlog.BrowserErrorLogDecorator; + +/** + * ErrorLogAnalysisListener represents the callback when OAP does the browser error log analysis. + */ +public interface ErrorLogAnalysisListener { + /** + * The last step of the analysis process. Typically, the implementations forward the analysis results to the source + * receiver. + */ + void build(); + + /** + * Parse the raw data from the probe (js-client). + */ + void parse(BrowserErrorLogDecorator decorator); +} diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/listener/ErrorLogListenerFactory.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/listener/ErrorLogListenerFactory.java new file mode 100644 index 000000000..d13bfd60f --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/listener/ErrorLogListenerFactory.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.receiver.browser.provider.parser.errorlog.listener; + +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.receiver.browser.provider.BrowserServiceModuleConfig; + +public interface ErrorLogListenerFactory { + ErrorLogAnalysisListener create(final ModuleManager moduleManager, final BrowserServiceModuleConfig moduleConfig); +} diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/listener/ErrorLogRecordListener.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/listener/ErrorLogRecordListener.java new file mode 100644 index 000000000..216875db8 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/listener/ErrorLogRecordListener.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.receiver.browser.provider.parser.errorlog.listener; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.apm.util.StringUtil; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.NodeType; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.browser.source.BrowserErrorCategory; +import org.apache.skywalking.oap.server.core.browser.source.BrowserErrorLog; +import org.apache.skywalking.oap.server.core.config.NamingControl; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.receiver.browser.provider.BrowserServiceModuleConfig; +import org.apache.skywalking.oap.server.receiver.browser.provider.parser.errorlog.BrowserErrorLogDecorator; + +/** + * ErrorLogRecordListener forwards the error log raw data to the persistence layer with the query required conditions. + */ +@Slf4j +@RequiredArgsConstructor +public class ErrorLogRecordListener implements ErrorLogAnalysisListener { + private final NamingControl namingControl; + private final SourceReceiver sourceReceiver; + private final BrowserErrorLog errorLog = new BrowserErrorLog(); + private final ErrorLogRecordSampler sampler; + private SampleStatus sampleStatus = SampleStatus.UNKNOWN; + + /** + * Send BrowserErrorLog to the oreceiver. + */ + @Override + public void build() { + if (sampleStatus.equals(SampleStatus.SAMPLED)) { + sourceReceiver.receive(errorLog); + } + } + + @Override + public void parse(final BrowserErrorLogDecorator decorator) { + // sample + if (StringUtil.isEmpty(decorator.getUniqueId())) { + if (log.isDebugEnabled()) { + log.debug("Because uniqueId is empty BrowserErrorLog is ignored."); + } + sampleStatus = SampleStatus.IGNORED; + return; + } + + if (!sampler.shouldSample(decorator.getUniqueId().hashCode())) { + sampleStatus = SampleStatus.IGNORED; + return; + } + sampleStatus = SampleStatus.SAMPLED; + + // error log + errorLog.setUniqueId(decorator.getUniqueId()); + errorLog.setTimeBucket(TimeBucket.getRecordTimeBucket(decorator.getTime())); + errorLog.setTimestamp(decorator.getTime()); + + // service + String serviceName = namingControl.formatServiceName(decorator.getService()); + errorLog.setServiceId(IDManager.ServiceID.buildId(serviceName, NodeType.Browser)); + + // service version + String serviceVersionName = namingControl.formatInstanceName(decorator.getServiceVersion()); + errorLog.setServiceVersionId(IDManager.ServiceInstanceID.buildId(serviceName, serviceVersionName)); + + // page + String pagePath = namingControl.formatEndpointName(serviceName, decorator.getPagePath()); + errorLog.setPagePath(pagePath); + errorLog.setPagePathId(IDManager.EndpointID.buildId(serviceName, pagePath)); + + // raw data + errorLog.setErrorCategory(BrowserErrorCategory.fromErrorCategory(decorator.getCategory())); + errorLog.setDataBinary(decorator.toByteArray()); + } + + private enum SampleStatus { + UNKNOWN, SAMPLED, IGNORED + } + + public static class Factory implements ErrorLogListenerFactory { + private final SourceReceiver sourceReceiver; + private final NamingControl namingControl; + private final ErrorLogRecordSampler sampler; + + public Factory(ModuleManager moduleManager, BrowserServiceModuleConfig moduleConfig) { + this.sourceReceiver = moduleManager.find(CoreModule.NAME).provider().getService(SourceReceiver.class); + this.namingControl = moduleManager.find(CoreModule.NAME).provider().getService(NamingControl.class); + this.sampler = new ErrorLogRecordSampler(moduleConfig.getSampleRate()); + } + + @Override + public ErrorLogAnalysisListener create(final ModuleManager moduleManager, + final BrowserServiceModuleConfig moduleConfig) { + return new ErrorLogRecordListener(namingControl, sourceReceiver, sampler); + } + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/listener/ErrorLogRecordSampler.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/listener/ErrorLogRecordSampler.java new file mode 100644 index 000000000..5e825b70b --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/listener/ErrorLogRecordSampler.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.receiver.browser.provider.parser.errorlog.listener; + +import lombok.RequiredArgsConstructor; +import org.apache.skywalking.apm.network.language.agent.v3.BrowserErrorLog; + +/** + * The sampler makes the sampling mechanism works at backend side. Sample result: [0,sampleRate) sampled, (sampleRate,~) + * ignored + */ +@RequiredArgsConstructor +public class ErrorLogRecordSampler { + private final int sampleRate; + + /** + * through {@link BrowserErrorLog#getUniqueId()} hash code. + */ + public boolean shouldSample(int hashCode) { + return hashCode % 10000 < sampleRate; + } + +} diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/listener/MultiScopesErrorLogAnalysisListener.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/listener/MultiScopesErrorLogAnalysisListener.java new file mode 100644 index 000000000..30ef1f3ca --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/listener/MultiScopesErrorLogAnalysisListener.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.receiver.browser.provider.parser.errorlog.listener; + +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.browser.source.BrowserAppTrafficCategory; +import org.apache.skywalking.oap.server.core.config.NamingControl; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.receiver.browser.provider.BrowserServiceModuleConfig; +import org.apache.skywalking.oap.server.receiver.browser.provider.parser.errorlog.BrowserErrorLogDecorator; + +/** + * MultiScopesErrorLogAnalysisListener analysis error log, the kinds of error and error sum metrics. + */ +public class MultiScopesErrorLogAnalysisListener implements ErrorLogAnalysisListener { + private final SourceReceiver sourceReceiver; + + private final SourceBuilder sourceBuilder; + + public MultiScopesErrorLogAnalysisListener(final SourceReceiver sourceReceiver, final NamingControl namingControl) { + this.sourceReceiver = sourceReceiver; + this.sourceBuilder = new SourceBuilder(namingControl); + } + + /** + * Send BrowserAppTraffic, BrowserAppSingleVersionTraffic, BrowserAppPageTraffic and BrowserAppPagePerf scope to the + * receiver. + */ + @Override + public void build() { + sourceReceiver.receive(sourceBuilder.toBrowserAppTraffic()); + sourceReceiver.receive(sourceBuilder.toBrowserAppSingleVersionTraffic()); + sourceReceiver.receive(sourceBuilder.toBrowserAppPageTraffic()); + } + + @Override + public void parse(final BrowserErrorLogDecorator decorator) { + sourceBuilder.setService(decorator.getService()); + sourceBuilder.setServiceVersion(decorator.getServiceVersion()); + sourceBuilder.setPatePath(decorator.getPagePath()); + + // time bucket + sourceBuilder.setTimeBucket(TimeBucket.getMinuteTimeBucket(decorator.getTime())); + + // category + sourceBuilder.setTrafficCategory( + decorator.isFirstReportedError() ? BrowserAppTrafficCategory.FIRST_ERROR : BrowserAppTrafficCategory.ERROR); + sourceBuilder.setErrorCategory(decorator.getCategory()); + } + + public static class Factory implements ErrorLogListenerFactory { + private final SourceReceiver sourceReceiver; + private final NamingControl namingControl; + + public Factory(ModuleManager moduleManager, BrowserServiceModuleConfig moduleConfig) { + this.sourceReceiver = moduleManager.find(CoreModule.NAME).provider().getService(SourceReceiver.class); + this.namingControl = moduleManager.find(CoreModule.NAME).provider().getService(NamingControl.class); + } + + @Override + public ErrorLogAnalysisListener create(final ModuleManager moduleManager, + final BrowserServiceModuleConfig moduleConfig) { + return new MultiScopesErrorLogAnalysisListener(sourceReceiver, namingControl); + } + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/listener/SourceBuilder.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/listener/SourceBuilder.java new file mode 100644 index 000000000..4541a5fd9 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/errorlog/listener/SourceBuilder.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.receiver.browser.provider.parser.errorlog.listener; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import org.apache.skywalking.apm.network.language.agent.v3.ErrorCategory; +import org.apache.skywalking.oap.server.core.browser.source.BrowserAppPageTraffic; +import org.apache.skywalking.oap.server.core.browser.source.BrowserAppSingleVersionTraffic; +import org.apache.skywalking.oap.server.core.browser.source.BrowserAppTraffic; +import org.apache.skywalking.oap.server.core.browser.source.BrowserAppTrafficCategory; +import org.apache.skywalking.oap.server.core.browser.source.BrowserAppTrafficSource; +import org.apache.skywalking.oap.server.core.browser.source.BrowserErrorCategory; +import org.apache.skywalking.oap.server.core.config.NamingControl; + +@RequiredArgsConstructor +class SourceBuilder { + private final NamingControl namingControl; + + @Getter + private String service; + + public void setService(final String service) { + this.service = namingControl.formatServiceName(service); + } + + @Getter + private String serviceVersion; + + public void setServiceVersion(final String serviceVersion) { + this.serviceVersion = namingControl.formatInstanceName(serviceVersion); + } + + @Getter + private String patePath; + + public void setPatePath(final String patePath) { + this.patePath = namingControl.formatEndpointName(service, patePath); + } + + @Setter + @Getter + private long timeBucket; + + @Setter + @Getter + private BrowserAppTrafficCategory trafficCategory; + + @Setter + @Getter + private BrowserErrorCategory errorCategory; + + public void setErrorCategory(ErrorCategory category) { + this.errorCategory = BrowserErrorCategory.fromErrorCategory(category); + } + + private void toBrowserAppTrafficSource(BrowserAppTrafficSource source) { + source.setTimeBucket(timeBucket); + source.setTrafficCategory(trafficCategory); + source.setErrorCategory(errorCategory); + } + + /** + * Browser service traffic error related source. + */ + BrowserAppTraffic toBrowserAppTraffic() { + BrowserAppTraffic traffic = new BrowserAppTraffic(); + toBrowserAppTrafficSource(traffic); + traffic.setName(service); + traffic.setTrafficCategory(trafficCategory); + traffic.setErrorCategory(errorCategory); + return traffic; + } + + /** + * Browser single version error metrics related source. + */ + BrowserAppSingleVersionTraffic toBrowserAppSingleVersionTraffic() { + BrowserAppSingleVersionTraffic traffic = new BrowserAppSingleVersionTraffic(); + toBrowserAppTrafficSource(traffic); + traffic.setName(serviceVersion); + traffic.setServiceName(service); + return traffic; + } + + /** + * Browser page error metrics related source. + */ + BrowserAppPageTraffic toBrowserAppPageTraffic() { + BrowserAppPageTraffic traffic = new BrowserAppPageTraffic(); + toBrowserAppTrafficSource(traffic); + traffic.setName(patePath); + traffic.setServiceName(service); + return traffic; + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/BrowserPerfDataDecorator.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/BrowserPerfDataDecorator.java new file mode 100644 index 000000000..f48637590 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/BrowserPerfDataDecorator.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance; + +import lombok.RequiredArgsConstructor; +import org.apache.skywalking.apm.network.language.agent.v3.BrowserPerfData; + +@RequiredArgsConstructor +public class BrowserPerfDataDecorator { + private boolean isOrigin = true; + private final BrowserPerfData browserPerfData; + private BrowserPerfData.Builder builder; + + public String getService() { + return isOrigin ? browserPerfData.getService() : builder.getService(); + } + + public String getServiceVersion() { + return isOrigin ? browserPerfData.getServiceVersion() : builder.getServiceVersion(); + } + + public long getTime() { + return isOrigin ? browserPerfData.getTime() : builder.getTime(); + } + + public String getPagePath() { + return isOrigin ? browserPerfData.getPagePath() : builder.getPagePath(); + } + + public int getRedirectTime() { + return isOrigin ? browserPerfData.getRedirectTime() : builder.getRedirectTime(); + } + + public int getDnsTime() { + return isOrigin ? browserPerfData.getDnsTime() : builder.getDnsTime(); + } + + public int getTtfbTime() { + return isOrigin ? browserPerfData.getTtfbTime() : builder.getTtfbTime(); + } + + public int getTcpTime() { + return isOrigin ? browserPerfData.getTcpTime() : builder.getTcpTime(); + } + + public int getTransTime() { + return isOrigin ? browserPerfData.getTransTime() : builder.getTransTime(); + } + + public int getDomAnalysisTime() { + return isOrigin ? browserPerfData.getDomAnalysisTime() : builder.getDomAnalysisTime(); + } + + public int getFptTime() { + return isOrigin ? browserPerfData.getFptTime() : builder.getFptTime(); + } + + public int getDomReadyTime() { + return isOrigin ? browserPerfData.getDomReadyTime() : builder.getDomReadyTime(); + } + + public int getLoadPageTime() { + return isOrigin ? browserPerfData.getLoadPageTime() : builder.getLoadPageTime(); + } + + public int getResTime() { + return isOrigin ? browserPerfData.getResTime() : builder.getResTime(); + } + + public int getSslTime() { + return isOrigin ? browserPerfData.getSslTime() : builder.getSslTime(); + } + + public int getTtlTime() { + return isOrigin ? browserPerfData.getTtlTime() : builder.getTtlTime(); + } + + public int getFirstPackTime() { + return isOrigin ? browserPerfData.getFirstPackTime() : builder.getFirstPackTime(); + } + + public int getFmpTime() { + return isOrigin ? browserPerfData.getFmpTime() : builder.getFmpTime(); + } + + public void setTime(long time) { + if (isOrigin) { + toBuilder(); + } + builder.setTime(time); + } + + void toBuilder() { + if (isOrigin) { + this.isOrigin = false; + this.builder = browserPerfData.toBuilder(); + } + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/PerfDataAnalyzer.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/PerfDataAnalyzer.java new file mode 100644 index 000000000..ae66fe138 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/PerfDataAnalyzer.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance; + +import java.util.LinkedList; +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.apm.network.language.agent.v3.BrowserPerfData; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.receiver.browser.provider.BrowserServiceModuleConfig; +import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.listener.PerfDataAnalysisListener; + +@Slf4j +@RequiredArgsConstructor +public class PerfDataAnalyzer { + private final ModuleManager moduleManager; + private final PerfDataParserListenerManager listenerManager; + private final BrowserServiceModuleConfig config; + + private final List analysisListeners = new LinkedList<>(); + + public void doAnalysis(BrowserPerfData browserPerfData) { + createAnalysisListeners(); + + try { + BrowserPerfDataDecorator decorator = new BrowserPerfDataDecorator(browserPerfData); + // Use the server side current time. + long nowMillis = System.currentTimeMillis(); + decorator.setTime(nowMillis); + + notifyListener(decorator); + + notifyListenerToBuild(); + } catch (Throwable e) { + log.error(e.getMessage(), e); + } + } + + private void notifyListener(BrowserPerfDataDecorator decorator) { + analysisListeners.forEach(listener -> listener.parse(decorator)); + } + + private void notifyListenerToBuild() { + analysisListeners.forEach(PerfDataAnalysisListener::build); + } + + private void createAnalysisListeners() { + listenerManager.getPerfDataListenerFactories() + .forEach(factory -> analysisListeners.add(factory.create(moduleManager, config))); + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/PerfDataParserListenerManager.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/PerfDataParserListenerManager.java new file mode 100644 index 000000000..1d369c233 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/PerfDataParserListenerManager.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance; + +import java.util.LinkedList; +import java.util.List; +import lombok.Getter; +import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.listener.PerfDataListenerFactory; + +public class PerfDataParserListenerManager { + @Getter + private final List perfDataListenerFactories = new LinkedList<>(); + + public void add(final PerfDataListenerFactory factory) { + perfDataListenerFactories.add(factory); + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/listener/MultiScopesPerfDataAnalysisListener.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/listener/MultiScopesPerfDataAnalysisListener.java new file mode 100644 index 000000000..4aed69033 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/listener/MultiScopesPerfDataAnalysisListener.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.listener; + +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.config.NamingControl; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.receiver.browser.provider.BrowserServiceModuleConfig; +import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.BrowserPerfDataDecorator; + +/** + * Browser traffic and page Performance related metrics. + */ +@Slf4j +public class MultiScopesPerfDataAnalysisListener implements PerfDataAnalysisListener { + private final SourceReceiver sourceReceiver; + + private final SourceBuilder sourceBuilder; + + public MultiScopesPerfDataAnalysisListener(final SourceReceiver sourceReceiver, + final NamingControl namingControl) { + this.sourceReceiver = sourceReceiver; + this.sourceBuilder = new SourceBuilder(namingControl); + } + + /** + * Send BrowserAppTraffic, BrowserAppSingleVersionTraffic, BrowserAppPageTraffic and BrowserAppPagePerf scope to the + * receiver. + */ + @Override + public void build() { + // traffic + sourceReceiver.receive(sourceBuilder.toBrowserAppTraffic()); + sourceReceiver.receive(sourceBuilder.toBrowserAppSingleVersionTraffic()); + sourceReceiver.receive(sourceBuilder.toBrowserAppPageTraffic()); + + // performance (currently only page level performance data is analyzed) + sourceReceiver.receive(sourceBuilder.toBrowserAppPagePerf()); + } + + /** + * Parse raw data + */ + @Override + public void parse(final BrowserPerfDataDecorator decorator) { + sourceBuilder.setService(decorator.getService()); + sourceBuilder.setServiceVersion(decorator.getServiceVersion()); + sourceBuilder.setPatePath(decorator.getPagePath()); + + // time + sourceBuilder.setTimeBucket(TimeBucket.getMinuteTimeBucket(decorator.getTime())); + + // performance related + sourceBuilder.setRedirectTime(decorator.getRedirectTime()); + sourceBuilder.setDnsTime(decorator.getDnsTime()); + sourceBuilder.setTtfbTime(decorator.getTtfbTime()); + sourceBuilder.setTcpTime(decorator.getTcpTime()); + sourceBuilder.setTransTime(decorator.getTransTime()); + sourceBuilder.setDomAnalysisTime(decorator.getDomAnalysisTime()); + sourceBuilder.setFptTime(decorator.getFptTime()); + sourceBuilder.setDomReadyTime(decorator.getDomReadyTime()); + sourceBuilder.setLoadPageTime(decorator.getLoadPageTime()); + sourceBuilder.setResTime(decorator.getResTime()); + sourceBuilder.setSslTime(decorator.getSslTime()); + sourceBuilder.setTtlTime(decorator.getTtlTime()); + sourceBuilder.setFirstPackTime(decorator.getFirstPackTime()); + sourceBuilder.setFmpTime(decorator.getFmpTime()); + } + + public static class Factory implements PerfDataListenerFactory { + + private final SourceReceiver sourceReceiver; + private final NamingControl namingControl; + + public Factory(ModuleManager moduleManager, BrowserServiceModuleConfig moduleConfig) { + this.sourceReceiver = moduleManager.find(CoreModule.NAME) + .provider() + .getService(SourceReceiver.class); + + this.namingControl = moduleManager.find(CoreModule.NAME) + .provider() + .getService(NamingControl.class); + } + + @Override + public PerfDataAnalysisListener create(final ModuleManager moduleManager, + final BrowserServiceModuleConfig moduleConfig) { + return new MultiScopesPerfDataAnalysisListener(sourceReceiver, namingControl); + } + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/listener/PerfDataAnalysisListener.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/listener/PerfDataAnalysisListener.java new file mode 100644 index 000000000..b5a45742e --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/listener/PerfDataAnalysisListener.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.listener; + +import org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.BrowserPerfDataDecorator; + +/** + * BrowserPerfDataListener represents the callback when OAP does the browser performance data analysis. + */ +public interface PerfDataAnalysisListener { + /** + * The last step of the analysis process. Typically, the implementations forward the analysis results to the source + * receiver. + */ + void build(); + + /** + * Parse the raw data from the probe (js-client). + */ + void parse(BrowserPerfDataDecorator decorator); +} diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/listener/PerfDataListenerFactory.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/listener/PerfDataListenerFactory.java new file mode 100644 index 000000000..24cd4e2f8 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/listener/PerfDataListenerFactory.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.listener; + +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.receiver.browser.provider.BrowserServiceModuleConfig; + +/** + * BrowserPerfDataListenerFactory implementation creates the listener instances when required. Every + * BrowserPerfDataListener could have its own creation factory. + */ +public interface PerfDataListenerFactory { + PerfDataAnalysisListener create(ModuleManager moduleManager, BrowserServiceModuleConfig moduleConfig); +} diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/listener/SourceBuilder.java b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/listener/SourceBuilder.java new file mode 100644 index 000000000..7d50ea456 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/browser/provider/parser/performance/listener/SourceBuilder.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.receiver.browser.provider.parser.performance.listener; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.browser.source.BrowserAppPagePerf; +import org.apache.skywalking.oap.server.core.browser.source.BrowserAppPageTraffic; +import org.apache.skywalking.oap.server.core.browser.source.BrowserAppPerf; +import org.apache.skywalking.oap.server.core.browser.source.BrowserAppPerfSource; +import org.apache.skywalking.oap.server.core.browser.source.BrowserAppSingleVersionPerf; +import org.apache.skywalking.oap.server.core.browser.source.BrowserAppSingleVersionTraffic; +import org.apache.skywalking.oap.server.core.browser.source.BrowserAppTraffic; +import org.apache.skywalking.oap.server.core.browser.source.BrowserAppTrafficCategory; +import org.apache.skywalking.oap.server.core.browser.source.BrowserAppTrafficSource; +import org.apache.skywalking.oap.server.core.config.NamingControl; +import org.apache.skywalking.oap.server.core.source.Source; + +/** + * Browser traffic and performance related source. + */ +@RequiredArgsConstructor +class SourceBuilder { + private final NamingControl namingControl; + + @Getter + private String service; + + public void setService(final String service) { + this.service = namingControl.formatServiceName(service); + } + + @Getter + private String serviceVersion; + + public void setServiceVersion(final String serviceVersion) { + this.serviceVersion = namingControl.formatInstanceName(serviceVersion); + } + + @Getter + private String patePath; + + public void setPatePath(final String patePath) { + this.patePath = namingControl.formatEndpointName(service, patePath); + } + + @Setter + @Getter + private long timeBucket; + + // performance data detail + @Setter + @Getter + private int redirectTime; + @Setter + @Getter + private int dnsTime; + @Setter + @Getter + private int ttfbTime; + @Setter + @Getter + private int tcpTime; + @Setter + @Getter + private int transTime; + @Setter + @Getter + private int domAnalysisTime; + @Setter + @Getter + private int fptTime; + @Setter + @Getter + private int domReadyTime; + @Setter + @Getter + private int loadPageTime; + @Setter + @Getter + private int resTime; + @Setter + @Getter + private int sslTime; + @Setter + @Getter + private int ttlTime; + @Setter + @Getter + private int firstPackTime; + @Setter + @Getter + private int fmpTime; + + private void toSource(Source source) { + source.setTimeBucket(timeBucket); + } + + private void toBrowserAppTrafficSource(BrowserAppTrafficSource source) { + toSource(source); + source.setTrafficCategory(BrowserAppTrafficCategory.NORMAL); + } + + /** + * Browser service meta and traffic metrics related source. + */ + BrowserAppTraffic toBrowserAppTraffic() { + BrowserAppTraffic traffic = new BrowserAppTraffic(); + traffic.setName(service); + toBrowserAppTrafficSource(traffic); + return traffic; + } + + /** + * Browser single version meta and traffic metrics related source. + */ + BrowserAppSingleVersionTraffic toBrowserAppSingleVersionTraffic() { + BrowserAppSingleVersionTraffic traffic = new BrowserAppSingleVersionTraffic(); + traffic.setName(serviceVersion); + traffic.setServiceName(service); + toBrowserAppTrafficSource(traffic); + return traffic; + } + + /** + * Browser page meta and traffic metrics related source. + */ + BrowserAppPageTraffic toBrowserAppPageTraffic() { + BrowserAppPageTraffic traffic = new BrowserAppPageTraffic(); + traffic.setName(patePath); + traffic.setServiceName(service); + toBrowserAppTrafficSource(traffic); + return traffic; + } + + private void toBrowserAppPerfSource(BrowserAppPerfSource source) { + toSource(source); + source.setRedirectTime(redirectTime); + source.setDnsTime(dnsTime); + source.setTtfbTime(ttfbTime); + source.setTcpTime(tcpTime); + source.setTransTime(transTime); + source.setDomAnalysisTime(domAnalysisTime); + source.setFptTime(fptTime); + source.setDomReadyTime(domReadyTime); + source.setLoadPageTime(loadPageTime); + source.setResTime(resTime); + source.setSslTime(sslTime); + source.setTtlTime(ttlTime); + source.setFirstPackTime(firstPackTime); + source.setFmpTime(fmpTime); + } + + /** + * Browser service performance related source. + */ + BrowserAppPerf toBrowserAppPerf() { + BrowserAppPerf perf = new BrowserAppPerf(); + perf.setName(service); + toBrowserAppPerfSource(perf); + return perf; + } + + /** + * Browser single version performance related source. + */ + BrowserAppSingleVersionPerf toBrowserAppSingleVersionPerf() { + BrowserAppSingleVersionPerf perf = new BrowserAppSingleVersionPerf(); + perf.setName(serviceVersion); + perf.setServiceName(service); + toBrowserAppPerfSource(perf); + return perf; + } + + /** + * Browser page performance related source. + */ + BrowserAppPagePerf toBrowserAppPagePerf() { + BrowserAppPagePerf perf = new BrowserAppPagePerf(); + perf.setName(patePath); + perf.setServiceName(service); + toBrowserAppPerfSource(perf); + return perf; + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine new file mode 100644 index 000000000..9c91d56e5 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.oap.server.receiver.browser.module.BrowserModule \ No newline at end of file diff --git a/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider new file mode 100644 index 000000000..2f2efd018 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-browser-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.oap.server.receiver.browser.provider.BrowserModuleProvider \ No newline at end of file diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java index 030b965ac..5515e3d78 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java @@ -42,6 +42,7 @@ import org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskQueryDA import org.apache.skywalking.oap.server.core.storage.profile.IProfileThreadSnapshotQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IAggregationQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IAlarmQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IBrowserLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.ILogQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IMetadataQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IMetricsQueryDAO; @@ -64,6 +65,7 @@ import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.TimeSe import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.cache.NetworkAddressAliasEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.AggregationQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.AlarmQueryEsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.BrowserLogQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.LogQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.MetadataQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.MetricsQueryEsDAO; @@ -168,6 +170,7 @@ public class StorageModuleElasticsearchProvider extends ModuleProvider { this.registerServiceImplementation(IMetricsQueryDAO.class, new MetricsQueryEsDAO(elasticSearchClient)); this.registerServiceImplementation( ITraceQueryDAO.class, new TraceQueryEsDAO(elasticSearchClient, config.getSegmentQueryMaxSize())); + this.registerServiceImplementation(IBrowserLogQueryDAO.class, new BrowserLogQueryEsDAO(elasticSearchClient)); this.registerServiceImplementation( IMetadataQueryDAO.class, new MetadataQueryEsDAO(elasticSearchClient, config.getMetadataQueryMaxSize())); this.registerServiceImplementation(IAggregationQueryDAO.class, new AggregationQueryEsDAO(elasticSearchClient)); diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/BrowserLogQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/BrowserLogQueryEsDAO.java new file mode 100644 index 000000000..daa0ed931 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/BrowserLogQueryEsDAO.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import com.google.common.base.Strings; +import java.io.IOException; +import org.apache.skywalking.apm.util.StringUtil; +import org.apache.skywalking.oap.server.core.browser.manual.errorlog.BrowserErrorLogRecord; +import org.apache.skywalking.oap.server.core.browser.source.BrowserErrorCategory; +import org.apache.skywalking.oap.server.core.query.type.BrowserErrorLog; +import org.apache.skywalking.oap.server.core.query.type.BrowserErrorLogs; +import org.apache.skywalking.oap.server.core.storage.query.IBrowserLogQueryDAO; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.MatchCNameBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +import static java.util.Objects.nonNull; + +public class BrowserLogQueryEsDAO extends EsDAO implements IBrowserLogQueryDAO { + public BrowserLogQueryEsDAO(final ElasticSearchClient client) { + super(client); + } + + @Override + public BrowserErrorLogs queryBrowserErrorLogs(final String serviceId, + final String serviceVersionId, + final String pagePathId, + final String pagePath, + final BrowserErrorCategory category, + final long startSecondTB, + final long endSecondTB, + final int limit, + final int from) throws IOException { + SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource(); + + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + sourceBuilder.query(boolQueryBuilder); + + if (startSecondTB != 0 && endSecondTB != 0) { + boolQueryBuilder.must().add( + QueryBuilders.rangeQuery(BrowserErrorLogRecord.TIME_BUCKET).gte(startSecondTB).lte(endSecondTB)); + } + + if (!Strings.isNullOrEmpty(pagePath)) { + String matchCName = MatchCNameBuilder.INSTANCE.build(BrowserErrorLogRecord.PAGE_PATH); + boolQueryBuilder.must().add(QueryBuilders.matchPhraseQuery(matchCName, pagePath)); + } + if (StringUtil.isNotEmpty(serviceId)) { + boolQueryBuilder.must().add(QueryBuilders.termQuery(BrowserErrorLogRecord.SERVICE_ID, serviceId)); + } + if (StringUtil.isNotEmpty(serviceVersionId)) { + boolQueryBuilder.must() + .add(QueryBuilders.termQuery(BrowserErrorLogRecord.SERVICE_VERSION_ID, serviceVersionId)); + } + if (StringUtil.isNotEmpty(pagePathId)) { + boolQueryBuilder.must().add(QueryBuilders.termQuery(BrowserErrorLogRecord.PAGE_PATH_ID, pagePathId)); + } + if (nonNull(category)) { + boolQueryBuilder.must() + .add(QueryBuilders.termQuery(BrowserErrorLogRecord.ERROR_CATEGORY, category.getValue())); + } + sourceBuilder.size(limit); + sourceBuilder.from(from); + + SearchResponse response = getClient().search(BrowserErrorLogRecord.INDEX_NAME, sourceBuilder); + + BrowserErrorLogs logs = new BrowserErrorLogs(); + logs.setTotal((int) response.getHits().totalHits); + + for (SearchHit searchHit : response.getHits().getHits()) { + String dataBinaryBase64 = (String) searchHit.getSourceAsMap().get(BrowserErrorLogRecord.DATA_BINARY); + if (nonNull(dataBinaryBase64)) { + BrowserErrorLog log = parserDataBinary(dataBinaryBase64); + logs.getLogs().add(log); + } + } + return logs; + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/StorageModuleElasticsearch7Provider.java b/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/StorageModuleElasticsearch7Provider.java index 414946fd7..9acab873a 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/StorageModuleElasticsearch7Provider.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/StorageModuleElasticsearch7Provider.java @@ -40,6 +40,7 @@ import org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskQueryDA import org.apache.skywalking.oap.server.core.storage.profile.IProfileThreadSnapshotQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IAggregationQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IAlarmQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IBrowserLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.ILogQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IMetadataQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IMetricsQueryDAO; @@ -66,6 +67,7 @@ import org.apache.skywalking.oap.server.storage.plugin.elasticsearch7.client.Ela import org.apache.skywalking.oap.server.storage.plugin.elasticsearch7.dao.StorageEs7DAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch7.query.AggregationQueryEs7DAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch7.query.AlarmQueryEs7DAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch7.query.BrowserLogQueryEs7DAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch7.query.LogQueryEs7DAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch7.query.MetadataQueryEs7DAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch7.query.MetricsQueryEs7DAO; @@ -120,7 +122,7 @@ public class StorageModuleElasticsearch7Provider extends ModuleProvider { } if (!StringUtil.isEmpty(config.getSecretsManagementFile())) { MultipleFilesChangeMonitor monitor = new MultipleFilesChangeMonitor( - 10, readableContents -> { + 10, readableContents -> { final byte[] secretsFileContent = readableContents.get(0); if (secretsFileContent == null) { return; @@ -148,58 +150,62 @@ public class StorageModuleElasticsearch7Provider extends ModuleProvider { } elasticSearch7Client = new ElasticSearch7Client( - config.getClusterNodes(), config.getProtocol(), config.getTrustStorePath(), config - .getTrustStorePass(), config.getUser(), config.getPassword(), - indexNameConverters(config.getNameSpace()) + config.getClusterNodes(), config.getProtocol(), config.getTrustStorePath(), config + .getTrustStorePass(), config.getUser(), config.getPassword(), + indexNameConverters(config.getNameSpace()) ); this.registerServiceImplementation( - IBatchDAO.class, new BatchProcessEsDAO(elasticSearch7Client, config.getBulkActions(), - config.getFlushInterval(), config.getConcurrentRequests() - )); + IBatchDAO.class, new BatchProcessEsDAO(elasticSearch7Client, config.getBulkActions(), + config.getFlushInterval(), config.getConcurrentRequests() + )); this.registerServiceImplementation(StorageDAO.class, new StorageEs7DAO(elasticSearch7Client)); this.registerServiceImplementation( - IHistoryDeleteDAO.class, new HistoryDeleteEsDAO(elasticSearch7Client)); + IHistoryDeleteDAO.class, new HistoryDeleteEsDAO(elasticSearch7Client)); this.registerServiceImplementation( - INetworkAddressAliasDAO.class, new NetworkAddressAliasEsDAO( - elasticSearch7Client, - config.getResultWindowMaxSize() - )); + INetworkAddressAliasDAO.class, new NetworkAddressAliasEsDAO( + elasticSearch7Client, + config.getResultWindowMaxSize() + )); this.registerServiceImplementation(ITopologyQueryDAO.class, new TopologyQueryEsDAO(elasticSearch7Client)); this.registerServiceImplementation(IMetricsQueryDAO.class, new MetricsQueryEs7DAO(elasticSearch7Client)); this.registerServiceImplementation( - ITraceQueryDAO.class, new TraceQueryEs7DAO(elasticSearch7Client, config.getSegmentQueryMaxSize())); + ITraceQueryDAO.class, new TraceQueryEs7DAO(elasticSearch7Client, config.getSegmentQueryMaxSize())); + this.registerServiceImplementation(IBrowserLogQueryDAO.class, new BrowserLogQueryEs7DAO(elasticSearch7Client)); this.registerServiceImplementation( - IMetadataQueryDAO.class, new MetadataQueryEs7DAO(elasticSearch7Client, config.getMetadataQueryMaxSize())); + IMetadataQueryDAO.class, new MetadataQueryEs7DAO(elasticSearch7Client, config.getMetadataQueryMaxSize())); this.registerServiceImplementation( - IAggregationQueryDAO.class, new AggregationQueryEs7DAO(elasticSearch7Client)); + IAggregationQueryDAO.class, new AggregationQueryEs7DAO(elasticSearch7Client)); this.registerServiceImplementation(IAlarmQueryDAO.class, new AlarmQueryEs7DAO(elasticSearch7Client)); this.registerServiceImplementation(ITopNRecordsQueryDAO.class, new TopNRecordsQueryEsDAO(elasticSearch7Client)); this.registerServiceImplementation(ILogQueryDAO.class, new LogQueryEs7DAO(elasticSearch7Client)); this.registerServiceImplementation( - IProfileTaskQueryDAO.class, new ProfileTaskQueryEsDAO( - elasticSearch7Client, - config.getProfileTaskQueryMaxSize() - )); + IProfileTaskQueryDAO.class, new ProfileTaskQueryEsDAO( + elasticSearch7Client, + config.getProfileTaskQueryMaxSize() + )); this.registerServiceImplementation( - IProfileTaskLogQueryDAO.class, new ProfileTaskLogEsDAO( - elasticSearch7Client, - config.getProfileTaskQueryMaxSize() - )); + IProfileTaskLogQueryDAO.class, new ProfileTaskLogEsDAO( + elasticSearch7Client, + config.getProfileTaskQueryMaxSize() + )); this.registerServiceImplementation( - IProfileThreadSnapshotQueryDAO.class, new ProfileThreadSnapshotQueryEs7DAO( - elasticSearch7Client, - config.getProfileTaskQueryMaxSize() - )); + IProfileThreadSnapshotQueryDAO.class, new ProfileThreadSnapshotQueryEs7DAO( + elasticSearch7Client, + config.getProfileTaskQueryMaxSize() + )); this.registerServiceImplementation( - UITemplateManagementDAO.class, new UITemplateManagementEsDAO(elasticSearch7Client)); + UITemplateManagementDAO.class, new UITemplateManagementEsDAO(elasticSearch7Client)); } @Override public void start() throws ModuleStartException { - MetricsCreator metricCreator = getManager().find(TelemetryModule.NAME).provider().getService(MetricsCreator.class); - HealthCheckMetrics healthChecker = metricCreator.createHealthCheckerGauge("storage_elasticsearch", MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE); + MetricsCreator metricCreator = getManager().find(TelemetryModule.NAME) + .provider() + .getService(MetricsCreator.class); + HealthCheckMetrics healthChecker = metricCreator.createHealthCheckerGauge( + "storage_elasticsearch", MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE); elasticSearch7Client.registerChecker(healthChecker); try { elasticSearch7Client.connect(); @@ -217,6 +223,6 @@ public class StorageModuleElasticsearch7Provider extends ModuleProvider { @Override public String[] requiredModules() { - return new String[]{CoreModule.NAME}; + return new String[] {CoreModule.NAME}; } } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/query/BrowserLogQueryEs7DAO.java b/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/query/BrowserLogQueryEs7DAO.java new file mode 100644 index 000000000..2f10c4977 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/query/BrowserLogQueryEs7DAO.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch7.query; + +import com.google.common.base.Strings; +import java.io.IOException; +import org.apache.skywalking.apm.util.StringUtil; +import org.apache.skywalking.oap.server.core.browser.manual.errorlog.BrowserErrorLogRecord; +import org.apache.skywalking.oap.server.core.browser.source.BrowserErrorCategory; +import org.apache.skywalking.oap.server.core.query.type.BrowserErrorLog; +import org.apache.skywalking.oap.server.core.query.type.BrowserErrorLogs; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.MatchCNameBuilder; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.BrowserLogQueryEsDAO; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +import static java.util.Objects.nonNull; + +public class BrowserLogQueryEs7DAO extends BrowserLogQueryEsDAO { + public BrowserLogQueryEs7DAO(final ElasticSearchClient client) { + super(client); + } + + @Override + public BrowserErrorLogs queryBrowserErrorLogs(final String serviceId, + final String serviceVersionId, + final String pagePathId, + final String pagePath, + final BrowserErrorCategory category, + final long startSecondTB, + final long endSecondTB, + final int limit, + final int from) throws IOException { + SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource(); + + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + sourceBuilder.query(boolQueryBuilder); + + if (startSecondTB != 0 && endSecondTB != 0) { + boolQueryBuilder.must().add( + QueryBuilders.rangeQuery(BrowserErrorLogRecord.TIME_BUCKET).gte(startSecondTB).lte(endSecondTB)); + } + + if (!Strings.isNullOrEmpty(pagePath)) { + String matchCName = MatchCNameBuilder.INSTANCE.build(BrowserErrorLogRecord.PAGE_PATH); + boolQueryBuilder.must().add(QueryBuilders.matchPhraseQuery(matchCName, pagePath)); + } + if (StringUtil.isNotEmpty(serviceId)) { + boolQueryBuilder.must().add(QueryBuilders.termQuery(BrowserErrorLogRecord.SERVICE_ID, serviceId)); + } + if (StringUtil.isNotEmpty(serviceVersionId)) { + boolQueryBuilder.must() + .add(QueryBuilders.termQuery(BrowserErrorLogRecord.SERVICE_VERSION_ID, serviceVersionId)); + } + if (StringUtil.isNotEmpty(pagePathId)) { + boolQueryBuilder.must().add(QueryBuilders.termQuery(BrowserErrorLogRecord.PAGE_PATH_ID, pagePathId)); + } + if (nonNull(category)) { + boolQueryBuilder.must() + .add(QueryBuilders.termQuery(BrowserErrorLogRecord.ERROR_CATEGORY, category.getValue())); + } + sourceBuilder.size(limit); + sourceBuilder.from(from); + + SearchResponse response = getClient().search(BrowserErrorLogRecord.INDEX_NAME, sourceBuilder); + + BrowserErrorLogs logs = new BrowserErrorLogs(); + logs.setTotal((int) response.getHits().getTotalHits().value); + + for (SearchHit searchHit : response.getHits().getHits()) { + String dataBinaryBase64 = (String) searchHit.getSourceAsMap().get(BrowserErrorLogRecord.DATA_BINARY); + if (nonNull(dataBinaryBase64)) { + BrowserErrorLog log = parserDataBinary(dataBinaryBase64); + logs.getLogs().add(log); + } + } + return logs; + } +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxStorageProvider.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxStorageProvider.java index 13d7edab1..c4e32e243 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxStorageProvider.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxStorageProvider.java @@ -33,6 +33,7 @@ import org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskQueryDA import org.apache.skywalking.oap.server.core.storage.profile.IProfileThreadSnapshotQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IAggregationQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IAlarmQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IBrowserLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.ILogQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IMetadataQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IMetricsQueryDAO; @@ -49,6 +50,7 @@ import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.HistoryDele import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.InfluxStorageDAO; import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.AggregationQuery; import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.AlarmQuery; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.BrowserLogQuery; import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.LogQuery; import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.MetadataQuery; import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.MetricsQuery; @@ -98,6 +100,7 @@ public class InfluxStorageProvider extends ModuleProvider { this.registerServiceImplementation(ITopologyQueryDAO.class, new TopologyQuery(client)); this.registerServiceImplementation(IMetricsQueryDAO.class, new MetricsQuery(client)); this.registerServiceImplementation(ITraceQueryDAO.class, new TraceQuery(client)); + this.registerServiceImplementation(IBrowserLogQueryDAO.class, new BrowserLogQuery(client)); this.registerServiceImplementation(IAggregationQueryDAO.class, new AggregationQuery(client)); this.registerServiceImplementation(IAlarmQueryDAO.class, new AlarmQuery(client)); this.registerServiceImplementation(ITopNRecordsQueryDAO.class, new TopNRecordsQuery(client)); @@ -105,12 +108,12 @@ public class InfluxStorageProvider extends ModuleProvider { this.registerServiceImplementation(IProfileTaskQueryDAO.class, new ProfileTaskQuery(client)); this.registerServiceImplementation( - IProfileThreadSnapshotQueryDAO.class, new ProfileThreadSnapshotQuery(client)); + IProfileThreadSnapshotQueryDAO.class, new ProfileThreadSnapshotQuery(client)); this.registerServiceImplementation( - IProfileTaskLogQueryDAO.class, new ProfileTaskLogQuery(client, config.getFetchTaskLogMaxSize())); + IProfileTaskLogQueryDAO.class, new ProfileTaskLogQuery(client, config.getFetchTaskLogMaxSize())); this.registerServiceImplementation( - IHistoryDeleteDAO.class, new HistoryDeleteDAO(client)); + IHistoryDeleteDAO.class, new HistoryDeleteDAO(client)); this.registerServiceImplementation(UITemplateManagementDAO.class, new UITemplateManagementDAOImpl(client)); } @@ -133,6 +136,6 @@ public class InfluxStorageProvider extends ModuleProvider { @Override public String[] requiredModules() { - return new String[]{CoreModule.NAME}; + return new String[] {CoreModule.NAME}; } } diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/BrowserLogQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/BrowserLogQuery.java new file mode 100644 index 000000000..7d1fa1b74 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/BrowserLogQuery.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.query; + +import java.io.IOException; +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.apm.util.StringUtil; +import org.apache.skywalking.oap.server.core.browser.manual.errorlog.BrowserErrorLogRecord; +import org.apache.skywalking.oap.server.core.browser.source.BrowserErrorCategory; +import org.apache.skywalking.oap.server.core.query.type.BrowserErrorLog; +import org.apache.skywalking.oap.server.core.query.type.BrowserErrorLogs; +import org.apache.skywalking.oap.server.core.storage.query.IBrowserLogQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants; +import org.influxdb.dto.Query; +import org.influxdb.dto.QueryResult; +import org.influxdb.querybuilder.SelectQueryImpl; +import org.influxdb.querybuilder.WhereQueryImpl; + +import static java.util.Objects.nonNull; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.contains; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.eq; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.gte; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.lte; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.select; + +@Slf4j +@RequiredArgsConstructor +public class BrowserLogQuery implements IBrowserLogQueryDAO { + private final InfluxClient client; + + @Override + public BrowserErrorLogs queryBrowserErrorLogs(final String serviceId, + final String serviceVersionId, + final String pagePathId, + final String pagePath, + final BrowserErrorCategory category, + final long startSecondTB, + final long endSecondTB, + final int limit, + final int from) throws IOException { + + WhereQueryImpl recallQuery = select() + .function(InfluxConstants.SORT_DES, BrowserErrorLogRecord.TIMESTAMP, limit + from) + .column(BrowserErrorLogRecord.DATA_BINARY) + .from( + client.getDatabase(), + BrowserErrorLogRecord.INDEX_NAME + ).where(); + if (startSecondTB != 0 && endSecondTB != 0) { + recallQuery.and(gte(BrowserErrorLogRecord.TIME_BUCKET, startSecondTB)) + .and(lte(BrowserErrorLogRecord.TIME_BUCKET, endSecondTB)); + } + if (StringUtil.isNotEmpty(serviceId)) { + recallQuery.and(eq(BrowserErrorLogRecord.SERVICE_ID, serviceId)); + } + if (StringUtil.isNotEmpty(serviceVersionId)) { + recallQuery.and(eq(BrowserErrorLogRecord.SERVICE_VERSION_ID, serviceVersionId)); + } + if (StringUtil.isNotEmpty(pagePathId)) { + recallQuery.and(eq(BrowserErrorLogRecord.PAGE_PATH_ID, pagePathId)); + } + if (nonNull(category)) { + recallQuery.and(eq(BrowserErrorLogRecord.ERROR_CATEGORY, category.getValue())); + } + if (StringUtil.isNotEmpty(pagePath)) { + recallQuery.and(contains(BrowserErrorLogRecord.PAGE_PATH, pagePath.replaceAll("/", "\\\\/"))); + } + + WhereQueryImpl countQuery = select() + .count(BrowserErrorLogRecord.SERVICE_ID) + .from(client.getDatabase(), BrowserErrorLogRecord.INDEX_NAME) + .where(); + recallQuery.getClauses().forEach(countQuery::where); + + Query query = new Query(countQuery.getCommand() + recallQuery.getCommand()); + + List results = client.query(query); + if (log.isDebugEnabled()) { + log.debug("SQL: {} result set: {}", query.getCommand(), results); + } + if (results.size() != 2) { + throw new IOException("Expecting to get 2 Results, but it is " + results.size()); + } + + List counter = results.get(0).getSeries(); + List result = results.get(1).getSeries(); + if (result == null || result.isEmpty()) { + return new BrowserErrorLogs(); + } + + BrowserErrorLogs logs = new BrowserErrorLogs(); + logs.setTotal(((Number) counter.get(0).getValues().get(0).get(1)).intValue()); + + result.get(0).getValues().stream().sorted((a, b) -> { + // Have to re-sort here. Because the function, top()/bottom(), get the result ordered by the `time`. + return Long.compare(((Number) b.get(1)).longValue(), ((Number) a.get(1)).longValue()); + }).skip(from).forEach(values -> { + String dataBinaryBase64 = (String) values.get(2); + if (nonNull(dataBinaryBase64)) { + BrowserErrorLog log = parserDataBinary(dataBinaryBase64); + logs.getLogs().add(log); + } + }); + return logs; + } +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java index 6b0d83828..1ff91a10e 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java @@ -83,7 +83,7 @@ public class MetadataQuery implements IMetadataQueryDAO { public List getAllBrowserServices(long startTimestamp, long endTimestamp) throws IOException { WhereQueryImpl query = select(ID_COLUMN, NAME) .from(client.getDatabase(), ServiceTraffic.INDEX_NAME) - .where(eq(InfluxConstants.TagName.NODE_TYPE, String.valueOf(NodeType.Normal.value()))); + .where(eq(InfluxConstants.TagName.NODE_TYPE, String.valueOf(NodeType.Browser.value()))); return buildServices(query); } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/H2StorageProvider.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/H2StorageProvider.java index 82db55355..b6f82ce81 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/H2StorageProvider.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/H2StorageProvider.java @@ -36,6 +36,7 @@ import org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskQueryDA import org.apache.skywalking.oap.server.core.storage.profile.IProfileThreadSnapshotQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IAggregationQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IAlarmQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IBrowserLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.ILogQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IMetadataQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IMetricsQueryDAO; @@ -51,6 +52,7 @@ import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedExcepti import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2AggregationQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2AlarmQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2BatchDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2BrowserLogQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2HistoryDeleteDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2LogQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2MetadataQueryDAO; @@ -129,6 +131,7 @@ public class H2StorageProvider extends ModuleProvider { config.getMaxSizeOfArrayColumn(), config.getNumOfSearchableValuesPerTag() )); + this.registerServiceImplementation(IBrowserLogQueryDAO.class, new H2BrowserLogQueryDAO(h2Client)); this.registerServiceImplementation( IMetadataQueryDAO.class, new H2MetadataQueryDAO(h2Client, config.getMetadataQueryMaxSize())); this.registerServiceImplementation(IAggregationQueryDAO.class, new H2AggregationQueryDAO(h2Client)); diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2BrowserLogQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2BrowserLogQueryDAO.java new file mode 100644 index 000000000..909bf5e38 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2BrowserLogQueryDAO.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao; + +import com.google.common.base.Strings; +import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import lombok.RequiredArgsConstructor; +import org.apache.skywalking.apm.util.StringUtil; +import org.apache.skywalking.oap.server.core.browser.manual.errorlog.BrowserErrorLogRecord; +import org.apache.skywalking.oap.server.core.browser.source.BrowserErrorCategory; +import org.apache.skywalking.oap.server.core.query.type.BrowserErrorLog; +import org.apache.skywalking.oap.server.core.query.type.BrowserErrorLogs; +import org.apache.skywalking.oap.server.core.storage.query.IBrowserLogQueryDAO; +import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCHikariCPClient; + +import static java.util.Objects.nonNull; + +@RequiredArgsConstructor +public class H2BrowserLogQueryDAO implements IBrowserLogQueryDAO { + private final JDBCHikariCPClient h2Client; + + @Override + public BrowserErrorLogs queryBrowserErrorLogs(String serviceId, + String serviceVersionId, + String pagePathId, + String pagePath, + BrowserErrorCategory category, + long startSecondTB, + long endSecondTB, + int limit, + int from) throws IOException { + StringBuilder sql = new StringBuilder(); + + List parameters = new ArrayList<>(9); + + sql.append("from ").append(BrowserErrorLogRecord.INDEX_NAME) + .append(" where ").append(" 1=1 "); + + if (startSecondTB != 0 && endSecondTB != 0) { + sql.append(" and ").append(BrowserErrorLogRecord.TIME_BUCKET).append(" >= ?"); + parameters.add(startSecondTB); + sql.append(" and ").append(BrowserErrorLogRecord.TIME_BUCKET).append(" <= ?"); + parameters.add(endSecondTB); + } + + if (StringUtil.isNotEmpty(serviceId)) { + sql.append(" and ").append(BrowserErrorLogRecord.SERVICE_ID).append(" = ?"); + parameters.add(serviceId); + } + if (StringUtil.isNotEmpty(serviceVersionId)) { + sql.append(" and ").append(BrowserErrorLogRecord.SERVICE_VERSION_ID).append(" = ?"); + parameters.add(serviceVersionId); + } + if (StringUtil.isNotEmpty(pagePathId)) { + sql.append(" and ").append(BrowserErrorLogRecord.PAGE_PATH_ID).append(" = ?"); + parameters.add(pagePathId); + } + if (nonNull(category)) { + sql.append(" and ").append(BrowserErrorLogRecord.ERROR_CATEGORY).append(" = ?"); + parameters.add(category.getValue()); + } + if (!Strings.isNullOrEmpty(pagePath)) { + sql.append(" and ").append(BrowserErrorLogRecord.PAGE_PATH).append(" like concat('%',?,'%')"); + parameters.add(pagePath); + } + + BrowserErrorLogs logs = new BrowserErrorLogs(); + try (Connection connection = h2Client.getConnection()) { + + try (ResultSet resultSet = h2Client.executeQuery(connection, buildCountStatement(sql.toString()), parameters + .toArray(new Object[0]))) { + while (resultSet.next()) { + logs.setTotal(resultSet.getInt("total")); + } + } + + buildLimit(sql, from, limit); + + try (ResultSet resultSet = h2Client.executeQuery( + connection, "select " + BrowserErrorLogRecord.DATA_BINARY + " " + sql.toString(), + parameters.toArray(new Object[0]) + )) { + while (resultSet.next()) { + String dataBinaryBase64 = resultSet.getString(BrowserErrorLogRecord.DATA_BINARY); + if (nonNull(dataBinaryBase64)) { + BrowserErrorLog log = parserDataBinary(dataBinaryBase64); + logs.getLogs().add(log); + } + } + } + } catch (SQLException e) { + throw new IOException(e); + } + return logs; + } + + protected String buildCountStatement(String sql) { + return "select count(1) total from (select 1 " + sql + " )"; + } + + protected void buildLimit(StringBuilder sql, int from, int limit) { + sql.append(" limit ").append(limit); + sql.append(" offset ").append(from); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageProvider.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageProvider.java index 765c252be..6c3f502ea 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageProvider.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageProvider.java @@ -35,6 +35,7 @@ import org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskQueryDA import org.apache.skywalking.oap.server.core.storage.profile.IProfileThreadSnapshotQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IAggregationQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IAlarmQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IBrowserLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.ILogQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IMetadataQueryDAO; import org.apache.skywalking.oap.server.core.storage.query.IMetricsQueryDAO; @@ -117,6 +118,7 @@ public class MySQLStorageProvider extends ModuleProvider { config.getNumOfSearchableValuesPerTag() ) ); + this.registerServiceImplementation(IBrowserLogQueryDAO.class, new MysqlBrowserLogQueryDAO(mysqlClient)); this.registerServiceImplementation( IMetadataQueryDAO.class, new H2MetadataQueryDAO(mysqlClient, config.getMetadataQueryMaxSize())); this.registerServiceImplementation(IAggregationQueryDAO.class, new MySQLAggregationQueryDAO(mysqlClient)); diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MysqlBrowserLogQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MysqlBrowserLogQueryDAO.java new file mode 100644 index 000000000..b710a9c4b --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MysqlBrowserLogQueryDAO.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql; + +import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCHikariCPClient; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2BrowserLogQueryDAO; + +public class MysqlBrowserLogQueryDAO extends H2BrowserLogQueryDAO { + public MysqlBrowserLogQueryDAO(final JDBCHikariCPClient h2Client) { + super(h2Client); + } + + @Override + protected String buildCountStatement(String sql) { + return "select count(1) total " + sql; + } + + @Override + protected void buildLimit(StringBuilder sql, int from, int limit) { + sql.append(" limit ").append(from).append(", ").append(limit); + } +} diff --git a/oap-server/server-tools/profile-exporter/tool-profile-snapshot-server-mock/src/main/java/org/apache/skywalking/oap/server/tool/profile/core/MockCoreModuleProvider.java b/oap-server/server-tools/profile-exporter/tool-profile-snapshot-server-mock/src/main/java/org/apache/skywalking/oap/server/tool/profile/core/MockCoreModuleProvider.java index 71f1d9ad6..abd983988 100755 --- a/oap-server/server-tools/profile-exporter/tool-profile-snapshot-server-mock/src/main/java/org/apache/skywalking/oap/server/tool/profile/core/MockCoreModuleProvider.java +++ b/oap-server/server-tools/profile-exporter/tool-profile-snapshot-server-mock/src/main/java/org/apache/skywalking/oap/server/tool/profile/core/MockCoreModuleProvider.java @@ -38,6 +38,7 @@ import org.apache.skywalking.oap.server.core.oal.rt.OALEngineLoaderService; import org.apache.skywalking.oap.server.core.profile.ProfileTaskMutationService; import org.apache.skywalking.oap.server.core.query.AggregationQueryService; import org.apache.skywalking.oap.server.core.query.AlarmQueryService; +import org.apache.skywalking.oap.server.core.query.BrowserLogQueryService; import org.apache.skywalking.oap.server.core.query.LogQueryService; import org.apache.skywalking.oap.server.core.query.MetadataQueryService; import org.apache.skywalking.oap.server.core.query.MetricsMetadataQueryService; @@ -146,6 +147,7 @@ public class MockCoreModuleProvider extends CoreModuleProvider { this.registerServiceImplementation(MetricsMetadataQueryService.class, new MetricsMetadataQueryService()); this.registerServiceImplementation(MetricsQueryService.class, new MetricsQueryService(getManager())); this.registerServiceImplementation(TraceQueryService.class, new TraceQueryService(getManager())); + this.registerServiceImplementation(BrowserLogQueryService.class, new BrowserLogQueryService(getManager())); this.registerServiceImplementation(LogQueryService.class, new LogQueryService(getManager())); this.registerServiceImplementation(MetadataQueryService.class, new MetadataQueryService(getManager())); this.registerServiceImplementation(AggregationQueryService.class, new AggregationQueryService(getManager())); diff --git a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/SimpleQueryClient.java b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/SimpleQueryClient.java index 5287dc316..f2ef797ae 100644 --- a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/SimpleQueryClient.java +++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/SimpleQueryClient.java @@ -25,14 +25,18 @@ import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.e2e.browser.BrowserErrorLog; +import org.apache.skywalking.e2e.browser.BrowserErrorLogQuery; +import org.apache.skywalking.e2e.browser.BrowserErrorLogsData; import org.apache.skywalking.e2e.metrics.Metrics; import org.apache.skywalking.e2e.metrics.MetricsData; import org.apache.skywalking.e2e.metrics.MetricsQuery; import org.apache.skywalking.e2e.metrics.MultiMetricsData; import org.apache.skywalking.e2e.metrics.ReadLabeledMetricsData; -import org.apache.skywalking.e2e.metrics.ReadMetricsQuery; import org.apache.skywalking.e2e.metrics.ReadMetrics; import org.apache.skywalking.e2e.metrics.ReadMetricsData; +import org.apache.skywalking.e2e.metrics.ReadMetricsQuery; import org.apache.skywalking.e2e.service.Service; import org.apache.skywalking.e2e.service.ServicesData; import org.apache.skywalking.e2e.service.ServicesQuery; @@ -43,8 +47,8 @@ import org.apache.skywalking.e2e.service.instance.InstancesQuery; import org.apache.skywalking.e2e.topo.ServiceInstanceTopology; import org.apache.skywalking.e2e.topo.ServiceInstanceTopologyQuery; import org.apache.skywalking.e2e.topo.ServiceInstanceTopologyResponse; -import org.apache.skywalking.e2e.topo.Topology; import org.apache.skywalking.e2e.topo.TopoQuery; +import org.apache.skywalking.e2e.topo.Topology; import org.apache.skywalking.e2e.topo.TopologyResponse; import org.apache.skywalking.e2e.trace.Trace; import org.apache.skywalking.e2e.trace.TracesData; @@ -56,8 +60,6 @@ import org.springframework.http.RequestEntity; import org.springframework.http.ResponseEntity; import org.springframework.web.client.RestTemplate; -import lombok.extern.slf4j.Slf4j; - @SuppressWarnings("UnstableApiUsage") @Slf4j public class SimpleQueryClient { @@ -100,8 +102,41 @@ public class SimpleQueryClient { return Objects.requireNonNull(responseEntity.getBody()).getData().getTraces().getData(); } + public List browserErrorLogs(final BrowserErrorLogQuery query) throws Exception { + final URL queryFileUrl = Resources.getResource("browser-error-logs.gql"); + final String queryString = Resources.readLines(queryFileUrl, StandardCharsets.UTF_8) + .stream() + .filter(it -> !it.startsWith("#")) + .collect(Collectors.joining()) + .replace("{start}", query.start()) + .replace("{end}", query.end()) + .replace("{step}", query.step()) + .replace("{pageNum}", query.pageNum()) + .replace("{pageSize}", query.pageSize()) + .replace("{needTotal}", query.needTotal()); + final ResponseEntity> responseEntity = restTemplate.exchange( + new RequestEntity<>(queryString, HttpMethod.POST, URI.create(endpointUrl)), + new ParameterizedTypeReference>() { + } + ); + + if (responseEntity.getStatusCode() != HttpStatus.OK) { + throw new RuntimeException("Response status != 200, actual: " + responseEntity.getStatusCode()); + } + + return Objects.requireNonNull(responseEntity.getBody().getData().getLogs().getData()); + } + public List services(final ServicesQuery query) throws Exception { - final URL queryFileUrl = Resources.getResource("services.gql"); + return services(query, "services.gql"); + } + + public List browserServices(final ServicesQuery query) throws Exception { + return services(query, "browser-services.gql"); + } + + private List services(final ServicesQuery query, String gql) throws Exception { + final URL queryFileUrl = Resources.getResource(gql); final String queryString = Resources.readLines(queryFileUrl, StandardCharsets.UTF_8) .stream() .filter(it -> !it.startsWith("#")) diff --git a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/browser/BrowserErrorLog.java b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/browser/BrowserErrorLog.java new file mode 100644 index 000000000..fea3b503c --- /dev/null +++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/browser/BrowserErrorLog.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.e2e.browser; + +import lombok.Data; +import lombok.experimental.Accessors; + +@Data +@Accessors(chain = true) +public class BrowserErrorLog { + private String service; + private String serviceVersion; + private Long time; + private String pagePath; + private String category; + private String grade; + private String message; + private Integer line; + private Integer col; + private String stack; + private String errorUrl; + private boolean firstReportedError; +} diff --git a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/browser/BrowserErrorLogMatcher.java b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/browser/BrowserErrorLogMatcher.java new file mode 100644 index 000000000..1ac95478e --- /dev/null +++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/browser/BrowserErrorLogMatcher.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.e2e.browser; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.apache.skywalking.e2e.verification.AbstractMatcher; + +import static java.util.Objects.nonNull; + +@Data +@ToString(callSuper = true) +@EqualsAndHashCode(callSuper = true) +public class BrowserErrorLogMatcher extends AbstractMatcher { + private String service; + private String serviceVersion; + private String time; + private String pagePath; + private String category; + private String grade; + private String message; + private String line; + private String col; + private String stack; + private String errorUrl; + private String firstReportedError; + + @Override + public void verify(final BrowserErrorLog log) { + if (nonNull(getService())) { + doVerify(getService(), log.getService()); + } + + if (nonNull(getServiceVersion())) { + doVerify(getServiceVersion(), log.getServiceVersion()); + } + + if (nonNull(getTime())) { + doVerify(getTime(), String.valueOf(log.getTime())); + } + + if (nonNull(getPagePath())) { + doVerify(getPagePath(), log.getPagePath()); + } + + if (nonNull(getCategory())) { + doVerify(getCategory(), log.getCategory()); + } + + if (nonNull(getGrade())) { + doVerify(getGrade(), log.getGrade()); + } + + if (nonNull(getMessage())) { + doVerify(getMessage(), log.getMessage()); + } + + if (nonNull(getLine())) { + doVerify(getLine(), String.valueOf(log.getLine())); + } + + if (nonNull(getCol())) { + doVerify(getCol(), String.valueOf(log.getCol())); + } + + if (nonNull(getStack())) { + doVerify(getStack(), log.getStack()); + } + + if (nonNull(getErrorUrl())) { + doVerify(getErrorUrl(), log.getErrorUrl()); + } + + if (nonNull(getFirstReportedError())) { + doVerify(getFirstReportedError(), String.valueOf(log.isFirstReportedError())); + } + } +} diff --git a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/browser/BrowserErrorLogQuery.java b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/browser/BrowserErrorLogQuery.java new file mode 100644 index 000000000..4cdfda28c --- /dev/null +++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/browser/BrowserErrorLogQuery.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.e2e.browser; + +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; +import org.apache.skywalking.e2e.AbstractQuery; + +@Setter +@Getter +@Accessors(chain = true) +public class BrowserErrorLogQuery extends AbstractQuery { + private String pageNum = "1"; + private String pageSize = "15"; + private String needTotal = "true"; + + public String pageNum() { + return pageNum; + } + + public String pageSize() { + return pageSize; + } + + public String needTotal() { + return needTotal; + } +} diff --git a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/browser/BrowserErrorLogs.java b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/browser/BrowserErrorLogs.java new file mode 100644 index 000000000..4997e7699 --- /dev/null +++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/browser/BrowserErrorLogs.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.e2e.browser; + +import java.util.List; +import lombok.Data; + +@Data +public class BrowserErrorLogs { + private List data; +} diff --git a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/browser/BrowserErrorLogsData.java b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/browser/BrowserErrorLogsData.java new file mode 100644 index 000000000..d21d0ffce --- /dev/null +++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/browser/BrowserErrorLogsData.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.e2e.browser; + +import lombok.Data; + +@Data +public class BrowserErrorLogsData { + private BrowserErrorLogs logs; +} diff --git a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/browser/BrowserErrorLogsMatcher.java b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/browser/BrowserErrorLogsMatcher.java new file mode 100644 index 000000000..c3871ed6d --- /dev/null +++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/browser/BrowserErrorLogsMatcher.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.e2e.browser; + +import java.util.LinkedList; +import java.util.List; +import lombok.Data; + +import static org.assertj.core.api.Assertions.fail; + +@Data +public class BrowserErrorLogsMatcher { + private List logs; + + public BrowserErrorLogsMatcher() { + this.logs = new LinkedList<>(); + } + + public void verifyLoosely(final List logs) { + for (final BrowserErrorLogMatcher matcher : getLogs()) { + boolean matched = false; + for (final BrowserErrorLog log : logs) { + try { + matcher.verify(log); + matched = true; + } catch (Throwable throwable) { + + } + } + if (!matched) { + fail("\nExpected: %s\n Actual: %s", getLogs(), logs); + } + } + } +} diff --git a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/BrowserMetricsQuery.java b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/BrowserMetricsQuery.java new file mode 100644 index 000000000..a8c910ee1 --- /dev/null +++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/BrowserMetricsQuery.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.e2e.metrics; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; +import org.apache.skywalking.e2e.AbstractQuery; + +@Data +@Accessors(fluent = true) +@EqualsAndHashCode(callSuper = true) +public class BrowserMetricsQuery extends AbstractQuery { + public static final String BROWSER_APP_PV = "browser_app_pv"; + public static final String BROWSER_APP_ERROR_RATE = "browser_app_error_rate"; + public static final String BROWSER_APP_ERROR_SUM = "browser_app_error_sum"; + + public static final String[] ALL_BROWSER_METRICS = { + BROWSER_APP_PV, + BROWSER_APP_ERROR_SUM, + BROWSER_APP_ERROR_RATE + }; + + public static final String BROWSER_APP_SINGLE_VERSION_PV = "browser_app_single_version_pv"; + public static final String BROWSER_APP_SINGLE_VERSION_ERROR_RATE = "browser_app_single_version_error_rate"; + public static final String BROWSER_APP_SINGLE_VERSION_ERROR_SUM = "browser_app_single_version_error_sum"; + + public static final String[] ALL_BROWSER_SINGLE_VERSION_METRICS = { + BROWSER_APP_SINGLE_VERSION_PV, + BROWSER_APP_SINGLE_VERSION_ERROR_RATE, + BROWSER_APP_SINGLE_VERSION_ERROR_SUM + }; + + public static final String BROWSER_APP_PAGE_PV = "browser_app_page_pv"; + public static final String BROWSER_APP_PAGE_ERROR_RATE = "browser_app_page_error_rate"; + public static final String BROWSER_APP_PAGE_ERROR_SUM = "browser_app_page_error_sum"; + + public static final String BROWSER_APP_PAGE_AJAX_ERROR_SUM = "browser_app_page_ajax_error_sum"; + public static final String BROWSER_APP_PAGE_RESOURCE_ERROR_SUM = "browser_app_page_resource_error_sum"; + public static final String BROWSER_APP_PAGE_JS_ERROR_SUM = "browser_app_page_js_error_sum"; + public static final String BROWSER_APP_PAGE_UNKNOWN_ERROR_SUM = "browser_app_page_unknown_error_sum"; + + public static final String BROWSER_APP_PAGE_REDIRECT_AVG = "browser_app_page_redirect_avg"; + public static final String BROWSER_APP_PAGE_DNS_AVG = "browser_app_page_dns_avg"; + public static final String BROWSER_APP_PAGE_TTFB_AVG = "browser_app_page_ttfb_avg"; + public static final String BROWSER_APP_PAGE_TCP_AVG = "browser_app_page_tcp_avg"; + public static final String BROWSER_APP_PAGE_TRANS_AVG = "browser_app_page_trans_avg"; + public static final String BROWSER_APP_PAGE_DOM_ANALYSIS_AVG = "browser_app_page_dom_analysis_avg"; + public static final String BROWSER_APP_PAGE_FPT_AVG = "browser_app_page_fpt_avg"; + public static final String BROWSER_APP_PAGE_DOM_READY_AVG = "browser_app_page_dom_ready_avg"; + public static final String BROWSER_APP_PAGE_LOAD_PAGE_AVG = "browser_app_page_load_page_avg"; + public static final String BROWSER_APP_PAGE_RES_AVG = "browser_app_page_res_avg"; + public static final String BROWSER_APP_PAGE_SSL_AVG = "browser_app_page_ssl_avg"; + public static final String BROWSER_APP_PAGE_TTL_AVG = "browser_app_page_ttl_avg"; + public static final String BROWSER_APP_PAGE_FIRST_PACK_AVG = "browser_app_page_first_pack_avg"; + public static final String BROWSER_APP_PAGE_FMP_AVG = "browser_app_page_fmp_avg"; + + public static final String[] ALL_BROWSER_PAGE_METRICS = { + BROWSER_APP_PAGE_PV, + BROWSER_APP_PAGE_ERROR_RATE, + BROWSER_APP_PAGE_ERROR_SUM, + BROWSER_APP_PAGE_AJAX_ERROR_SUM, + BROWSER_APP_PAGE_RESOURCE_ERROR_SUM, + BROWSER_APP_PAGE_JS_ERROR_SUM, + BROWSER_APP_PAGE_UNKNOWN_ERROR_SUM, + BROWSER_APP_PAGE_REDIRECT_AVG, + BROWSER_APP_PAGE_DNS_AVG, + BROWSER_APP_PAGE_TTFB_AVG, + BROWSER_APP_PAGE_TCP_AVG, + BROWSER_APP_PAGE_TRANS_AVG, + BROWSER_APP_PAGE_DOM_ANALYSIS_AVG, + BROWSER_APP_PAGE_FPT_AVG, + BROWSER_APP_PAGE_DOM_READY_AVG, + BROWSER_APP_PAGE_LOAD_PAGE_AVG, + BROWSER_APP_PAGE_RES_AVG, + BROWSER_APP_PAGE_SSL_AVG, + BROWSER_APP_PAGE_TTL_AVG, + BROWSER_APP_PAGE_FIRST_PACK_AVG, + BROWSER_APP_PAGE_FMP_AVG + }; + + public static final String BROWSER_APP_PAGE_FPT_PERCENTILE = "browser_app_page_fpt_percentile"; + public static final String BROWSER_APP_PAGE_DOM_READY_PERCENTILE = "browser_app_page_dom_ready_percentile"; + public static final String BROWSER_APP_PAGE_LOAD_PAGE_PERCENTILE = "browser_app_page_load_page_percentile"; + public static final String BROWSER_APP_PAGE_TTL_PERCENTILE = "browser_app_page_ttl_percentile"; + public static final String BROWSER_APP_PAGE_FIRST_PACK_PERCENTILE = "browser_app_page_first_pack_percentile"; + public static final String BROWSER_APP_PAGE_FMP_PERCENTILE = "browser_app_page_fmp_percentile"; + + public static final String[] ALL_BROWSER_PAGE_MULTIPLE_LINEAR_METRICS = { + BROWSER_APP_PAGE_FPT_PERCENTILE, + BROWSER_APP_PAGE_TTL_PERCENTILE, + BROWSER_APP_PAGE_DOM_READY_PERCENTILE, + BROWSER_APP_PAGE_LOAD_PAGE_PERCENTILE, + BROWSER_APP_PAGE_FIRST_PACK_PERCENTILE, + BROWSER_APP_PAGE_FMP_PERCENTILE + }; +} diff --git a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsMatcher.java b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsMatcher.java index fdb8ac5b6..0ae938c88 100644 --- a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsMatcher.java +++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsMatcher.java @@ -19,10 +19,10 @@ package org.apache.skywalking.e2e.metrics; import java.time.LocalDateTime; -import java.time.ZoneOffset; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.e2e.SimpleQueryClient; +import org.apache.skywalking.e2e.utils.Times; @Slf4j public class MetricsMatcher { @@ -46,7 +46,7 @@ public class MetricsMatcher { new MetricsQuery().stepByMinute() .metricsName(metricName) .start(minutesAgo) - .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1)) + .end(Times.now().plusMinutes(1)) .id(id)); LOGGER.info("{}: {}", metricName, metrics); AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher(); @@ -67,6 +67,13 @@ public class MetricsMatcher { } } + public static void verifyPercentileMetrics(final SimpleQueryClient queryClient, + final String metricName, + final String id, + final LocalDateTime minutesAgo) throws Exception { + verifyPercentileMetrics(queryClient, metricName, id, minutesAgo, 0, null); + } + public static void verifyPercentileMetrics(final SimpleQueryClient queryClient, final String metricName, final String id, @@ -79,7 +86,7 @@ public class MetricsMatcher { new MetricsQuery().stepByMinute() .metricsName(metricName) .start(minutesAgo) - .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1)) + .end(Times.now().plusMinutes(1)) .id(id), "5"); LOGGER.info("{}: {}", metricName, metricsArray); AtLeastOneOfMetricsMatcher matcher = new AtLeastOneOfMetricsMatcher(); diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/utils/Times.java b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/utils/Times.java similarity index 94% rename from test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/utils/Times.java rename to test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/utils/Times.java index 6c35a883a..71b922122 100644 --- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/utils/Times.java +++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/utils/Times.java @@ -6,14 +6,13 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package org.apache.skywalking.e2e.utils; diff --git a/test/e2e/e2e-data/src/main/resources/browser-error-logs.gql b/test/e2e/e2e-data/src/main/resources/browser-error-logs.gql new file mode 100644 index 000000000..912eeb7ea --- /dev/null +++ b/test/e2e/e2e-data/src/main/resources/browser-error-logs.gql @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{ + "query": "query queryBrowserLog($condition: BrowserErrorLogQueryCondition) { + logs: queryBrowserErrorLogs(condition: $condition) { + data: logs { + service serviceVersion time pagePath category grade, message, line, col, stack, errorUrl, firstReportedError + } + total + } + }", + "variables": { + "condition": { + "queryDuration": { + "start": "{start}", + "end": "{end}", + "step": "{step}" + }, + "paging": { + "pageNum": {pageNum}, + "pageSize": {pageSize}, + "needTotal": {needTotal} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/e2e-data/src/main/resources/browser-services.gql b/test/e2e/e2e-data/src/main/resources/browser-services.gql new file mode 100644 index 000000000..5257d6409 --- /dev/null +++ b/test/e2e/e2e-data/src/main/resources/browser-services.gql @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{ + "query": "query queryServices($duration: Duration!) { + services: getAllBrowserServices(duration: $duration) { + key: id + label: name + } + }", + "variables": { + "duration": { + "start": "{start}", + "end": "{end}", + "step": "{step}" + } + } +} \ No newline at end of file diff --git a/test/e2e/e2e-protocol/src/main/proto b/test/e2e/e2e-protocol/src/main/proto index 7e563891f..b2f381e63 160000 --- a/test/e2e/e2e-protocol/src/main/proto +++ b/test/e2e/e2e-protocol/src/main/proto @@ -1 +1 @@ -Subproject commit 7e563891f3cfaf5c19ad086434af93aaab996719 +Subproject commit b2f381e63702bc43216ef5576637195102302c6e diff --git a/test/e2e/e2e-test/docker/browser/Dockerfile.h2 b/test/e2e/e2e-test/docker/browser/Dockerfile.h2 new file mode 100644 index 000000000..b77f407a5 --- /dev/null +++ b/test/e2e/e2e-test/docker/browser/Dockerfile.h2 @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM adoptopenjdk/openjdk8:alpine + +WORKDIR /h2 + +VOLUME /h2/data + +ADD https://repo.maven.apache.org/maven2/com/h2database/h2/1.4.200/h2-1.4.200.jar /h2 + +CMD ["sh", "-c", "java -cp /h2/*.jar org.h2.tools.Server -tcp -tcpAllowOthers -tcpPort 1521 -ifNotExists -baseDir /h2/data"] \ No newline at end of file diff --git a/test/e2e/e2e-test/docker/browser/docker-compose.es6.yml b/test/e2e/e2e-test/docker/browser/docker-compose.es6.yml new file mode 100644 index 000000000..beefbff91 --- /dev/null +++ b/test/e2e/e2e-test/docker/browser/docker-compose.es6.yml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + es: + image: elastic/elasticsearch:6.3.2 + expose: + - 9200 + networks: + - e2e + environment: + - discovery.type=single-node + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"] + interval: 5s + timeout: 60s + retries: 120 + + oap: + extends: + file: ../base-compose.yml + service: oap + environment: + SW_STORAGE: elasticsearch + depends_on: + es: + condition: service_healthy + +networks: + e2e: \ No newline at end of file diff --git a/test/e2e/e2e-test/docker/browser/docker-compose.es7.yml b/test/e2e/e2e-test/docker/browser/docker-compose.es7.yml new file mode 100644 index 000000000..28c79a7ec --- /dev/null +++ b/test/e2e/e2e-test/docker/browser/docker-compose.es7.yml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + es: + image: elastic/elasticsearch:7.4.2 + expose: + - 9200 + networks: + - e2e + environment: + - discovery.type=single-node + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9200"] + interval: 5s + timeout: 60s + retries: 120 + + oap: + extends: + file: ../base-compose.yml + service: oap-es7 + environment: + SW_STORAGE: elasticsearch7 + depends_on: + es: + condition: service_healthy + +networks: + e2e: \ No newline at end of file diff --git a/test/e2e/e2e-test/docker/browser/docker-compose.h2.yml b/test/e2e/e2e-test/docker/browser/docker-compose.h2.yml new file mode 100644 index 000000000..1c2b5364c --- /dev/null +++ b/test/e2e/e2e-test/docker/browser/docker-compose.h2.yml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + h2db: + build: + context: . + dockerfile: Dockerfile.h2 + networks: + - e2e + expose: + - 1521 + healthcheck: + test: ["CMD", "sh", "-c", "nc -z 127.0.0.1 1521"] + interval: 5s + timeout: 60s + retries: 120 + + oap: + extends: + file: ../base-compose.yml + service: oap + environment: + SW_STORAGE: h2 + SW_STORAGE_H2_URL: jdbc:h2:tcp://h2db:1521/skywalking-oap-db + depends_on: + h2db: + condition: service_healthy + +networks: + e2e: \ No newline at end of file diff --git a/test/e2e/e2e-test/docker/browser/docker-compose.influxdb.yml b/test/e2e/e2e-test/docker/browser/docker-compose.influxdb.yml new file mode 100644 index 000000000..6313f352f --- /dev/null +++ b/test/e2e/e2e-test/docker/browser/docker-compose.influxdb.yml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + influxdb: + image: influxdb:1.7.9 + expose: + - 8086 + networks: + - e2e + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/8086"] + interval: 5s + timeout: 60s + retries: 120 + + oap: + extends: + file: ../base-compose.yml + service: oap + environment: + SW_STORAGE: influxdb + depends_on: + influxdb: + condition: service_healthy + +networks: + e2e: \ No newline at end of file diff --git a/test/e2e/e2e-test/docker/browser/docker-compose.mysql.yml b/test/e2e/e2e-test/docker/browser/docker-compose.mysql.yml new file mode 100644 index 000000000..d577079f1 --- /dev/null +++ b/test/e2e/e2e-test/docker/browser/docker-compose.mysql.yml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + mysql: + image: mysql/mysql-server:8.0.13 + networks: + - e2e + expose: + - 3306 + environment: + - MYSQL_ROOT_PASSWORD=root@1234 + - MYSQL_DATABASE=swtest + - MYSQL_ROOT_HOST=% + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/3306"] + interval: 5s + timeout: 60s + retries: 120 + + oap: + extends: + file: ../base-compose.yml + service: oap + environment: + SW_STORAGE: mysql + depends_on: + mysql: + condition: service_healthy + entrypoint: ['sh', '-c', '/download-mysql.sh && /skywalking/docker-entrypoint.sh'] + +networks: + e2e: \ No newline at end of file diff --git a/test/e2e/e2e-test/docker/browser/download-mysql.sh b/test/e2e/e2e-test/docker/browser/download-mysql.sh new file mode 100644 index 000000000..f93caa499 --- /dev/null +++ b/test/e2e/e2e-test/docker/browser/download-mysql.sh @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +SW_HOME=/skywalking +MYSQL_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.13/mysql-connector-java-8.0.13.jar" +MYSQL_DRIVER="mysql-connector-java-8.0.13.jar" + +curl -L -o "${SW_HOME}/oap-libs/${MYSQL_DRIVER}" ${MYSQL_URL} +if [[ $? -ne 0 ]]; then + echo "Fail to download ${MYSQL_DRIVER}." + exit 1 +fi \ No newline at end of file diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/ClusterE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/ClusterE2E.java index 05a7704a4..e36586062 100755 --- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/ClusterE2E.java +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/ClusterE2E.java @@ -226,7 +226,7 @@ public class ClusterE2E extends SkyWalkingTestAdapter { } for (String metricName : ALL_ENDPOINT_MULTIPLE_LINEAR_METRICS) { - verifyPercentileMetrics(graphql, metricName, endpoint.getKey(), startTime, 0, null); + verifyPercentileMetrics(graphql, metricName, endpoint.getKey(), startTime); } } } @@ -238,7 +238,7 @@ public class ClusterE2E extends SkyWalkingTestAdapter { verifyMetrics(graphql, metricName, service.getKey(), startTime); } for (String metricName : ALL_SERVICE_MULTIPLE_LINEAR_METRICS) { - verifyPercentileMetrics(graphql, metricName, service.getKey(), startTime, 0, null); + verifyPercentileMetrics(graphql, metricName, service.getKey(), startTime); } } diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/base/SkyWalkingTestAdapter.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/base/SkyWalkingTestAdapter.java index 53e9d42d3..3f07569bc 100644 --- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/base/SkyWalkingTestAdapter.java +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/base/SkyWalkingTestAdapter.java @@ -21,15 +21,15 @@ package org.apache.skywalking.e2e.base; import com.google.common.collect.ImmutableMap; import java.net.URL; import java.time.LocalDateTime; -import java.time.ZoneOffset; import java.util.Map; import org.apache.skywalking.e2e.SimpleQueryClient; import org.apache.skywalking.e2e.common.HostAndPort; +import org.apache.skywalking.e2e.utils.Times; import org.springframework.web.client.RestTemplate; public abstract class SkyWalkingTestAdapter { protected final RestTemplate restTemplate = new RestTemplate(); - protected final LocalDateTime startTime = LocalDateTime.now(ZoneOffset.UTC); + protected final LocalDateTime startTime = Times.now(); protected Map trafficData = ImmutableMap.of("name", "SkyWalking"); diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/browser/BrowserE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/browser/BrowserE2E.java new file mode 100644 index 000000000..dd31599bd --- /dev/null +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/browser/BrowserE2E.java @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.e2e.browser; + +import io.grpc.ManagedChannel; +import io.grpc.internal.DnsNameResolverProvider; +import io.grpc.netty.NettyChannelBuilder; +import io.grpc.stub.StreamObserver; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.apm.network.common.v3.Commands; +import org.apache.skywalking.apm.network.language.agent.v3.BrowserPerfData; +import org.apache.skywalking.apm.network.language.agent.v3.BrowserPerfServiceGrpc; +import org.apache.skywalking.apm.network.language.agent.v3.ErrorCategory; +import org.apache.skywalking.e2e.annotation.ContainerHostAndPort; +import org.apache.skywalking.e2e.annotation.DockerCompose; +import org.apache.skywalking.e2e.base.SkyWalkingE2E; +import org.apache.skywalking.e2e.base.SkyWalkingTestAdapter; +import org.apache.skywalking.e2e.base.TrafficController; +import org.apache.skywalking.e2e.common.HostAndPort; +import org.apache.skywalking.e2e.retryable.RetryableTest; +import org.apache.skywalking.e2e.service.Service; +import org.apache.skywalking.e2e.service.ServicesMatcher; +import org.apache.skywalking.e2e.service.ServicesQuery; +import org.apache.skywalking.e2e.service.endpoint.Endpoint; +import org.apache.skywalking.e2e.service.endpoint.EndpointQuery; +import org.apache.skywalking.e2e.service.endpoint.Endpoints; +import org.apache.skywalking.e2e.service.endpoint.EndpointsMatcher; +import org.apache.skywalking.e2e.service.instance.Instance; +import org.apache.skywalking.e2e.service.instance.Instances; +import org.apache.skywalking.e2e.service.instance.InstancesMatcher; +import org.apache.skywalking.e2e.service.instance.InstancesQuery; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.containers.DockerComposeContainer; + +import static org.apache.skywalking.e2e.metrics.BrowserMetricsQuery.ALL_BROWSER_METRICS; +import static org.apache.skywalking.e2e.metrics.BrowserMetricsQuery.ALL_BROWSER_PAGE_METRICS; +import static org.apache.skywalking.e2e.metrics.BrowserMetricsQuery.ALL_BROWSER_PAGE_MULTIPLE_LINEAR_METRICS; +import static org.apache.skywalking.e2e.metrics.BrowserMetricsQuery.ALL_BROWSER_SINGLE_VERSION_METRICS; +import static org.apache.skywalking.e2e.metrics.MetricsMatcher.verifyMetrics; +import static org.apache.skywalking.e2e.metrics.MetricsMatcher.verifyPercentileMetrics; +import static org.apache.skywalking.e2e.utils.Times.now; +import static org.apache.skywalking.e2e.utils.Yamls.load; + +@Slf4j +@SkyWalkingE2E +public class BrowserE2E extends SkyWalkingTestAdapter { + private static final int MAX_INBOUND_MESSAGE_SIZE = 1024 * 1024 * 50; + + private static final String BROWSER_NAME = "e2e"; + + private static final String BROWSER_SINGLE_VERSION_NAME = "v1.0.0"; + + @SuppressWarnings("unused") + @DockerCompose({ + "docker/browser/docker-compose.${SW_STORAGE}.yml", + }) + private DockerComposeContainer justForSideEffects; + + @SuppressWarnings("unused") + @ContainerHostAndPort(name = "oap", port = 12800) + private HostAndPort swWebappHostPort; + + @SuppressWarnings("unused") + @ContainerHostAndPort(name = "oap", port = 11800) + private HostAndPort oapHostPort; + + private BrowserPerfServiceGrpc.BrowserPerfServiceStub browserPerfServiceStub; + + @BeforeAll + public void setUp() { + queryClient(swWebappHostPort); + + final ManagedChannel channel = + NettyChannelBuilder.forAddress(oapHostPort.host(), oapHostPort.port()) + .nameResolverFactory(new DnsNameResolverProvider()) + .maxInboundMessageSize(MAX_INBOUND_MESSAGE_SIZE) + .usePlaintext() + .build(); + + browserPerfServiceStub = BrowserPerfServiceGrpc.newStub(channel); + + generateTraffic(); + } + + @AfterAll + public void tearDown() { + trafficController.stop(); + } + + @RetryableTest + public void verifyBrowserData() throws Exception { + final List services = graphql.browserServices( + new ServicesQuery().start(startTime).end(now())); + LOGGER.info("services: {}", services); + + load("expected/browser/services.yml").as(ServicesMatcher.class).verify(services); + + for (Service service : services) { + LOGGER.info("verifying service version: {}", service); + // browser metrics + verifyBrowserMetrics(service); + + // browser single version + verifyBrowserSingleVersion(service); + + // browser page path + verifyBrowserPagePath(service); + } + } + + @RetryableTest + public void errorLogs() throws Exception { + List logs = graphql.browserErrorLogs(new BrowserErrorLogQuery().start(startTime).end(now())); + + LOGGER.info("errorLogs: {}", logs); + + load("expected/browser/error-log.yml").as(BrowserErrorLogsMatcher.class).verifyLoosely(logs); + } + + private void verifyBrowserMetrics(final Service service) throws Exception { + for (String metricName : ALL_BROWSER_METRICS) { + verifyMetrics(graphql, metricName, service.getKey(), startTime); + } + } + + private void verifyBrowserSingleVersion(final Service service) throws Exception { + Instances instances = graphql.instances( + new InstancesQuery().serviceId(service.getKey()).start(startTime).end(now()) + ); + LOGGER.info("instances: {}", instances); + load("expected/browser/version.yml").as(InstancesMatcher.class).verify(instances); + // service version metrics + for (Instance instance : instances.getInstances()) { + verifyBrowserSingleVersionMetrics(instance); + } + } + + private void verifyBrowserSingleVersionMetrics(Instance instance) throws Exception { + for (String metricName : ALL_BROWSER_SINGLE_VERSION_METRICS) { + verifyMetrics(graphql, metricName, instance.getKey(), startTime); + } + } + + private void verifyBrowserPagePath(final Service service) throws Exception { + Endpoints endpoints = graphql.endpoints(new EndpointQuery().serviceId(String.valueOf(service.getKey()))); + LOGGER.info("endpoints: {}", endpoints); + load("expected/browser/page-path.yml").as(EndpointsMatcher.class).verify(endpoints); + // service page metrics + for (Endpoint endpoint : endpoints.getEndpoints()) { + verifyBrowserPagePathMetrics(endpoint); + } + } + + private void verifyBrowserPagePathMetrics(Endpoint endpoint) throws Exception { + for (String metricName : ALL_BROWSER_PAGE_METRICS) { + verifyMetrics(graphql, metricName, endpoint.getKey(), startTime); + } + + for (String metricName : ALL_BROWSER_PAGE_MULTIPLE_LINEAR_METRICS) { + verifyPercentileMetrics(graphql, metricName, endpoint.getKey(), startTime); + } + } + + private void generateTraffic() { + trafficController = TrafficController.builder() + .sender(this::sendBrowserData) + .build() + .start(); + } + + private boolean sendBrowserData() { + try { + BrowserPerfData.Builder builder = BrowserPerfData.newBuilder() + .setService(BROWSER_NAME) + .setServiceVersion(BROWSER_SINGLE_VERSION_NAME) + .setPagePath("/e2e-browser") + .setRedirectTime(10) + .setDnsTime(10) + .setTtfbTime(10) + .setTcpTime(10) + .setTransTime(10) + .setDomAnalysisTime(10) + .setFptTime(10) + .setDomReadyTime(10) + .setLoadPageTime(10) + .setResTime(10) + .setSslTime(10) + .setTtlTime(10) + .setFirstPackTime(10) + .setFmpTime(10); + + sendBrowserPerfData(builder.build()); + + for (ErrorCategory category : ErrorCategory.values()) { + if (category == ErrorCategory.UNRECOGNIZED) { + continue; + } + org.apache.skywalking.apm.network.language.agent.v3.BrowserErrorLog.Builder errorLogBuilder = org.apache.skywalking.apm.network.language.agent.v3.BrowserErrorLog + .newBuilder() + .setUniqueId(UUID.randomUUID().toString()) + .setService(BROWSER_NAME) + .setServiceVersion(BROWSER_SINGLE_VERSION_NAME) + .setPagePath("/e2e-browser") + .setCategory(category) + .setMessage("test") + .setLine(1) + .setCol(1) + .setStack("e2e") + .setErrorUrl("/e2e-browser"); + if (category == ErrorCategory.js) { + errorLogBuilder.setFirstReportedError(true); + } + sendBrowserErrorLog(errorLogBuilder.build()); + } + return true; + } catch (Exception e) { + LOGGER.warn(e.getMessage(), e); + return false; + } + } + + private void sendBrowserPerfData(BrowserPerfData browserPerfData) throws InterruptedException { + final CountDownLatch latch = new CountDownLatch(1); + + browserPerfServiceStub.collectPerfData(browserPerfData, new StreamObserver() { + @Override + public void onNext(Commands commands) { + + } + + @Override + public void onError(Throwable throwable) { + throwable.printStackTrace(); + latch.countDown(); + } + + @Override + public void onCompleted() { + latch.countDown(); + } + }); + latch.await(); + } + + private void sendBrowserErrorLog(org.apache.skywalking.apm.network.language.agent.v3.BrowserErrorLog browserErrorLog) throws InterruptedException { + final CountDownLatch latch = new CountDownLatch(1); + + StreamObserver collectStream = browserPerfServiceStub + .collectErrorLogs( + new StreamObserver() { + @Override + public void onNext(Commands commands) { + + } + + @Override + public void onError(Throwable throwable) { + throwable.printStackTrace(); + latch.countDown(); + } + + @Override + public void onCompleted() { + latch.countDown(); + } + }); + collectStream.onNext(browserErrorLog); + collectStream.onCompleted(); + latch.await(); + } +} diff --git a/test/e2e/e2e-test/src/test/resources/expected/browser/error-log.yml b/test/e2e/e2e-test/src/test/resources/expected/browser/error-log.yml new file mode 100644 index 000000000..0631f79f0 --- /dev/null +++ b/test/e2e/e2e-test/src/test/resources/expected/browser/error-log.yml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +logs: + - service: "e2e" + serviceVersion: "v1.0.0" + time: not null + pagePath: "/e2e-browser" + category: not null + grade: not null + message: "test" + line: 1 + col: 1 + stack: "e2e" + errorUrl: "/e2e-browser" + firstReportedError: not null \ No newline at end of file diff --git a/test/e2e/e2e-test/src/test/resources/expected/browser/page-path.yml b/test/e2e/e2e-test/src/test/resources/expected/browser/page-path.yml new file mode 100644 index 000000000..5112247b6 --- /dev/null +++ b/test/e2e/e2e-test/src/test/resources/expected/browser/page-path.yml @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +endpoints: + - key: ZTJl.0_L2UyZS1icm93c2Vy + label: /e2e-browser \ No newline at end of file diff --git a/test/e2e/e2e-test/src/test/resources/expected/browser/services.yml b/test/e2e/e2e-test/src/test/resources/expected/browser/services.yml new file mode 100644 index 000000000..37e86cbee --- /dev/null +++ b/test/e2e/e2e-test/src/test/resources/expected/browser/services.yml @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + - key: ZTJl.0 + label: "e2e" \ No newline at end of file diff --git a/test/e2e/e2e-test/src/test/resources/expected/browser/version.yml b/test/e2e/e2e-test/src/test/resources/expected/browser/version.yml new file mode 100644 index 000000000..b868b1550 --- /dev/null +++ b/test/e2e/e2e-test/src/test/resources/expected/browser/version.yml @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +instances: + - key: ZTJl.0_djEuMC4w + label: not null \ No newline at end of file diff --git a/test/e2e/pom.xml b/test/e2e/pom.xml index eb1d9cbb3..d206cdb4f 100644 --- a/test/e2e/pom.xml +++ b/test/e2e/pom.xml @@ -169,6 +169,10 @@ **/*.yaml, **/*.xml + + **/.asf.yaml, + **/.github/** + import.control=${maven.multiModuleProjectDirectory}/apm-checkstyle/importControl.xml