kazusa
/
skywalking-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Set the endpoint name length to 150, to keep safe and avoid over-length. (#4595) 

* Set the endpoint name length to 150, to keep safe and avoid over-length.

* Set endpointNameMaxLength in the application.yml

* Fix a mischange.

* Correct the endpoint name length control logic

Co-authored-by: kezhenxu94 <kezhenxu94@apache.org>
This commit is contained in:
吴晟 Wu Sheng 2020-03-31 09:04:10 +08:00 committed by GitHub
parent b28b811af7
commit 54fb6e7285
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 72 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java
View File

@ -18,14 +18,13 @@
package org.apache.skywalking.apm.agent.core.conf;
import java.util.HashMap;
import java.util.Map;
import org.apache.skywalking.apm.agent.core.context.trace.TraceSegment;
import org.apache.skywalking.apm.agent.core.logging.core.LogLevel;
import org.apache.skywalking.apm.agent.core.logging.core.LogOutput;
import org.apache.skywalking.apm.agent.core.logging.core.WriterFactory;
import java.util.HashMap;
import java.util.Map;
/**
* This is the core config in sniffer agent.
*/
@ -102,9 +101,12 @@ public class Config {
public static long FORCE_RECONNECTION_PERIOD = 1;
/**
* Limit the length of the operationName to prevent errors when inserting elasticsearch
**/
public static int OPERATION_NAME_THRESHOLD = 500;
* Limit the length of the operationName to prevent the overlength issue in the storage.
*
* <p>NOTICE</p>
* In the current practice, we don't recommend the length over 190.
*/
public static int OPERATION_NAME_THRESHOLD = 150;
}
public static class Collector {
@ -173,11 +175,9 @@ public class Config {
public static class Dictionary {
/**
* The buffer size of application codes and peer
* The buffer size of the registered network addresses.
*/
public static int SERVICE_CODE_BUFFER_SIZE = 10 * 10000;
public static int ENDPOINT_NAME_BUFFER_SIZE = 1000 * 10000;
public static int NETWORK_ADDRESS_BUFFER_SIZE = 10 * 10000;
}
public static class Logging {

4
apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/dictionary/NetworkAddressDictionary.java
View File

@ -26,7 +26,7 @@ import org.apache.skywalking.apm.network.register.v2.NetAddressMapping;
import org.apache.skywalking.apm.network.register.v2.NetAddresses;
import org.apache.skywalking.apm.network.register.v2.RegisterGrpc;
import static org.apache.skywalking.apm.agent.core.conf.Config.Dictionary.SERVICE_CODE_BUFFER_SIZE;
import static org.apache.skywalking.apm.agent.core.conf.Config.Dictionary.NETWORK_ADDRESS_BUFFER_SIZE;
/**
* Map of network address id to network literal address, which is from the collector side.
@ -41,7 +41,7 @@ public enum NetworkAddressDictionary {
if (applicationId != null) {
return new Found(applicationId);
} else {
if (serviceDictionary.size() + unRegisterServices.size() < SERVICE_CODE_BUFFER_SIZE) {
if (serviceDictionary.size() + unRegisterServices.size() < NETWORK_ADDRESS_BUFFER_SIZE) {
unRegisterServices.add(networkAddress);
}
return new NotFound();

5
apm-sniffer/config/agent.config
View File

@ -29,7 +29,7 @@ agent.service_name=${SW_AGENT_NAME:Your_ApplicationName}
# The max amount of spans in a single segment.
# Through this config item, SkyWalking keep your application memory cost estimated.
# agent.span_limit_per_segment=${SW_AGENT_SPAN_LIMIT:300}
# agent.span_limit_per_segment=${SW_AGENT_SPAN_LIMIT:150}
# Ignore the segments if their operation names end with these suffix.
# agent.ignore_suffix=${SW_AGENT_IGNORE_SUFFIX:.jpg,.jpeg,.js,.css,.png,.bmp,.gif,.ico,.mp3,.mp4,.html,.svg}
@ -39,7 +39,8 @@ agent.service_name=${SW_AGENT_NAME:Your_ApplicationName}
# agent.is_open_debugging_class = ${SW_AGENT_OPEN_DEBUG:true}
# The operationName max length
# agent.operation_name_threshold=${SW_AGENT_OPERATION_NAME_THRESHOLD:500}
# Notice, in the current practice, we don't recommend the length over 190.
# agent.operation_name_threshold=${SW_AGENT_OPERATION_NAME_THRESHOLD:150}
# If true, skywalking agent will enable profile when user create a new profile task. Otherwise disable profile.
# profile.active=${SW_AGENT_PROFILE_ACTIVE:true}

4
dist-material/application.yml
View File

@ -85,7 +85,9 @@ core:
# and it will cause more load for memory, network of OAP and storage.
# But, being activated, user could see the name in the storage entities, which make users easier to use 3rd party tool, such as Kibana->ES, to query the data by themselves.
activeExtraModelColumns: ${SW_CORE_ACTIVE_EXTRA_MODEL_COLUMNS:false}
# The max length of the endpoint name.
# In the current practice, we don't recommend the length over 190.
endpointNameMaxLength: ${SW_CORE_ENDPOINT_NAME_MAX_LENGTH:150}
storage:
selector: ${SW_STORAGE:h2}
elasticsearch:

4
oap-server/server-bootstrap/src/main/resources/application.yml
View File

@ -84,7 +84,9 @@ core:
# and it will cause more load for memory, network of OAP and storage.
# But, being activated, user could see the name in the storage entities, which make users easier to use 3rd party tool, such as Kibana->ES, to query the data by themselves.
activeExtraModelColumns: ${SW_CORE_ACTIVE_EXTRA_MODEL_COLUMNS:false}
# The max length of the endpoint name.
# In the current practice, we don't recommend the length over 190.
endpointNameMaxLength: ${SW_CORE_ENDPOINT_NAME_MAX_LENGTH:150}
storage:
selector: ${SW_STORAGE:h2}
elasticsearch:

22
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java
View File

@ -57,8 +57,8 @@ import org.apache.skywalking.oap.server.library.module.ModuleDefine;
* Core module definition. Define all open services to other modules.
*/
public class CoreModule extends ModuleDefine {
public static final String NAME = "core";
private static int ENDPOINT_NAME_MAX_LENGTH = 150;
public CoreModule() {
super(NAME);
@ -87,6 +87,26 @@ public class CoreModule extends ModuleDefine {
return classes.toArray(new Class[] {});
}
/**
* Format endpoint name by using the length config in the core module. This is a global rule, every place including
* endpoint as the {@link org.apache.skywalking.oap.server.core.source.Source} should follow this for any core
* module implementation.
*
* @param endpointName raw data, literal string.
* @return the string, which length less than or equals {@link #ENDPOINT_NAME_MAX_LENGTH};
*/
public static String formatEndpointName(String endpointName) {
if (endpointName.length() > ENDPOINT_NAME_MAX_LENGTH) {
return endpointName.substring(0, ENDPOINT_NAME_MAX_LENGTH);
} else {
return endpointName;
}
}
public static void setEndpointNameMaxLength(final int endpointNameMaxLength) {
ENDPOINT_NAME_MAX_LENGTH = endpointNameMaxLength;
}
private void addProfileService(List<Class> classes) {
classes.add(ProfileTaskMutationService.class);
classes.add(ProfileTaskQueryService.class);

16
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java
View File

@ -88,7 +88,6 @@ public class CoreModuleConfig extends ModuleConfig {
*/
@Setter
private int remoteTimeout = 20;
/**
* Following are cache settings for inventory(s)
*/
@ -96,31 +95,34 @@ public class CoreModuleConfig extends ModuleConfig {
private long maxSizeOfServiceInstanceInventory = 1_000_000L;
private long maxSizeOfEndpointInventory = 1_000_000L;
private long maxSizeOfNetworkInventory = 1_000_000L;
/**
* Following are cache setting for none stream(s)
*/
private long maxSizeOfProfileTask = 10_000L;
/**
* Analyze profile snapshots paging size.
*/
private int maxPageSizeOfQueryProfileSnapshot = 500;
/**
* Analyze profile snapshots max size.
*/
private int maxSizeOfAnalyzeProfileSnapshot = 12000;
/**
* Extra model column are the column defined by {@link ScopeDefaultColumn.DefinedByField#requireDynamicActive()} ==
* true. These columns of model are not required logically in aggregation or further query, and it will cause more load for
* memory, network of OAP and storage.
* true. These columns of model are not required logically in aggregation or further query, and it will cause more
* load for memory, network of OAP and storage.
*
* But, being activated, user could see the name in the storage entities, which make users easier to use 3rd party
* tool, such as Kibana->ES, to query the data by themselves.
*/
private boolean activeExtraModelColumns = false;
/**
* The max length of the endpoint name.
*
* <p>NOTICE</p>
* In the current practice, we don't recommend the length over 190.
*/
private int endpointNameMaxLength = 150;
public CoreModuleConfig() {
this.downsampling = new ArrayList<>();

3
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java
View File

@ -139,6 +139,9 @@ public class CoreModuleProvider extends ModuleProvider {
if (moduleConfig.isActiveExtraModelColumns()) {
DefaultScopeDefine.activeExtraModelColumns();
}
if (moduleConfig.getEndpointNameMaxLength() > 0) {
CoreModule.setEndpointNameMaxLength(moduleConfig.getEndpointNameMaxLength());
}
StreamAnnotationListener streamAnnotationListener = new StreamAnnotationListener(getManager());

7
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java
View File

@ -20,6 +20,7 @@ package org.apache.skywalking.oap.server.core.source;
import lombok.Getter;
import lombok.Setter;
import org.apache.skywalking.oap.server.core.CoreModule;
import org.apache.skywalking.oap.server.core.analysis.manual.endpoint.EndpointTraffic;
import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.ENDPOINT;
@ -42,9 +43,13 @@ public class Endpoint extends Source {
}
@Getter
@Setter
@ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true)
private String name;
public void setName(final String name) {
this.name = CoreModule.formatEndpointName(name);
}
@Getter
@Setter
@ScopeDefaultColumn.DefinedByField(columnName = "service_id")

17
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/EndpointRelation.java
View File

@ -20,6 +20,7 @@ package org.apache.skywalking.oap.server.core.source;
import lombok.Getter;
import lombok.Setter;
import org.apache.skywalking.oap.server.core.CoreModule;
import org.apache.skywalking.oap.server.core.analysis.manual.RelationDefineUtil;
import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.ENDPOINT_RELATION;
@ -44,9 +45,13 @@ public class EndpointRelation extends Source {
}
@Getter
@Setter
@ScopeDefaultColumn.DefinedByField(columnName = "source_endpoint_name")
private String endpoint;
public void setEndpoint(final String endpoint) {
this.endpoint = CoreModule.formatEndpointName(endpoint);
}
@Getter
@Setter
@ScopeDefaultColumn.DefinedByField(columnName = "service_id")
@ -61,17 +66,20 @@ public class EndpointRelation extends Source {
@Getter
@Setter
private String serviceInstanceName;
@Getter
@Setter
@ScopeDefaultColumn.DefinedByField(columnName = "child_endpoint_name")
private String childEndpoint;
public void setChildEndpoint(final String childEndpoint) {
this.childEndpoint = CoreModule.formatEndpointName(childEndpoint);
}
@Getter
@Setter
@ScopeDefaultColumn.DefinedByField(columnName = "child_service_id")
private int childServiceId;
@Getter
@Setter
@Getter
@ScopeDefaultColumn.DefinedByField(columnName = "child_service_name", requireDynamicActive = true)
private String childServiceName;
@Getter
@ -80,7 +88,6 @@ public class EndpointRelation extends Source {
@Getter
@Setter
private String childServiceInstanceName;
@Getter
@Setter
private int componentId;