kazusa
/
skywalking-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Save error trace segment even that segment will abandoned by sampling mechanism (#5407) 

* Save some error trace segment, event this segment abandoned by server side trace sampling mechanism.
* Support forceSaveErrorSegment config to control force save some error segment.

Co-authored-by: echo <zifeihan@foxmail.com>
Co-authored-by: 吴晟 Wu Sheng <wu.sheng@foxmail.com>
This commit is contained in:
zifeihan 2020-08-28 20:37:19 +08:00 committed by GitHub
parent afc0aef567
commit 6227db19cd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 28 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
docs/en/setup/backend/trace-sampling.md
View File

@ -30,4 +30,9 @@ When you set the rate different, let's say
And we assume the agents reported all trace segments to backend,
Then the 35% traces in the global will be collected and saved in storage consistent/complete, with all spans.
20% trace segments, which reported to Backend-Instance**B**, will saved in storage, maybe miss some trace segments,
because they are reported to Backend-Instance**A** and ignored.
because they are reported to Backend-Instance**A** and ignored.
# Note
When you open sampling, the actual sample rate could be over sampleRate. Because currently, all error segments will be saved, meanwhile, the upstream and downstream may not be sampled. This feature is going to make sure you could have the error stacks and segments, but don't guarantee you would have the whole trace.
Also, the side effect would be, if most of the accesses are fail, the sampling rate would be closing to 100%, which could crash the backend or storage clusters.

7
oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/AnalyzerModuleConfig.java
View File

@ -78,4 +78,11 @@ public class AnalyzerModuleConfig extends ModuleConfig {
@Getter
private final String configPath = "meter-receive-config";
/**
* Sample the trace segment if the segment has span(s) tagged as error status, and ignore the sampleRate configuration.
*/
@Setter
@Getter
private boolean forceSampleErrorSegment = true;
}

26
oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/SegmentAnalysisListener.java
View File

@ -47,6 +47,7 @@ import org.apache.skywalking.oap.server.library.util.BooleanUtils;
public class SegmentAnalysisListener implements FirstAnalysisListener, EntryAnalysisListener, SegmentListener {
private final SourceReceiver sourceReceiver;
private final TraceSegmentSampler sampler;
private final boolean forceSampleErrorSegment;
private final NamingControl namingControl;
private final List<String> searchableTagKeys;
@ -125,18 +126,6 @@ public class SegmentAnalysisListener implements FirstAnalysisListener, EntryAnal
@Override
public void parseSegment(SegmentObject segmentObject) {
if (sampleStatus.equals(SAMPLE_STATUS.UNKNOWN) || sampleStatus.equals(SAMPLE_STATUS.IGNORE)) {
if (sampler.shouldSample(segmentObject.getTraceId())) {
sampleStatus = SAMPLE_STATUS.SAMPLED;
} else {
sampleStatus = SAMPLE_STATUS.IGNORE;
}
}
if (sampleStatus.equals(SAMPLE_STATUS.IGNORE)) {
return;
}
segment.setTraceId(segmentObject.getTraceId());
segmentObject.getSpansList().forEach(span -> {
if (startTimestamp == 0 || startTimestamp > span.getStartTime()) {
@ -153,6 +142,16 @@ public class SegmentAnalysisListener implements FirstAnalysisListener, EntryAnal
});
final long accurateDuration = endTimestamp - startTimestamp;
duration = accurateDuration > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) accurateDuration;
if (sampleStatus.equals(SAMPLE_STATUS.UNKNOWN) || sampleStatus.equals(SAMPLE_STATUS.IGNORE)) {
if (sampler.shouldSample(segmentObject.getTraceId())) {
sampleStatus = SAMPLE_STATUS.SAMPLED;
} else if (isError && forceSampleErrorSegment) {
sampleStatus = SAMPLE_STATUS.SAMPLED;
} else {
sampleStatus = SAMPLE_STATUS.IGNORE;
}
}
}
private void appendSearchableTags(SpanObject span) {
@ -186,6 +185,7 @@ public class SegmentAnalysisListener implements FirstAnalysisListener, EntryAnal
public static class Factory implements AnalysisListenerFactory {
private final SourceReceiver sourceReceiver;
private final TraceSegmentSampler sampler;
private final boolean forceSampleErrorSegment;
private final NamingControl namingControl;
private final List<String> searchTagKeys;
@ -196,6 +196,7 @@ public class SegmentAnalysisListener implements FirstAnalysisListener, EntryAnal
.getService(ConfigService.class);
this.searchTagKeys = Arrays.asList(configService.getSearchableTracesTags().split(Const.COMMA));
this.sampler = new TraceSegmentSampler(config.getTraceSampleRateWatcher());
this.forceSampleErrorSegment = config.isForceSampleErrorSegment();
this.namingControl = moduleManager.find(CoreModule.NAME)
.provider()
.getService(NamingControl.class);
@ -206,6 +207,7 @@ public class SegmentAnalysisListener implements FirstAnalysisListener, EntryAnal
return new SegmentAnalysisListener(
sourceReceiver,
sampler,
forceSampleErrorSegment,
namingControl,
searchTagKeys
);

1
oap-server/server-bootstrap/src/main/resources/application.yml
View File

@ -175,6 +175,7 @@ agent-analyzer:
default:
sampleRate: ${SW_TRACE_SAMPLE_RATE:10000} # The sample rate precision is 1/10000. 10000 means 100% sample in default.
slowDBAccessThreshold: ${SW_SLOW_DB_THRESHOLD:default:200,mongodb:100} # The slow database access thresholds. Unit ms.
forceSampleErrorSegment: ${SW_FORCE_SAMPLE_ERROR_SEGMENT:true} # When sampling mechanism active, this config can open(true) force save some error segment. true is default.
receiver-sharing-server:
selector: ${SW_RECEIVER_SHARING_SERVER:default}