Table name and time bucket error (#900)

* Get the MQ server address id from reference.

* Fixed the remote exception bug when running cluster mode collector.
How: Register remote data entity into remote module.

* Fixed the bug which the end second time bucket calculate result is incorrect.

* Fixed a test case error.
This commit is contained in:
彭勇升 pengys 2018-03-07 20:03:35 +08:00 committed by 吴晟 Wu Sheng
parent d04d85900e
commit f75b067bb7
12 changed files with 100 additions and 37 deletions

View File

@ -39,7 +39,7 @@ public class TimePyramidTableNameBuilder {
tableName = tableName + Const.ID_SPLIT + TimePyramid.Day.getName();
break;
case HOUR:
tableName = tableName + Const.ID_SPLIT + TimePyramid.Day.getName();
tableName = tableName + Const.ID_SPLIT + TimePyramid.Hour.getName();
break;
case MINUTE:
tableName = tableName + Const.ID_SPLIT + TimePyramid.Minute.getName();

View File

@ -106,6 +106,7 @@ public class InstanceEsUIDAO extends EsDAO implements IInstanceUIDAO {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
//TODO second
BoolQueryBuilder timeBoolQuery = QueryBuilders.boolQuery();
timeBoolQuery.should().add(QueryBuilders.rangeQuery(InstanceTable.COLUMN_REGISTER_TIME).gte(startSecondTimeBucket).lte(endSecondTimeBucket));
timeBoolQuery.should().add(QueryBuilders.rangeQuery(InstanceTable.COLUMN_HEARTBEAT_TIME).gte(startSecondTimeBucket).lte(endSecondTimeBucket));
@ -171,6 +172,7 @@ public class InstanceEsUIDAO extends EsDAO implements IInstanceUIDAO {
}
boolQuery.must().add(QueryBuilders.termQuery(InstanceTable.COLUMN_IS_ADDRESS, BooleanUtils.FALSE));
//TODO second
BoolQueryBuilder timeBoolQuery = QueryBuilders.boolQuery();
timeBoolQuery.should().add(QueryBuilders.rangeQuery(InstanceTable.COLUMN_REGISTER_TIME).gte(startSecondTimeBucket).lte(endSecondTimeBucket));
timeBoolQuery.should().add(QueryBuilders.rangeQuery(InstanceTable.COLUMN_HEARTBEAT_TIME).gte(startSecondTimeBucket).lte(endSecondTimeBucket));
@ -197,6 +199,7 @@ public class InstanceEsUIDAO extends EsDAO implements IInstanceUIDAO {
boolQuery.must().add(QueryBuilders.termQuery(InstanceTable.COLUMN_APPLICATION_ID, applicationId));
boolQuery.must().add(QueryBuilders.termQuery(InstanceTable.COLUMN_IS_ADDRESS, BooleanUtils.FALSE));
//TODO second
BoolQueryBuilder timeBoolQuery = QueryBuilders.boolQuery();
timeBoolQuery.should().add(QueryBuilders.rangeQuery(InstanceTable.COLUMN_REGISTER_TIME).gte(startSecondTimeBucket).lte(endSecondTimeBucket));
timeBoolQuery.should().add(QueryBuilders.rangeQuery(InstanceTable.COLUMN_HEARTBEAT_TIME).gte(startSecondTimeBucket).lte(endSecondTimeBucket));

View File

@ -57,6 +57,7 @@ public class SegmentDurationEsUIDAO extends EsDAO implements ISegmentDurationUID
List<QueryBuilder> mustQueryList = boolQueryBuilder.must();
if (startSecondTimeBucket != 0 && endSecondTimeBucket != 0) {
//TODO second
mustQueryList.add(QueryBuilders.rangeQuery(SegmentDurationTable.COLUMN_TIME_BUCKET).gte(startSecondTimeBucket).lte(endSecondTimeBucket));
}

View File

@ -51,8 +51,8 @@ public class AlarmQuery implements Query {
public Alarm loadAlarmList(String keyword, AlarmType alarmType, Duration duration,
Pagination paging) throws ParseException {
long startTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getStart()) / 100;
long endTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getEnd()) / 100;
long startTimeBucket = DurationUtils.INSTANCE.startTimeDurationToSecondTimeBucket(duration.getStep(), duration.getStart()) / 100;
long endTimeBucket = DurationUtils.INSTANCE.endTimeDurationToSecondTimeBucket(duration.getStep(), duration.getEnd()) / 100;
PaginationUtils.Page page = PaginationUtils.INSTANCE.exchange(paging);

View File

@ -69,8 +69,8 @@ public class ApplicationQuery implements Query {
}
public List<Application> getAllApplication(Duration duration) throws ParseException {
long startSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getEnd());
long startSecondTimeBucket = DurationUtils.INSTANCE.startTimeDurationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.endTimeDurationToSecondTimeBucket(duration.getStep(), duration.getEnd());
return getApplicationService().getApplications(startSecondTimeBucket, endSecondTimeBucket);
}
@ -79,8 +79,8 @@ public class ApplicationQuery implements Query {
long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart());
long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd());
long startSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getEnd());
long startSecondTimeBucket = DurationUtils.INSTANCE.startTimeDurationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.endTimeDurationToSecondTimeBucket(duration.getStep(), duration.getEnd());
return getApplicationTopologyService().getApplicationTopology(duration.getStep(), applicationId, startTimeBucket, endTimeBucket, startSecondTimeBucket, endSecondTimeBucket);
}
@ -90,8 +90,8 @@ public class ApplicationQuery implements Query {
long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart());
long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd());
long startSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getEnd());
long startSecondTimeBucket = DurationUtils.INSTANCE.startTimeDurationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.endTimeDurationToSecondTimeBucket(duration.getStep(), duration.getEnd());
return getApplicationService().getSlowService(applicationId, duration.getStep(), startTimeBucket, endTimeBucket, startSecondTimeBucket, endSecondTimeBucket, topN);
}
@ -101,8 +101,8 @@ public class ApplicationQuery implements Query {
long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart());
long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd());
long startSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getEnd());
long startSecondTimeBucket = DurationUtils.INSTANCE.startTimeDurationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.endTimeDurationToSecondTimeBucket(duration.getStep(), duration.getEnd());
return getServerService().getServerThroughput(applicationId, duration.getStep(), startTimeBucket, endTimeBucket, startSecondTimeBucket, endSecondTimeBucket, topN);
}

View File

@ -92,15 +92,15 @@ public class OverViewLayerQuery implements Query {
long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart());
long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd());
long startSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getEnd());
long startSecondTimeBucket = DurationUtils.INSTANCE.startTimeDurationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.endTimeDurationToSecondTimeBucket(duration.getStep(), duration.getEnd());
return getClusterTopologyService().getClusterTopology(duration.getStep(), startTimeBucket, endTimeBucket, startSecondTimeBucket, endSecondTimeBucket);
}
public ClusterBrief getClusterBrief(Duration duration) throws ParseException {
long startSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getEnd());
long startSecondTimeBucket = DurationUtils.INSTANCE.startTimeDurationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.endTimeDurationToSecondTimeBucket(duration.getStep(), duration.getEnd());
ClusterBrief clusterBrief = new ClusterBrief();
clusterBrief.setNumOfApplication(getApplicationService().getApplications(startSecondTimeBucket, endSecondTimeBucket).size());
@ -115,15 +115,15 @@ public class OverViewLayerQuery implements Query {
long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart());
long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd());
long startSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getEnd());
long startSecondTimeBucket = DurationUtils.INSTANCE.startTimeDurationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.endTimeDurationToSecondTimeBucket(duration.getStep(), duration.getEnd());
return getAlarmService().getApplicationAlarmTrend(duration.getStep(), startTimeBucket, endTimeBucket, startSecondTimeBucket, endSecondTimeBucket);
}
public ConjecturalAppBrief getConjecturalApps(Duration duration) throws ParseException {
long startSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getEnd());
long startSecondTimeBucket = DurationUtils.INSTANCE.startTimeDurationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.endTimeDurationToSecondTimeBucket(duration.getStep(), duration.getEnd());
return getApplicationService().getConjecturalApps(duration.getStep(), startSecondTimeBucket, endSecondTimeBucket);
}
@ -132,8 +132,8 @@ public class OverViewLayerQuery implements Query {
long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart());
long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd());
long startSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getEnd());
long startSecondTimeBucket = DurationUtils.INSTANCE.startTimeDurationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.endTimeDurationToSecondTimeBucket(duration.getStep(), duration.getEnd());
return getServiceNameService().getSlowService(duration.getStep(), startTimeBucket, endTimeBucket, startSecondTimeBucket, endSecondTimeBucket, topN);
}

View File

@ -53,14 +53,14 @@ public class ServerQuery implements Query {
}
public List<AppServerInfo> searchServer(String keyword, Duration duration) throws ParseException {
long startSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getEnd());
long startSecondTimeBucket = DurationUtils.INSTANCE.startTimeDurationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.endTimeDurationToSecondTimeBucket(duration.getStep(), duration.getEnd());
return getServerService().searchServer(keyword, startSecondTimeBucket, endSecondTimeBucket);
}
public List<AppServerInfo> getAllServer(int applicationId, Duration duration) throws ParseException {
long startSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getEnd());
long startSecondTimeBucket = DurationUtils.INSTANCE.startTimeDurationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.endTimeDurationToSecondTimeBucket(duration.getStep(), duration.getEnd());
return getServerService().getAllServer(applicationId, startSecondTimeBucket, endSecondTimeBucket);
}

View File

@ -74,8 +74,8 @@ public class ServiceQuery implements Query {
long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart());
long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd());
long startSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getEnd());
long startSecondTimeBucket = DurationUtils.INSTANCE.startTimeDurationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.endTimeDurationToSecondTimeBucket(duration.getStep(), duration.getEnd());
return getServiceNameService().getServiceTPSTrend(serviceId, duration.getStep(), startTimeBucket, endTimeBucket, startSecondTimeBucket, endSecondTimeBucket);
}
@ -90,8 +90,8 @@ public class ServiceQuery implements Query {
long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart());
long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd());
long startSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(duration.getStep(), duration.getEnd());
long startSecondTimeBucket = DurationUtils.INSTANCE.startTimeDurationToSecondTimeBucket(duration.getStep(), duration.getStart());
long endSecondTimeBucket = DurationUtils.INSTANCE.endTimeDurationToSecondTimeBucket(duration.getStep(), duration.getEnd());
return getServiceTopologyService().getServiceTopology(duration.getStep(), serviceId, startTimeBucket, endTimeBucket, startSecondTimeBucket, endSecondTimeBucket);
}

View File

@ -68,8 +68,8 @@ public class TraceQuery implements Query {
if (StringUtils.isNotEmpty(condition.getTraceId())) {
traceId = condition.getTraceId();
} else if (ObjectUtils.isNotEmpty(condition.getQueryDuration())) {
startSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(condition.getQueryDuration().getStep(), condition.getQueryDuration().getStart());
endSecondTimeBucket = DurationUtils.INSTANCE.durationToSecondTimeBucket(condition.getQueryDuration().getStep(), condition.getQueryDuration().getEnd());
startSecondTimeBucket = DurationUtils.INSTANCE.startTimeDurationToSecondTimeBucket(condition.getQueryDuration().getStep(), condition.getQueryDuration().getStart());
endSecondTimeBucket = DurationUtils.INSTANCE.endTimeDurationToSecondTimeBucket(condition.getQueryDuration().getStep(), condition.getQueryDuration().getEnd());
} else {
throw new UnexpectedException("The condition must contains either queryDuration or traceId.");
}

View File

@ -42,7 +42,7 @@ public enum DurationUtils {
return Long.valueOf(dateStr);
}
public long durationToSecondTimeBucket(Step step, String dateStr) throws ParseException {
public long startTimeDurationToSecondTimeBucket(Step step, String dateStr) throws ParseException {
long secondTimeBucket = 0;
switch (step) {
case MONTH:
@ -64,6 +64,28 @@ public enum DurationUtils {
return secondTimeBucket;
}
public long endTimeDurationToSecondTimeBucket(Step step, String dateStr) throws ParseException {
long secondTimeBucket = 0;
switch (step) {
case MONTH:
secondTimeBucket = (((exchangeToTimeBucket(dateStr) * 100 + 99) * 100 + 99) * 100 + 99) * 100 + 99;
break;
case DAY:
secondTimeBucket = ((exchangeToTimeBucket(dateStr) * 100 + 99) * 100 + 99) * 100 + 99;
break;
case HOUR:
secondTimeBucket = (exchangeToTimeBucket(dateStr) * 100 + 99) * 100 + 99;
break;
case MINUTE:
secondTimeBucket = exchangeToTimeBucket(dateStr) * 100 + 99;
break;
case SECOND:
secondTimeBucket = exchangeToTimeBucket(dateStr);
break;
}
return secondTimeBucket;
}
public int secondsBetween(Step step, long startTimeBucket, long endTimeBucket) throws ParseException {
Date startDate = null;
Date endDate = null;
@ -139,7 +161,8 @@ public enum DurationUtils {
return dateTime;
}
public List<DurationPoint> getDurationPoints(Step step, long startTimeBucket, long endTimeBucket) throws ParseException {
public List<DurationPoint> getDurationPoints(Step step, long startTimeBucket,
long endTimeBucket) throws ParseException {
DateTime dateTime = parseToDateTime(step, startTimeBucket);
List<DurationPoint> durations = new LinkedList<>();

View File

@ -41,7 +41,7 @@ public class ApplicationQueryTestCase {
Mockito.when(applicationService.getApplications(Mockito.anyLong(), Mockito.anyLong())).then(invocation -> {
Object[] arguments = invocation.getArguments();
Assert.assertEquals(20170100000000L, arguments[0]);
Assert.assertEquals(20170100000000L, arguments[1]);
Assert.assertEquals(20170199999999L, arguments[1]);
return null;
});
@ -62,7 +62,7 @@ public class ApplicationQueryTestCase {
Mockito.when(applicationService.getApplications(Mockito.anyLong(), Mockito.anyLong())).then(invocation -> {
Object[] arguments = invocation.getArguments();
Assert.assertEquals(20170101000000L, arguments[0]);
Assert.assertEquals(20170101000000L, arguments[1]);
Assert.assertEquals(20170101999999L, arguments[1]);
return null;
});
@ -83,7 +83,7 @@ public class ApplicationQueryTestCase {
Mockito.when(applicationService.getApplications(Mockito.anyLong(), Mockito.anyLong())).then(invocation -> {
Object[] arguments = invocation.getArguments();
Assert.assertEquals(20170101010000L, arguments[0]);
Assert.assertEquals(20170101010000L, arguments[1]);
Assert.assertEquals(20170101019999L, arguments[1]);
return null;
});
@ -104,7 +104,7 @@ public class ApplicationQueryTestCase {
Mockito.when(applicationService.getApplications(Mockito.anyLong(), Mockito.anyLong())).then(invocation -> {
Object[] arguments = invocation.getArguments();
Assert.assertEquals(20170101010100L, arguments[0]);
Assert.assertEquals(20170101010100L, arguments[1]);
Assert.assertEquals(20170101010199L, arguments[1]);
return null;
});

View File

@ -109,4 +109,40 @@ public class DurationUtilsTestCase {
public void testGetDurationPointsErrorDuration() throws ParseException {
DurationUtils.INSTANCE.getDurationPoints(Step.MONTH, 20171001, 20180301);
}
@Test
public void testStartTimeDurationToSecondTimeBucket() throws ParseException {
long secondTimeBucket = DurationUtils.INSTANCE.startTimeDurationToSecondTimeBucket(Step.MONTH, "201710");
Assert.assertEquals(20171000000000L, secondTimeBucket);
secondTimeBucket = DurationUtils.INSTANCE.startTimeDurationToSecondTimeBucket(Step.DAY, "20171001");
Assert.assertEquals(20171001000000L, secondTimeBucket);
secondTimeBucket = DurationUtils.INSTANCE.startTimeDurationToSecondTimeBucket(Step.HOUR, "2017100108");
Assert.assertEquals(20171001080000L, secondTimeBucket);
secondTimeBucket = DurationUtils.INSTANCE.startTimeDurationToSecondTimeBucket(Step.MINUTE, "201710010805");
Assert.assertEquals(20171001080500L, secondTimeBucket);
secondTimeBucket = DurationUtils.INSTANCE.startTimeDurationToSecondTimeBucket(Step.SECOND, "20171001080501");
Assert.assertEquals(20171001080501L, secondTimeBucket);
}
@Test
public void testEndTimeDurationToSecondTimeBucket() throws ParseException {
long secondTimeBucket = DurationUtils.INSTANCE.endTimeDurationToSecondTimeBucket(Step.MONTH, "201710");
Assert.assertEquals(20171099999999L, secondTimeBucket);
secondTimeBucket = DurationUtils.INSTANCE.endTimeDurationToSecondTimeBucket(Step.DAY, "20171001");
Assert.assertEquals(20171001999999L, secondTimeBucket);
secondTimeBucket = DurationUtils.INSTANCE.endTimeDurationToSecondTimeBucket(Step.HOUR, "2017100108");
Assert.assertEquals(20171001089999L, secondTimeBucket);
secondTimeBucket = DurationUtils.INSTANCE.endTimeDurationToSecondTimeBucket(Step.MINUTE, "201710010805");
Assert.assertEquals(20171001080599L, secondTimeBucket);
secondTimeBucket = DurationUtils.INSTANCE.endTimeDurationToSecondTimeBucket(Step.SECOND, "20171001080501");
Assert.assertEquals(20171001080501L, secondTimeBucket);
}
}