From b0a875fe3faefd64fddb8c4d0b4714eefdc126f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=88=98=E5=A8=81?=
<51618159+LIU-WEI-git@users.noreply.github.com>
Date: Sun, 30 Jan 2022 09:02:39 +0800
Subject: [PATCH] Bump up iotdb-session to 0.12.4 and fix IoTDB storage option
bug (#8484)
* Fix IoTDB Storage Option insert null index value.
* Set the default value of SW_STORAGE_IOTDB_SESSIONPOOL_SIZE to 8
* Bump up iotdb-session to 0.12.4
* Update doc
---
CHANGES.md | 3 +++
dist-material/release-docs/LICENSE | 8 ++++----
docs/en/setup/backend/backend-storage.md | 2 +-
docs/en/setup/backend/configuration-vocabulary.md | 8 ++++++++
oap-server-bom/pom.xml | 2 +-
.../server-starter/src/main/resources/application.yml | 2 +-
.../oap/server/storage/plugin/iotdb/IoTDBClient.java | 7 +++++--
.../storage/plugin/iotdb/base/IoTDBInsertRequest.java | 4 ++--
.../server/storage/plugin/iotdb/base/IoTDBRecordDAO.java | 3 ---
test/e2e-v2/cases/alarm/iotdb/docker-compose.yml | 3 +--
test/e2e-v2/cases/event/iotdb/docker-compose.yml | 2 +-
test/e2e-v2/cases/log/iotdb/docker-compose.yml | 2 +-
test/e2e-v2/cases/profile/iotdb/docker-compose.yml | 2 +-
test/e2e-v2/cases/storage/iotdb/docker-compose.yml | 2 +-
test/e2e-v2/cases/ttl/iotdb/docker-compose.yml | 2 +-
test/e2e-v2/script/docker-compose/base-compose.yml | 1 +
tools/dependencies/known-oap-backend-dependencies.txt | 8 ++++----
17 files changed, 36 insertions(+), 25 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index c9597b178a..77b63fe00d 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -47,6 +47,9 @@ Release Notes.
* Doc update: `configuration-vocabulary`,`backend-cluster` about env variables `SW_CLUSTER_INTERNAL_COM_HOST` and `SW_CLUSTER_INTERNAL_COM_PORT`.
* Add Python MysqlClient component ID(7013) with mapping information.
* Support Java thread pool metrics analysis.
+* Fix IoTDB Storage Option insert null index value.
+* Set the default value of SW_STORAGE_IOTDB_SESSIONPOOL_SIZE to 8.
+* Bump up iotdb-session to 0.12.4.
#### UI
diff --git a/dist-material/release-docs/LICENSE b/dist-material/release-docs/LICENSE
index c23091ad5c..bfe5f6ac9a 100755
--- a/dist-material/release-docs/LICENSE
+++ b/dist-material/release-docs/LICENSE
@@ -330,10 +330,10 @@ The text of each license is the standard Apache 2.0 license.
Armeria 1.13.4, http://github.com/line/armeria, Apache 2.0
Brotli4j 1.6.0, https://github.com/hyperxpro/Brotli4j, Apache 2.0
micrometer 1.7.6, https://github.com/micrometer-metrics/micrometer, Apache 2.0
- iotdb-session 0.12.3: https://github.com/apache/iotdb, Apache 2.0
- iotdb-thrift 0.12.3: https://github.com/apache/iotdb, Apache 2.0
- service-rpc 0.12.3: https://github.com/apache/iotdb, Apache 2.0
- tsfile 0.12.3 https://github.com/apache/iotdb Apache 2.0
+ iotdb-session 0.12.4: https://github.com/apache/iotdb, Apache 2.0
+ iotdb-thrift 0.12.4: https://github.com/apache/iotdb, Apache 2.0
+ service-rpc 0.12.4: https://github.com/apache/iotdb, Apache 2.0
+ tsfile 0.12.4 https://github.com/apache/iotdb Apache 2.0
libthrift 0.14.1: https://github.com/apache/thrift Apache 2.0
j2objc 1.3: https://github.com/google/j2objc Apache 2.0
diff --git a/docs/en/setup/backend/backend-storage.md b/docs/en/setup/backend/backend-storage.md
index d420b0eae4..51215f18c3 100644
--- a/docs/en/setup/backend/backend-storage.md
+++ b/docs/en/setup/backend/backend-storage.md
@@ -294,7 +294,7 @@ storage:
username: ${SW_STORAGE_IOTDB_USERNAME:root}
password: ${SW_STORAGE_IOTDB_PASSWORD:root}
storageGroup: ${SW_STORAGE_IOTDB_STORAGE_GROUP:root.skywalking}
- sessionPoolSize: ${SW_STORAGE_IOTDB_SESSIONPOOL_SIZE:16}
+ sessionPoolSize: ${SW_STORAGE_IOTDB_SESSIONPOOL_SIZE:8} # If it's zero, the SessionPool size will be 2*CPU_Cores
fetchTaskLogMaxSize: ${SW_STORAGE_IOTDB_FETCH_TASK_LOG_MAX_SIZE:1000} # the max number of fetch task log in a request
```
All connection related settings, including host, rpcPort, username, and password are found in `application.yml`. Please ensure the IoTDB version >= 0.12.3.
diff --git a/docs/en/setup/backend/configuration-vocabulary.md b/docs/en/setup/backend/configuration-vocabulary.md
index 22b935b330..518ffe5dfe 100644
--- a/docs/en/setup/backend/configuration-vocabulary.md
+++ b/docs/en/setup/backend/configuration-vocabulary.md
@@ -143,6 +143,14 @@ core|default|role|Option values: `Mixed/Receiver/Aggregator`. **Receiver** mode
| - | - | batchEnabled | If true, write points with batch API. | SW_STORAGE_INFLUXDB_BATCH_ENABLED | true|
| - | - | fetchTaskLogMaxSize | The maximum number of fetch task log in a request. | SW_STORAGE_INFLUXDB_FETCH_TASK_LOG_MAX_SIZE | 5000|
| - | - | connectionResponseFormat | The response format of connection to influxDB. It can only be MSGPACK or JSON. | SW_STORAGE_INFLUXDB_CONNECTION_RESPONSE_FORMAT | MSGPACK |
+| - | iotdb | - | IoTDB storage. | - | - |
+| - | - | host | The host of IoTDB server. | SW_STORAGE_IOTDB_HOST | 127.0.0.1 |
+| - | - | rpcPort | The port listened by IoTDB server. | SW_STORAGE_IOTDB_RPC_PORT | 6667 |
+| - | - | username | The username of IoTDB | SW_STORAGE_IOTDB_USERNAME | root |
+| - | - | password | The password of IoTDB | SW_STORAGE_IOTDB_PASSWORD | root |
+| - | - | storageGroup | The path of Storage Group and it must start with `root.` | SW_STORAGE_IOTDB_STORAGE_GROUP | root.skywalking |
+| - | - | sessionPoolSize | The connection pool size for IoTDB. If the value is 0, the size of SessionPool will be 2 * CPU_Cores | SW_STORAGE_IOTDB_SESSIONPOOL_SIZE | 8 |
+| - | - | fetchTaskLogMaxSize | the max number of fetch task log in a request | SW_STORAGE_IOTDB_FETCH_TASK_LOG_MAX_SIZE | 1000 |
| agent-analyzer | default | Agent Analyzer. | SW_AGENT_ANALYZER | default |
| - | - | traceSamplingPolicySettingsFile | The sampling policy including `sampling rate` and `the threshold of trace segment latency` can be configured by the `traceSamplingPolicySettingsFile` file. | SW_TRACE_SAMPLING_POLICY_SETTINGS_FILE | `trace-sampling-policy-settings.yml` |
| - | - |slowDBAccessThreshold| The slow database access threshold (in milliseconds). |SW_SLOW_DB_THRESHOLD|default:200,mongodb:100|
diff --git a/oap-server-bom/pom.xml b/oap-server-bom/pom.xml
index eb305ceca8..4f55aa2fbe 100644
--- a/oap-server-bom/pom.xml
+++ b/oap-server-bom/pom.xml
@@ -78,7 +78,7 @@
3.0.0
4.4.13
1.21
- 0.12.3
+ 0.12.4
1.6.0
diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml
index aac3acaeaf..dcbcd35d27 100755
--- a/oap-server/server-starter/src/main/resources/application.yml
+++ b/oap-server/server-starter/src/main/resources/application.yml
@@ -258,7 +258,7 @@ storage:
username: ${SW_STORAGE_IOTDB_USERNAME:root}
password: ${SW_STORAGE_IOTDB_PASSWORD:root}
storageGroup: ${SW_STORAGE_IOTDB_STORAGE_GROUP:root.skywalking}
- sessionPoolSize: ${SW_STORAGE_IOTDB_SESSIONPOOL_SIZE:16}
+ sessionPoolSize: ${SW_STORAGE_IOTDB_SESSIONPOOL_SIZE:8} # If it's zero, the SessionPool size will be 2*CPU_Cores
fetchTaskLogMaxSize: ${SW_STORAGE_IOTDB_FETCH_TASK_LOG_MAX_SIZE:1000} # the max number of fetch task log in a request
agent-analyzer:
diff --git a/oap-server/server-storage-plugin/storage-iotdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/iotdb/IoTDBClient.java b/oap-server/server-storage-plugin/storage-iotdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/iotdb/IoTDBClient.java
index ba712a7ed2..969238a42e 100644
--- a/oap-server/server-storage-plugin/storage-iotdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/iotdb/IoTDBClient.java
+++ b/oap-server/server-storage-plugin/storage-iotdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/iotdb/IoTDBClient.java
@@ -67,8 +67,11 @@ public class IoTDBClient implements Client, HealthCheckable {
@Override
public void connect() throws IoTDBConnectionException, StatementExecutionException {
try {
+ final int sessionPoolSize = config.getSessionPoolSize() == 0 ?
+ Runtime.getRuntime().availableProcessors() * 2 : config.getSessionPoolSize();
+ log.info("SessionPool Size: {}", sessionPoolSize);
sessionPool = new SessionPool(config.getHost(), config.getRpcPort(), config.getUsername(),
- config.getPassword(), config.getSessionPoolSize());
+ config.getPassword(), sessionPoolSize, false, false);
sessionPool.setStorageGroup(storageGroup);
healthChecker.health();
@@ -254,7 +257,7 @@ public class IoTDBClient implements Client, HealthCheckable {
/**
* Query with aggregation function: count, sum, avg, last_value, first_value, min_time, max_time, min_value, max_value
*
- * @param querySQL the SQL for query which should contain aggregation function
+ * @param querySQL the SQL for query which should contain aggregation function
* @return the result of aggregation function
* @throws IOException IoTDBConnectionException or StatementExecutionException
*/
diff --git a/oap-server/server-storage-plugin/storage-iotdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/iotdb/base/IoTDBInsertRequest.java b/oap-server/server-storage-plugin/storage-iotdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/iotdb/base/IoTDBInsertRequest.java
index 659a5be029..1b64a30251 100644
--- a/oap-server/server-storage-plugin/storage-iotdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/iotdb/base/IoTDBInsertRequest.java
+++ b/oap-server/server-storage-plugin/storage-iotdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/iotdb/base/IoTDBInsertRequest.java
@@ -61,8 +61,8 @@ public class IoTDBInsertRequest implements InsertRequest, UpdateRequest {
if (index.equals(IoTDBIndexes.ID_IDX)) {
indexValues.add(storageData.id());
} else if (storageMap.containsKey(index)) {
- // avoid `service_group` be "null" when inserting
- if (index.equals(IoTDBIndexes.GROUP_IDX) && storageMap.get(index) == null) {
+ // avoid indexValue be "null" when inserting
+ if (storageMap.get(index) == null) {
indexValues.add("");
} else {
indexValues.add(String.valueOf(storageMap.get(index)));
diff --git a/oap-server/server-storage-plugin/storage-iotdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/iotdb/base/IoTDBRecordDAO.java b/oap-server/server-storage-plugin/storage-iotdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/iotdb/base/IoTDBRecordDAO.java
index c6ae21afd7..712d5abefb 100644
--- a/oap-server/server-storage-plugin/storage-iotdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/iotdb/base/IoTDBRecordDAO.java
+++ b/oap-server/server-storage-plugin/storage-iotdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/iotdb/base/IoTDBRecordDAO.java
@@ -74,9 +74,6 @@ public class IoTDBRecordDAO implements IRecordDAO {
measurementValues.add(rawTag.getValue());
});
}
- request.setMeasurements(measurements);
- request.setMeasurementTypes(measurementTypes);
- request.setMeasurementValues(measurementValues);
return request;
}
}
diff --git a/test/e2e-v2/cases/alarm/iotdb/docker-compose.yml b/test/e2e-v2/cases/alarm/iotdb/docker-compose.yml
index be0a45de52..f00f91fd80 100644
--- a/test/e2e-v2/cases/alarm/iotdb/docker-compose.yml
+++ b/test/e2e-v2/cases/alarm/iotdb/docker-compose.yml
@@ -17,7 +17,7 @@ version: '2.1'
services:
iotdb:
- image: apache/iotdb:0.12.3-node
+ image: apache/iotdb:0.12.4-node
expose:
- 6667
networks:
@@ -33,7 +33,6 @@ services:
file: ../../../script/docker-compose/base-compose.yml
service: oap
environment:
- SW_STORAGE: iotdb
SW_SEARCHABLE_ALARM_TAG_KEYS: level,receivers
ports:
- 12800
diff --git a/test/e2e-v2/cases/event/iotdb/docker-compose.yml b/test/e2e-v2/cases/event/iotdb/docker-compose.yml
index d5639a5069..557238855e 100644
--- a/test/e2e-v2/cases/event/iotdb/docker-compose.yml
+++ b/test/e2e-v2/cases/event/iotdb/docker-compose.yml
@@ -17,7 +17,7 @@ version: '3.8'
services:
iotdb:
- image: apache/iotdb:0.12.3-node
+ image: apache/iotdb:0.12.4-node
expose:
- 6667
networks:
diff --git a/test/e2e-v2/cases/log/iotdb/docker-compose.yml b/test/e2e-v2/cases/log/iotdb/docker-compose.yml
index ac0c160666..32e40745c6 100644
--- a/test/e2e-v2/cases/log/iotdb/docker-compose.yml
+++ b/test/e2e-v2/cases/log/iotdb/docker-compose.yml
@@ -17,7 +17,7 @@ version: '2.1'
services:
iotdb:
- image: apache/iotdb:0.12.3-node
+ image: apache/iotdb:0.12.4-node
expose:
- 6667
networks:
diff --git a/test/e2e-v2/cases/profile/iotdb/docker-compose.yml b/test/e2e-v2/cases/profile/iotdb/docker-compose.yml
index 1292925972..b28c0e1bf9 100644
--- a/test/e2e-v2/cases/profile/iotdb/docker-compose.yml
+++ b/test/e2e-v2/cases/profile/iotdb/docker-compose.yml
@@ -17,7 +17,7 @@ version: '2.1'
services:
iotdb:
- image: apache/iotdb:0.12.3-node
+ image: apache/iotdb:0.12.4-node
expose:
- 6667
networks:
diff --git a/test/e2e-v2/cases/storage/iotdb/docker-compose.yml b/test/e2e-v2/cases/storage/iotdb/docker-compose.yml
index 7626b2f0d5..f22e55db4d 100644
--- a/test/e2e-v2/cases/storage/iotdb/docker-compose.yml
+++ b/test/e2e-v2/cases/storage/iotdb/docker-compose.yml
@@ -17,7 +17,7 @@ version: '2.1'
services:
iotdb:
- image: apache/iotdb:0.12.3-node
+ image: apache/iotdb:0.12.4-node
expose:
- 6667
networks:
diff --git a/test/e2e-v2/cases/ttl/iotdb/docker-compose.yml b/test/e2e-v2/cases/ttl/iotdb/docker-compose.yml
index 131c72febe..39068a15f3 100644
--- a/test/e2e-v2/cases/ttl/iotdb/docker-compose.yml
+++ b/test/e2e-v2/cases/ttl/iotdb/docker-compose.yml
@@ -17,7 +17,7 @@ version: '2.1'
services:
iotdb:
- image: apache/iotdb:0.12.3-node
+ image: apache/iotdb:0.12.4-node
expose:
- 6667
networks:
diff --git a/test/e2e-v2/script/docker-compose/base-compose.yml b/test/e2e-v2/script/docker-compose/base-compose.yml
index 6059c5c564..bcd586ceb0 100644
--- a/test/e2e-v2/script/docker-compose/base-compose.yml
+++ b/test/e2e-v2/script/docker-compose/base-compose.yml
@@ -34,6 +34,7 @@ services:
SW_JDBC_URL: jdbc:mysql://mysql:3306/swtest
SW_STORAGE_INFLUXDB_URL: http://influxdb:8086
SW_STORAGE_IOTDB_HOST: iotdb
+ SW_STORAGE_IOTDB_SESSIONPOOL_SIZE: 2
SW_CONFIG_ETCD_PERIOD: 1
SW_CONFIG_ETCD_ENDPOINTS: http://etcd:2379
SW_CLUSTER_ETCD_ENDPOINTS: http://etcd:2379
diff --git a/tools/dependencies/known-oap-backend-dependencies.txt b/tools/dependencies/known-oap-backend-dependencies.txt
index 2ecdb7e2db..fea02d4526 100755
--- a/tools/dependencies/known-oap-backend-dependencies.txt
+++ b/tools/dependencies/known-oap-backend-dependencies.txt
@@ -63,8 +63,8 @@ httpclient-4.5.13.jar
httpcore-4.4.13.jar
httpcore-nio-4.4.13.jar
influxdb-java-2.15.jar
-iotdb-session-0.12.3.jar
-iotdb-thrift-0.12.3.jar
+iotdb-session-0.12.4.jar
+iotdb-thrift-0.12.4.jar
j2objc-annotations-1.3.jar
jackson-annotations-2.12.2.jar
jackson-core-2.12.2.jar
@@ -144,7 +144,7 @@ protobuf-java-3.19.2.jar
protobuf-java-util-3.19.2.jar
reactive-streams-1.0.2.jar
retrofit-2.5.0.jar
-service-rpc-0.12.3.jar
+service-rpc-0.12.4.jar
simpleclient-0.6.0.jar
simpleclient_common-0.6.0.jar
simpleclient_hotspot-0.6.0.jar
@@ -153,7 +153,7 @@ slf4j-api-1.7.30.jar
snakeyaml-1.28.jar
snappy-java-1.1.7.3.jar
swagger-annotations-1.6.3.jar
-tsfile-0.12.3.jar
+tsfile-0.12.4.jar
vavr-0.10.3.jar
vavr-match-0.10.3.jar
zipkin-2.9.1.jar