diff --git a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java index 47f0e40d3..337b96b5e 100644 --- a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java +++ b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java @@ -21,6 +21,7 @@ package org.apache.skywalking.oap.server.library.client.elasticsearch; import com.google.gson.*; import java.io.*; import java.util.*; + import org.apache.commons.lang3.StringUtils; import org.apache.http.*; import org.apache.http.auth.*; @@ -121,32 +122,43 @@ public class ElasticSearchClient implements Client { public List retrievalIndexByAliases(String aliases) throws IOException { aliases = formatIndexName(aliases); - Response response = client.getLowLevelClient().performRequest(HttpGet.METHOD_NAME, "/_alias/" + aliases); - - List indexes = new ArrayList<>(); if (HttpStatus.SC_OK == response.getStatusLine().getStatusCode()) { Gson gson = new Gson(); InputStreamReader reader = new InputStreamReader(response.getEntity().getContent()); JsonObject responseJson = gson.fromJson(reader, JsonObject.class); logger.debug("retrieval indexes by aliases {}, response is {}", aliases, responseJson); - indexes.addAll(responseJson.keySet()); + return new ArrayList<>(responseJson.keySet()); } - return indexes; + return Collections.EMPTY_LIST; } - public JsonObject getIndex(String indexName) throws IOException { - indexName = formatIndexName(indexName); - GetIndexRequest request = new GetIndexRequest(); - request.indices(indexName); - Response response = client.getLowLevelClient().performRequest(HttpGet.METHOD_NAME, "/" + indexName); - InputStreamReader reader = new InputStreamReader(response.getEntity().getContent()); - Gson gson = new Gson(); - return gson.fromJson(reader, JsonObject.class); + /** + * If your indexName is retrieved from elasticsearch through {@link #retrievalIndexByAliases(String)} or some other method and it already contains namespace. + * Then you should delete the index by this method, this method will no longer concatenate namespace. + * + * https://github.com/apache/skywalking/pull/3017 + * + */ + public boolean deleteByIndexName(String indexName) throws IOException { + return deleteIndex(indexName, false); } - public boolean deleteIndex(String indexName) throws IOException { - indexName = formatIndexName(indexName); + /** + * If your indexName is obtained from metadata or configuration and without namespace. + * Then you should delete the index by this method, this method automatically concatenates namespace. + * + * https://github.com/apache/skywalking/pull/3017 + * + */ + public boolean deleteByModelName(String modelName) throws IOException { + return deleteIndex(modelName, true); + } + + private boolean deleteIndex(String indexName, boolean formatIndexName) throws IOException { + if (formatIndexName) { + indexName = formatIndexName(indexName); + } DeleteIndexRequest request = new DeleteIndexRequest(indexName); DeleteIndexResponse response; response = client.indices().delete(request); diff --git a/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ITElasticSearchClient.java b/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ITElasticSearchClient.java index f2f6d2884..c0c085499 100644 --- a/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ITElasticSearchClient.java +++ b/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ITElasticSearchClient.java @@ -18,11 +18,17 @@ package org.apache.skywalking.oap.server.library.client.elasticsearch; +import com.google.gson.Gson; import com.google.gson.JsonObject; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.client.methods.HttpGet; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.bulk.BulkProcessor; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilders; @@ -31,11 +37,14 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.powermock.reflect.Whitebox; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.io.InputStreamReader; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -48,10 +57,20 @@ public class ITElasticSearchClient { private ElasticSearchClient client; + private final String namespace; + + public ITElasticSearchClient() { + namespace = ""; + } + + protected ITElasticSearchClient(String namespace) { + this.namespace = namespace; + } + @Before public void before() throws IOException { final String esAddress = System.getProperty("elastic.search.address"); - client = new ElasticSearchClient(esAddress, "", "test", "test"); + client = new ElasticSearchClient(esAddress, namespace, "test", "test"); client.connect(); } @@ -82,7 +101,7 @@ public class ITElasticSearchClient { client.createIndex(indexName, settings, doc); Assert.assertTrue(client.isExistsIndex(indexName)); - JsonObject index = client.getIndex(indexName); + JsonObject index = getIndex(indexName); logger.info(index.toString()); Assert.assertEquals(2, index.getAsJsonObject(indexName).getAsJsonObject("settings").getAsJsonObject("index").get("number_of_shards").getAsInt()); @@ -90,7 +109,7 @@ public class ITElasticSearchClient { Assert.assertEquals("text", index.getAsJsonObject(indexName).getAsJsonObject("mappings").getAsJsonObject("type").getAsJsonObject("properties").getAsJsonObject("column1").get("type").getAsString()); - Assert.assertTrue(client.deleteIndex(indexName)); + Assert.assertTrue(client.deleteByModelName(indexName)); } @Test @@ -157,8 +176,9 @@ public class ITElasticSearchClient { .endObject(); client.forceInsert(indexName + "-2019", "testid", builder); - JsonObject index = client.getIndex(indexName + "-2019"); + JsonObject index = getIndex(indexName + "-2019"); logger.info(index.toString()); + Assert.assertEquals(1, index.getAsJsonObject(indexName + "-2019").getAsJsonObject("settings").getAsJsonObject("index").get("number_of_shards").getAsInt()); Assert.assertEquals(0, index.getAsJsonObject(indexName + "-2019").getAsJsonObject("settings").getAsJsonObject("index").get("number_of_replicas").getAsInt()); @@ -183,4 +203,67 @@ public class ITElasticSearchClient { bulkProcessor.flush(); bulkProcessor.awaitClose(2, TimeUnit.SECONDS); } + + @Test + public void timeSeriesOperate() throws IOException { + String indexName = "test_time_series_operate"; + String timeSeriesIndexName = indexName + "-2019"; + + JsonObject mapping = new JsonObject(); + mapping.add("type", new JsonObject()); + JsonObject doc = mapping.getAsJsonObject("type"); + + JsonObject properties = new JsonObject(); + doc.add("properties", properties); + + JsonObject column = new JsonObject(); + column.addProperty("type", "text"); + properties.add("name", column); + + client.createTemplate(indexName, new JsonObject(), mapping); + + XContentBuilder builder = XContentFactory.jsonBuilder().startObject() + .field("name", "pengys") + .endObject(); + client.forceInsert(timeSeriesIndexName, "testid", builder); + + List indexes = client.retrievalIndexByAliases(indexName); + Assert.assertEquals(1, indexes.size()); + String index = indexes.get(0); + Assert.assertTrue(client.deleteByIndexName(index)); + Assert.assertFalse(client.isExistsIndex(timeSeriesIndexName)); + } + + private JsonObject getIndex(String indexName) throws IOException { + indexName = client.formatIndexName(indexName); + GetIndexRequest request = new GetIndexRequest(); + request.indices(indexName); + + Response response = getRestHighLevelClient().getLowLevelClient().performRequest(HttpGet.METHOD_NAME, "/" + indexName); + InputStreamReader reader = new InputStreamReader(response.getEntity().getContent()); + Gson gson = new Gson(); + return undoFormatIndexName(gson.fromJson(reader, JsonObject.class)); + } + + private RestHighLevelClient getRestHighLevelClient() { + return (RestHighLevelClient) Whitebox.getInternalState(client, "client"); + } + + private JsonObject undoFormatIndexName(JsonObject index) { + if (StringUtils.isNotEmpty(namespace) && index != null && index.size() > 0) { + logger.info("UndoFormatIndexName before " + index.toString()); + String namespacePrefix = namespace + "_"; + index.entrySet().forEach(entry -> { + String oldIndexName = entry.getKey(); + if (oldIndexName.startsWith(namespacePrefix)) { + index.add(oldIndexName.substring(namespacePrefix.length()), entry.getValue()); + index.remove(oldIndexName); + } else { + throw new RuntimeException("The indexName must contain the " + namespace + " prefix, but it is " + entry.getKey()); + } + }); + logger.info("UndoFormatIndexName after " + index.toString()); + } + return index; + } } diff --git a/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ITElasticSearchClientOfNamespace.java b/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ITElasticSearchClientOfNamespace.java new file mode 100644 index 000000000..1c11ca477 --- /dev/null +++ b/oap-server/server-library/library-client/src/test/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ITElasticSearchClientOfNamespace.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.client.elasticsearch; + +/** + * @author: zhangwei + */ +public class ITElasticSearchClientOfNamespace extends ITElasticSearchClient { + + + public ITElasticSearchClientOfNamespace() { + super("test"); + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAO.java index b2a273f6b..1299cc673 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAO.java @@ -69,7 +69,7 @@ public class HistoryDeleteEsDAO extends EsDAO implements IHistoryDeleteDAO { } for (String prepareDeleteIndex : prepareDeleteIndexes) { - client.deleteIndex(prepareDeleteIndex); + client.deleteByIndexName(prepareDeleteIndex); } } else { int statusCode = client.delete(model.getName(), timeBucketColumnName, timeBefore); diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/lock/RegisterLockInstaller.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/lock/RegisterLockInstaller.java index f34aeece8..26342033c 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/lock/RegisterLockInstaller.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/lock/RegisterLockInstaller.java @@ -64,7 +64,7 @@ public class RegisterLockInstaller { } private void deleteIndex() throws IOException { - client.deleteIndex(RegisterLockIndex.NAME); + client.deleteByModelName(RegisterLockIndex.NAME); } private void createIndex() throws IOException {