Do not log error logs when failed to create ElasticSearch index because the index is created already (#9856)

docs/en/changes/changes.md

@@ -69,8 +69,9 @@
 * [**Breaking Change**] Support new records query protocol, rename the column named `service_id` to `entity_id` for support difference entity.
   Please re-create `top_n_database_statement` index/table.
 * Remove improper self-obs metrics in JvmMetricsHandler(for Kafka channel).
-* gRPC stream canceling code is not logged as an error when the client cancels the stream. The client 
+* gRPC stream canceling code is not logged as an error when the client cancels the stream. The client
   cancels the stream when the pod is terminated.
+* Do not log error logs when failed to create ElasticSearch index because the index is created already.
 
 #### UI
 

oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/client/IndexClient.java

@@ -30,6 +30,7 @@ import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.skywalking.library.elasticsearch.ElasticSearchVersion;
+import org.apache.skywalking.library.elasticsearch.exception.ResponseException;
 import org.apache.skywalking.library.elasticsearch.response.Index;
 import org.apache.skywalking.library.elasticsearch.response.Mappings;
 
@@ -108,7 +109,7 @@ public final class IndexClient {
                     if (response.status() == HttpStatus.OK) {
                         return true;
                     }
-                    throw new RuntimeException(response.contentUtf8());
+                    throw new ResponseException(response.contentUtf8(), response.status().code());
                 }));
         future.whenComplete((result, exception) -> {
             if (exception != null) {

oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/exception/ResponseException.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.skywalking.library.elasticsearch.exception;
+
+import lombok.Getter;
+
+@Getter
+public class ResponseException extends RuntimeException {
+    private final int statusCode;
+
+    public ResponseException(String message, int statusCode) {
+        super(message);
+
+        this.statusCode = statusCode;
+    }
+}

oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAO.java

@@ -24,6 +24,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.skywalking.library.elasticsearch.exception.ResponseException;
 import org.apache.skywalking.oap.server.core.analysis.DownSampling;
 import org.apache.skywalking.oap.server.core.storage.IHistoryDeleteDAO;
 import org.apache.skywalking.oap.server.core.storage.model.Model;
@@ -87,7 +88,18 @@ public class HistoryDeleteEsDAO extends EsDAO implements IHistoryDeleteDAO {
         String latestIndex = TimeSeriesUtils.latestWriteIndexName(model);
         String formattedLatestIndex = client.formatIndexName(latestIndex);
         if (!leftIndices.contains(formattedLatestIndex)) {
-            client.createIndex(latestIndex);
+            try {
+                client.createIndex(latestIndex);
+            } catch (ResponseException e) {
+                if (e.getStatusCode() == 400 && client.isExistsIndex(latestIndex)) {
+                    if (log.isDebugEnabled()) {
+                        log.debug(
+                            "Failed to create index {}, index is already created.", latestIndex);
+                    }
+                } else {
+                    throw e;
+                }
+            }
         }
         this.indexLatestSuccess.put(tableName, deadline);
     }