From 58352ce6dcdcc49dec0876d707aad505089b107d Mon Sep 17 00:00:00 2001
From: Wan Kai <wankai123@foxmail.com>
Date: Thu, 20 Jul 2023 21:37:10 +0800
Subject: [PATCH] Fix Elasticsearch storage TTL timer, move creation of the
 latest index before retrieval indexes by aliases to avoid the 404 exception.
 (#11119)

---
 docs/en/changes/changes.md                    |  1 +
 .../base/HistoryDeleteEsDAO.java              | 47 ++++++++++---------
 2 files changed, 25 insertions(+), 23 deletions(-)

diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index 391424959d..09de7230c4 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -50,6 +50,7 @@
 * Apply MQE on Virtual-Cache layer UI-templates
 * Add Echo component ID(5015) language: Golang.
 * Fix `index out of bounds exception` in `aggregate_labels` MQE function.
+* Move created the latest index before retrieval indexes by aliases to avoid the 404 exception. This just prevents some interference from manual operations.
 
 #### UI
 
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAO.java
index 06bc7eda12..ebafa6451f 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAO.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAO.java
@@ -63,33 +63,14 @@ public class HistoryDeleteEsDAO extends EsDAO implements IHistoryDeleteDAO {
             }
             return;
         }
-        Collection<String> indices = client.retrievalIndexByAliases(tableName);
 
-        if (log.isDebugEnabled()) {
-            log.debug("Deadline = {}, indices = {}, ttl = {}", deadline, indices, ttl);
-        }
-
-        List<String> prepareDeleteIndexes = new ArrayList<>();
-        List<String> leftIndices = new ArrayList<>();
-        for (String index : indices) {
-            long timeSeries = TimeSeriesUtils.isolateTimeFromIndexName(index);
-            if (deadline >= timeSeries) {
-                prepareDeleteIndexes.add(index);
-            } else {
-                leftIndices.add(index);
-            }
-        }
-        if (log.isDebugEnabled()) {
-            log.debug("Indices to be deleted: {}", prepareDeleteIndexes);
-        }
-        for (String prepareDeleteIndex : prepareDeleteIndexes) {
-            client.deleteByIndexName(prepareDeleteIndex);
-        }
         String latestIndex = TimeSeriesUtils.latestWriteIndexName(model);
-        String formattedLatestIndex = client.formatIndexName(latestIndex);
-        if (!leftIndices.contains(formattedLatestIndex)) {
+        if (!client.isExistsIndex(latestIndex)) {
             try {
                 client.createIndex(latestIndex);
+                if (log.isDebugEnabled()) {
+                    log.debug("Latest index = {} is not exist, create.", latestIndex);
+                }
             } catch (ResponseException e) {
                 if (e.getStatusCode() == 400 && client.isExistsIndex(latestIndex)) {
                     if (log.isDebugEnabled()) {
@@ -101,6 +82,26 @@ public class HistoryDeleteEsDAO extends EsDAO implements IHistoryDeleteDAO {
                 }
             }
         }
+
+        Collection<String> indices = client.retrievalIndexByAliases(tableName);
+
+        if (log.isDebugEnabled()) {
+            log.debug("Deadline = {}, indices = {}, ttl = {}", deadline, indices, ttl);
+        }
+
+        List<String> prepareDeleteIndexes = new ArrayList<>();
+        for (String index : indices) {
+            long timeSeries = TimeSeriesUtils.isolateTimeFromIndexName(index);
+            if (deadline >= timeSeries) {
+                prepareDeleteIndexes.add(index);
+            }
+        }
+        if (log.isDebugEnabled()) {
+            log.debug("Indices to be deleted: {}", prepareDeleteIndexes);
+        }
+        for (String prepareDeleteIndex : prepareDeleteIndexes) {
+            client.deleteByIndexName(prepareDeleteIndex);
+        }
         this.indexLatestSuccess.put(tableName, deadline);
     }
 }