From f7da99d6324b44bfdd5d83915512245887d5f827 Mon Sep 17 00:00:00 2001 From: Wan Kai Date: Mon, 10 Mar 2025 11:01:32 +0800 Subject: [PATCH] Add more model installation log info for OAP storage initialization. (#13094) --- docs/en/changes/changes.md | 3 +- .../core/storage/model/ModelInstaller.java | 60 ++++++++++++++---- .../banyandb/BanyanDBIndexInstaller.java | 62 ++++++++++++++++--- .../base/StorageEsInstaller.java | 59 ++++++++++++++++-- .../jdbc/common/JDBCTableInstaller.java | 41 ++++++++++-- 5 files changed, 194 insertions(+), 31 deletions(-) diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 4e4272b310..fa05f8d7f2 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -77,7 +77,8 @@ * Optimize metrics cache loading when trace latency greater than cache timeout. * Allow calling `lang.groovy.GString` in DSL. * BanyanDB: fix alarm query result without sort. -* Add a component ID for Virtual thread executor +* Add a component ID for Virtual thread executor. +* Add more model installation log info for OAP storage initialization. #### UI diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelInstaller.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelInstaller.java index 5c67abb498..016b90b325 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelInstaller.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelInstaller.java @@ -18,7 +18,9 @@ package org.apache.skywalking.oap.server.core.storage.model; +import lombok.Getter; import lombok.RequiredArgsConstructor; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.RunningMode; @@ -38,20 +40,29 @@ public abstract class ModelInstaller implements ModelCreator.CreatingListener { @Override public void whenCreating(Model model) throws StorageException { if (RunningMode.isNoInitMode()) { - while (!isExists(model)) { - try { - log.info( - "table: {} does not exist. OAP is running in 'no-init' mode, waiting... retry 3s later.", - model.getName() - ); - Thread.sleep(3000L); - } catch (InterruptedException e) { - log.error(e.getMessage()); + while (true) { + InstallInfo info = isExists(model); + if (!info.isAllExist()) { + try { + log.info( + "install info: {}.table for model: [{}] not all required resources exist. OAP is running in 'no-init' mode, waiting create or update... retry 3s later.", + info.buildInstallInfoMsg(), model.getName() + ); + Thread.sleep(3000L); + } catch (InterruptedException e) { + log.error(e.getMessage()); + } + } else { + break; } } } else { - if (!isExists(model)) { - log.info("table: {} does not exist", model.getName()); + InstallInfo info = isExists(model); + if (!info.isAllExist()) { + log.info( + "install info: {}. table for model: [{}] not all required resources exist, creating or updating...", + info.buildInstallInfoMsg(), model.getName() + ); createTable(model); } } @@ -74,10 +85,35 @@ public abstract class ModelInstaller implements ModelCreator.CreatingListener { /** * Check whether the storage entity exists. Need to implement based on the real storage. */ - public abstract boolean isExists(Model model) throws StorageException; + public abstract InstallInfo isExists(Model model) throws StorageException; /** * Create the storage entity. All creations should be after the {@link #isExists(Model)} check. */ public abstract void createTable(Model model) throws StorageException; + + @Getter + @Setter + public abstract static class InstallInfo { + private final String modelName; + private final boolean timeSeries; + private final boolean superDataset; + private final String modelType; + private boolean allExist; + + protected InstallInfo(Model model) { + this.modelName = model.getName(); + this.timeSeries = model.isTimeSeries(); + this.superDataset = model.isSuperDataset(); + if (model.isMetric()) { + this.modelType = "metric"; + } else if (model.isRecord()) { + this.modelType = "record"; + } else { + this.modelType = "unknown"; + } + } + + public abstract String buildInstallInfoMsg(); + } } diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBIndexInstaller.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBIndexInstaller.java index 12ad68f420..806903b99f 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBIndexInstaller.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBIndexInstaller.java @@ -25,6 +25,8 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.banyandb.common.v1.BanyandbCommon; import org.apache.skywalking.banyandb.common.v1.BanyandbCommon.Group; @@ -41,6 +43,7 @@ import org.apache.skywalking.banyandb.v1.client.metadata.MetadataCache; import org.apache.skywalking.banyandb.v1.client.metadata.ResourceExist; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.RunningMode; +import org.apache.skywalking.oap.server.core.analysis.DownSampling; import org.apache.skywalking.oap.server.core.config.DownSamplingConfigService; import org.apache.skywalking.oap.server.core.storage.StorageException; import org.apache.skywalking.oap.server.core.storage.model.Model; @@ -62,21 +65,31 @@ public class BanyanDBIndexInstaller extends ModelInstaller { } @Override - public boolean isExists(Model model) throws StorageException { + public InstallInfo isExists(Model model) throws StorageException { + InstallInfoBanyanDB installInfo = new InstallInfoBanyanDB(model); + installInfo.setDownSampling(model.getDownsampling()); if (!model.isTimeSeries()) { - return true; + installInfo.setTableName(model.getName()); + installInfo.setAllExist(true); + return installInfo; } final DownSamplingConfigService downSamplingConfigService = moduleManager.find(CoreModule.NAME) .provider() .getService(DownSamplingConfigService.class); final MetadataRegistry.SchemaMetadata metadata = MetadataRegistry.INSTANCE.parseMetadata( model, config, downSamplingConfigService); + installInfo.setTableName(metadata.name()); + installInfo.setKind(metadata.getKind()); + installInfo.setGroup(metadata.getGroup()); try { final BanyanDBClient c = ((BanyanDBStorageClient) this.client).client; // first check resource existence and create group if necessary - final boolean resourceExist = checkResourceExistence(metadata, c); - if (!resourceExist) { - return false; + final ResourceExist resourceExist = checkResourceExistence(metadata, c); + installInfo.setGroupExist(resourceExist.hasGroup()); + installInfo.setTableExist(resourceExist.hasResource()); + if (!resourceExist.hasResource()) { + installInfo.setAllExist(false); + return installInfo; } else { // register models only locally(Schema cache) but not remotely if (model.isRecord()) { // stream @@ -108,7 +121,8 @@ public class BanyanDBIndexInstaller extends ModelInstaller { if (remoteMeta == null) { throw new IllegalStateException("inconsistent state: metadata:" + metadata + ", remoteMeta: null"); } - return true; + installInfo.setAllExist(true); + return installInfo; } } catch (BanyanDBException ex) { throw new StorageException("fail to check existence", ex); @@ -206,7 +220,7 @@ public class BanyanDBIndexInstaller extends ModelInstaller { || g.getResourceOpts().getTtl().getNum() != metadata.getTtlDays(); } - private boolean checkResourceExistence(MetadataRegistry.SchemaMetadata metadata, + private ResourceExist checkResourceExistence(MetadataRegistry.SchemaMetadata metadata, BanyanDBClient client) throws BanyanDBException { ResourceExist resourceExist; Group.Builder gBuilder @@ -265,7 +279,7 @@ public class BanyanDBIndexInstaller extends ModelInstaller { groupAligned.add(metadata.getGroup()); } } - return resourceExist.hasResource(); + return resourceExist; } /** @@ -559,4 +573,36 @@ public class BanyanDBIndexInstaller extends ModelInstaller { } } } + + @Getter + @Setter + private static class InstallInfoBanyanDB extends InstallInfo { + private DownSampling downSampling; + private String tableName; + private MetadataRegistry.Kind kind; + private String group; + private boolean tableExist; + private boolean groupExist; + + protected InstallInfoBanyanDB(Model model) { + super(model); + } + + @Override + public String buildInstallInfoMsg() { + return "InstallInfoBanyanDB{" + + "modelName=" + getModelName() + + ", modelType=" + getModelType() + + ", timeSeries=" + isTimeSeries() + + ", superDataset=" + isSuperDataset() + + ", downSampling=" + downSampling.name() + + ", tableName=" + tableName + + ", kind=" + kind.name() + + ", group=" + group + + ", allResourcesExist=" + isAllExist() + + " [groupExist=" + groupExist + + ", tableExist=" + tableExist + + "]}"; + } + } } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java index 7024929332..7d0a577621 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java @@ -21,6 +21,7 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Strings; import com.google.gson.Gson; +import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.library.elasticsearch.response.Index; @@ -81,12 +82,15 @@ public class StorageEsInstaller extends ModelInstaller { } @Override - public boolean isExists(Model model) throws StorageException { + public InstallInfo isExists(Model model) throws StorageException { + InstallInfoES installInfo = new InstallInfoES(model, config); ElasticSearchClient esClient = (ElasticSearchClient) client; String tableName = IndexController.INSTANCE.getTableName(model); IndexController.LogicIndicesRegister.registerRelation(model, tableName); + installInfo.setTableName(esClient.formatIndexName(tableName)); if (!model.isTimeSeries()) { boolean exist = esClient.isExistsIndex(tableName); + installInfo.setTableExist(exist); if (exist) { Optional index = esClient.getIndex(tableName); Mappings historyMapping = index.map(Index::getMappings).orElseGet(Mappings::new); @@ -97,15 +101,21 @@ public class StorageEsInstaller extends ModelInstaller { // or updating field types, it just cares about whether the data can be ingested without // reporting errors. exist = structures.containsFieldNames(tableName, createMapping(model)); + installInfo.setAllFieldsExist(exist); } else { boolean containsMapping = structures.containsMapping(tableName, createMapping(model)); - exist = containsMapping && structures.compareIndexSetting(tableName, createSetting(model)); + installInfo.setAllFieldsExist(containsMapping); + boolean containsSetting = structures.compareIndexSetting(tableName, createSetting(model)); + installInfo.setAllIndexSettingsExist(containsSetting); + exist = containsMapping && containsSetting; } } - return exist; + installInfo.setAllExist(exist); + return installInfo; } boolean templateExists = esClient.isExistsTemplate(tableName); + installInfo.setTableExist(templateExists); final Optional template = esClient.getTemplate(tableName); if ((templateExists && template.isEmpty()) || (!templateExists && template.isPresent())) { @@ -123,12 +133,17 @@ public class StorageEsInstaller extends ModelInstaller { // because the no-init mode OAP server doesn't take responsibility for index settings. if (RunningMode.isNoInitMode()) { exist = structures.containsFieldNames(tableName, createMapping(model)); + installInfo.setAllFieldsExist(exist); } else { boolean containsMapping = structures.containsMapping(tableName, createMapping(model)); - exist = containsMapping && structures.compareIndexSetting(tableName, createSetting(model)); + installInfo.setAllFieldsExist(containsMapping); + boolean containsSetting = structures.compareIndexSetting(tableName, createSetting(model)); + installInfo.setAllIndexSettingsExist(containsSetting); + exist = containsMapping && containsSetting; } } - return exist; + installInfo.setAllExist(exist); + return installInfo; } @Override @@ -373,4 +388,38 @@ public class StorageEsInstaller extends ModelInstaller { return mappings; } + + @Getter + @Setter + public static class InstallInfoES extends InstallInfo { + private String tableName; + private boolean tableExist; + private boolean allFieldsExist; + private boolean allIndexSettingsExist; + private StorageModuleElasticsearchConfig config; + + protected InstallInfoES(Model model, StorageModuleElasticsearchConfig config) { + super(model); + this.config = config; + } + + @Override + public String buildInstallInfoMsg() { + String tableNameMsg = isTimeSeries() ? "indexTemplateName=" + tableName : "indexName=" + tableName; + String tableExistMsg = isTimeSeries() ? "indexTemplateExists=" + tableExist : "indexExists=" + tableExist; + return "InstallInfoES:{" + + "modelName=" + getModelName() + + ", modelType=" + getModelType() + + ", timeSeries=" + isTimeSeries() + + ", superDataset=" + isSuperDataset() + + ", logicSharding=" + config.isLogicSharding() + + ", indexNamespace=" + config.getNamespace() + + ", " + tableNameMsg + + ", allResourcesExist=" + isAllExist() + + " [" + tableExistMsg + + ", allFieldsExist=" + allFieldsExist + + ", allIndexSettingsExist=" + allIndexSettingsExist + + "]}"; + } + } } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCTableInstaller.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCTableInstaller.java index c41eb58363..d6560cf6f5 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCTableInstaller.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCTableInstaller.java @@ -19,6 +19,8 @@ package org.apache.skywalking.oap.server.storage.plugin.jdbc.common; import com.google.gson.JsonObject; +import lombok.Getter; +import lombok.Setter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.oap.server.core.analysis.DownSampling; @@ -64,14 +66,16 @@ public class JDBCTableInstaller extends ModelInstaller { @Override @SneakyThrows - public boolean isExists(Model model) { + public InstallInfo isExists(Model model) { + InstallInfoJDBC installInfo = new InstallInfoJDBC(model); TableMetaInfo.addModel(model); final var table = TableHelper.getLatestTableForWrite(model); - + installInfo.setTableName(table); final var jdbcClient = (JDBCClient) client; if (!jdbcClient.tableExists(table)) { - return false; + installInfo.setAllExist(false); + return installInfo; } final var databaseColumns = getDatabaseColumns(table); @@ -81,8 +85,9 @@ public class JDBCTableInstaller extends ModelInstaller { .map(ModelColumn::getColumnName) .map(ColumnName::getStorageName) .anyMatch(not(databaseColumns::contains)); - - return !isAnyColumnNotCreated; + installInfo.setAllColumnsExist(isAnyColumnNotCreated); + installInfo.setAllExist(!isAnyColumnNotCreated); + return installInfo; } @Override @@ -277,4 +282,30 @@ public class JDBCTableInstaller extends ModelInstaller { executeSQL(sql); } + + @Getter + @Setter + private static class InstallInfoJDBC extends InstallInfo { + private String tableName; + private boolean tableExist; + private boolean allColumnsExist; + + protected InstallInfoJDBC(Model model) { + super(model); + } + + @Override + public String buildInstallInfoMsg() { + return "InstallInfoJDBC{" + + "modelName=" + getModelName() + + ", modelType=" + getModelType() + + ", timeSeries=" + isTimeSeries() + + ", superDataset=" + isSuperDataset() + + ", tableName=" + tableName + + ", allResourcesExist=" + isAllExist() + + " [tableExist=" + tableExist + + ", allColumnsExist=" + allColumnsExist + + "]}"; + } + } }