/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
package org.elasticsearch.indices;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
import org.apache.lucene.util.automaton.Operations;
import org.apache.lucene.util.automaton.RegExp;
import org.elasticsearch.action.admin.indices.create.AutoCreateAction;
import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.ProjectMetadata;
import org.elasticsearch.cluster.metadata.SystemIndexMetadataUpgradeService;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.indices.system.IndexPatternMatcher;
import org.elasticsearch.indices.system.SystemResourceDescriptor;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xcontent.ToXContent;
import org.elasticsearch.xcontent.XContentBuilder;
import org.elasticsearch.xcontent.XContentType;
import org.elasticsearch.xcontent.json.JsonXContent;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import static org.apache.lucene.util.automaton.Operations.DEFAULT_DETERMINIZE_WORK_LIMIT;
/**
* Uses a pattern string to define a protected space for indices belonging to a system feature, and, if needed, provides metadata for
* managing indices that match the pattern.
*
* Any index name that matches a descriptor’s index pattern belongs to the descriptor. For example, if a descriptor had a pattern of
* {@code ".example-index-*"}, then indices named {@code ".example-index-1"}, {@code ".example-index-reindex"}, and {@code
* ".example-index-old"} would all belong to the descriptor. If a node gains a new system index descriptor after an upgrade, then matching
* indices will automatically be marked as system indices (see {@link SystemIndexMetadataUpgradeService}).
*
*
SystemIndexDescriptor index patterns must begin with a "{@code .}" character. Index patterns also have a "gotcha": the pattern
* definitions may look like the standard Elasticsearch multi-target syntax but the underlying implementation is different. Index
* patterns are actually defined in a mangled regex syntax where "{@code .}" is always interpreted as a literal character and "{@code *}"
* is expanded to "{@code .*}". We don’t support date math or the "-" operator from the
* {@link org.elasticsearch.cluster.metadata.IndexNameExpressionResolver} class. We intend for developers to use only the "{@code *}",
* "{@code +}", "{@code ~}" (complement), "{@code (}", "{@code )}", and character class operators, but because of the implementation,
* other regex operators probably work.
*
*
Sample index patterns that we want to handle:
*
* - {@code .system-*} - covers all index names beginning with ".system-".
*
- {@code .system-[0-9]+} - covers all index names beginning with ".system-" and containing only one or more numerals after that
*
- {@code .system-~(other-*)} - covers all system indices beginning with ".system-", except for those beginning with
* ".system-other-"
*
*
* The descriptor defines which, if any, Elasticsearch products are expected to read or modify it with calls to the REST API.
* Requests that do not include the correct product header should, in most cases, generate deprecation warnings. The exception is for
* "net new" system index descriptors, described below.
*
*
The descriptor also provides names for the thread pools that Elasticsearch should use to read, search, or modify the descriptor’s
* indices.
*
*
A SystemIndexDescriptor may be one of several types (see {@link SystemIndexDescriptor.Type}). The four types come from two different
* distinctions. The first is between "internal" and "external" system indices. The second is between "managed and unmanaged" system
* indices. The "internal/external" distinction is simple. Access to internal system indices via standard index APIs is deprecated,
* and system features that use internal system indices should provide any necessary APIs for operating on their state. An "external"
* system index, on the other hand, does not deprecate the use of standard index APIs.
*
*
The distinction between managed and unmanaged is simple in theory but not observed very well in our code. A "managed" system index
* is one whose settings, mappings, and aliases are defined by the SystemIndexDescriptor and managed by Elasticsearch. Many of the
* fields in this class, when added, were meant to be used only by managed system indices, and use of them should always be
* conditional on whether the system index is managed or not. However, we have not consistently enforced this, so our code may have
* inconsistent expectations about what fields will be defined for an unmanaged index. (In the future, we should refactor so that it
* is clear which fields are ignored by unmanaged system indices.)
*
*
A managed system index defines a "primary index" which is intended to be the main write index for the descriptor. The current
* behavior when creating a non-primary index is a little strange. A request to create a non-primary index with the Create Index
* API will fail. (See PR #86707) However, auto-creating the index by
* writing a document to it will succeed. (See PR #77045)
*
*
The mappings for managed system indices are automatically upgraded when all nodes in the cluster are compatible with the
* descriptor's mappings. See {@link SystemIndexMappingUpdateService} for details.
* When the mappings change add the previous index descriptors with
* {@link SystemIndexDescriptor.Builder#setPriorSystemIndexDescriptors(List)}. In a mixed cluster setting this enables auto creation
* of the index with compatible mappings.
*
*
We hope to remove the currently deprecated forms of access to system indices in a future release. A newly added system index with
* no backwards-compatibility requirements may opt into our desired behavior by setting isNetNew to true. A "net new system index"
* strictly enforces its allowed product origins, and cannot be accessed by any REST API request that lacks a correct product header.
* A system index that is fully internal to Elasticsearch will not allow any product origins; such an index is fully "locked down,"
* and in general can only be changed by restoring feature states from snapshots.
*/
public class SystemIndexDescriptor implements IndexPatternMatcher, SystemResourceDescriptor, Comparable {
public static final Settings DEFAULT_SETTINGS = Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build();
/**
* A pattern, either with a wildcard or simple regex. Indices that match one of these patterns are considered system indices.
* Note that this pattern must not overlap with any other {@link SystemIndexDescriptor}s and must allow an alphanumeric suffix
* (see {@link SystemIndices#UPGRADED_INDEX_SUFFIX} for the specific suffix that's checked) to ensure that there's a name within the
* pattern we can use to create a new index when upgrading.
* */
private final String indexPattern;
/**
* For internally-managed indices, specifies the name of the concrete index to create and update. This is required
* since the {@link #indexPattern} can match many indices.
*/
private final String primaryIndex;
/** A description of the index or indices */
private final String description;
/** Used to determine whether an index name matches the {@link #indexPattern} */
private final CharacterRunAutomaton indexPatternAutomaton;
/** For internally-managed indices, contains the index mappings JSON */
private final String mappings;
/** For internally-managed indices, contains the index settings */
private final Settings settings;
/** For internally-managed indices, an optional alias to create */
private final String aliasName;
/** For internally-managed indices, an optional {@link IndexMetadata#INDEX_FORMAT_SETTING} value to expect */
private final int indexFormat;
/** The version meta key for the integer system index mapping version */
public static final String VERSION_META_KEY = "managed_index_mappings_version";
/** For internally-managed indices, specifies the origin to use when creating or updating the index */
private final String origin;
/**
* An optional reindexing script to use when migrating an index created
* before {@link SystemIndices#NO_UPGRADE_REQUIRED_INDEX_VERSION}.
* This script can be used to modify documents before they are added to the new index.
* For example, it can be used to remove deprecated fields from the index.
*
* Note: the script usually should only exist in the versions supporting migration to the next major release -
* specifically, the last (two) minors of the current major.
* It should be created once the last minor branch has diverged from the next major branch (main).
* This ensures the script is available only in the versions where it is needed
* and avoids removing and maintaining it in the next major branch.
* For example: In order to migrate an index created in v7 when upgrading to v9,
* the script should be in the v8 minors supporting upgrade to v9 - 8.18 and 8.19.
*
* See: Reindex scripts
*/
private final String migrationScript;
/** Mapping version from the descriptor */
private final MappingsVersion mappingsVersion;
/** Whether there are dynamic fields in this descriptor's mappings */
private final boolean hasDynamicMappings;
/** The {@link Type} of system index this descriptor represents */
private final Type type;
/** A list of allowed product origins that may access an external system index */
private final List allowedElasticProductOrigins;
/**
* A list of prior system index descriptors that can be used when one or more data/master nodes is on a version lower than the
* minimum supported version for this descriptor
*/
private final List priorSystemIndexDescriptors;
/**
* A system index that is not net-new allows deprecated access from API requests, but issues warnings. A net-new system
* index has the desired future behavior of only being accessible by requests with the correct product origin header.
*/
private final boolean isNetNew;
/**
* We typically don't want to apply user defined templates on system indices, since they may have unexpected
* behaviour when upgrading Elasticsearch versions. Currently, only the .kibana_ indices use templates, so we
* are making this property by default as false.
*/
private final boolean allowsTemplates;
/**
* The thread pools that actions will use to operate on this descriptor's system indices
*/
private final ExecutorNames executorNames;
/**
* Creates a descriptor for system indices matching the supplied pattern. These indices will be managed
* by Elasticsearch internally if mappings or settings are provided.
*
* @param indexPattern The pattern of index names that this descriptor will be used for. Must start with a '.' character, must not
* overlap with any other descriptor patterns, and must allow a suffix (see note on
* {@link SystemIndexDescriptor#indexPattern} for details).
* @param primaryIndex The primary index name of this descriptor. Used when creating the system index for the first time.
* @param description The name of the plugin responsible for this system index.
* @param mappings The mappings to apply to this index when auto-creating, if appropriate
* @param settings The settings to apply to this index when auto-creating, if appropriate
* @param aliasName An alias for the index, or null
* @param indexFormat A value for the `index.format` setting. Pass 0 or higher.
* @param origin the client origin to use when creating this index. Internal system indices must not provide an origin, while external
* system indices must do so.
* @param type The {@link Type} of system index
* @param migrationScript The script to apply when migrating this system index, or null
* @param allowedElasticProductOrigins A list of allowed origin values that should be allowed access in the case of external system
* indices
* @param priorSystemIndexDescriptors A list of system index descriptors that describe the same index in a way that is compatible with
* older versions of Elasticsearch
* @param allowsTemplates if this system index descriptor allows templates to affect its settings (e.g. .kibana_ indices)
*/
protected SystemIndexDescriptor(
String indexPattern,
String primaryIndex,
String description,
String mappings,
Settings settings,
String aliasName,
int indexFormat,
String origin,
String migrationScript,
Type type,
List allowedElasticProductOrigins,
List priorSystemIndexDescriptors,
ExecutorNames executorNames,
boolean isNetNew,
boolean allowsTemplates
) {
Objects.requireNonNull(indexPattern, "system index pattern must not be null");
if (indexPattern.length() < 2) {
throw new IllegalArgumentException(
"system index pattern provided as [" + indexPattern + "] but must at least 2 characters in length"
);
}
if (indexPattern.charAt(0) != '.') {
throw new IllegalArgumentException(
"system index pattern provided as [" + indexPattern + "] but must start with the character [.]"
);
}
if (indexPattern.charAt(1) == '*') {
throw new IllegalArgumentException(
"system index pattern provided as ["
+ indexPattern
+ "] but must not start with the character sequence [.*] to prevent conflicts"
);
}
if (primaryIndex != null) {
if (primaryIndex.charAt(0) != '.') {
throw new IllegalArgumentException(
"system primary index provided as [" + primaryIndex + "] but must start with the character [.]"
);
}
if (primaryIndex.matches("^\\.[\\w-]+$") == false) {
throw new IllegalArgumentException(
"system primary index provided as [" + primaryIndex + "] but cannot contain special characters or patterns"
);
}
}
if (indexFormat < 0) {
throw new IllegalArgumentException("Index format cannot be negative");
}
Strings.requireNonEmpty(indexPattern, "indexPattern must be supplied");
Objects.requireNonNull(type, "type must not be null");
if (type.isManaged()) {
Objects.requireNonNull(settings, "Must supply settings for a managed system index");
Strings.requireNonEmpty(mappings, "Must supply mappings for a managed system index");
Strings.requireNonEmpty(primaryIndex, "Must supply primaryIndex for a managed system index");
Strings.requireNonEmpty(origin, "Must supply origin for a managed system index");
if (settings.getAsInt(IndexMetadata.INDEX_FORMAT_SETTING.getKey(), 0) != indexFormat) {
throw new IllegalArgumentException("Descriptor index format does not match index format in managed settings");
}
this.mappingsVersion = extractVersionFromMappings(mappings);
assert mappingsVersion.version >= 0 : "The mappings version must not be negative";
} else {
assert Objects.isNull(settings) : "Unmanaged index descriptors should not have settings";
assert Objects.isNull(mappings) : "Unmanaged index descriptors should not have mappings";
assert Objects.isNull(primaryIndex) : "Unmanaged index descriptors should not have a primary index";
this.mappingsVersion = null;
}
Objects.requireNonNull(allowedElasticProductOrigins, "allowedProductOrigins must not be null");
if (type.isExternal() == false && allowedElasticProductOrigins.isEmpty() == false) {
throw new IllegalArgumentException("Allowed origins are not valid for internal system indices");
} else if (type.isExternal() && allowedElasticProductOrigins.isEmpty()) {
throw new IllegalArgumentException("External system indices without allowed products is not a valid combination");
}
Objects.requireNonNull(priorSystemIndexDescriptors, "priorSystemIndexDescriptors must not be null");
if (priorSystemIndexDescriptors.isEmpty() == false) {
// the rules for prior system index descriptors
// 1. No values with the same minimum mappings version
// 2. All prior system index descriptors must have a lower mappings version
// 3. Prior system index descriptors may not have other prior system index descriptors
// to avoid multiple branches that need followed
// 4. Must have same indexPattern, primaryIndex, and alias
Set versions = Sets.newHashSetWithExpectedSize(priorSystemIndexDescriptors.size() + 1);
versions.add(mappingsVersion);
for (SystemIndexDescriptor prior : priorSystemIndexDescriptors) {
if (versions.add(prior.mappingsVersion) == false) {
throw new IllegalArgumentException(prior + " has the same mappings version as another descriptor");
}
if (prior.mappingsVersion.version() > mappingsVersion.version()) {
throw new IllegalArgumentException(
prior
+ " has mappings version ["
+ prior.mappingsVersion.version()
+ "] which is after ["
+ mappingsVersion.version()
+ "]"
);
}
if (prior.priorSystemIndexDescriptors.isEmpty() == false) {
throw new IllegalArgumentException(prior + " has its own prior descriptors but only a depth of 1 is allowed");
}
if (prior.indexPattern.equals(indexPattern) == false) {
throw new IllegalArgumentException("index pattern must be the same");
}
if (prior.primaryIndex.equals(primaryIndex) == false) {
throw new IllegalArgumentException("primary index must be the same");
}
if (Objects.equals(prior.aliasName, aliasName) == false) {
throw new IllegalArgumentException("alias name must be the same");
}
}
}
if (Objects.nonNull(executorNames)) {
if (ThreadPool.THREAD_POOL_TYPES.containsKey(executorNames.threadPoolForGet()) == false) {
throw new IllegalArgumentException(executorNames.threadPoolForGet() + " is not a valid thread pool");
}
if (ThreadPool.THREAD_POOL_TYPES.containsKey(executorNames.threadPoolForSearch()) == false) {
throw new IllegalArgumentException(executorNames.threadPoolForGet() + " is not a valid thread pool");
}
if (ThreadPool.THREAD_POOL_TYPES.containsKey(executorNames.threadPoolForWrite()) == false) {
throw new IllegalArgumentException(executorNames.threadPoolForGet() + " is not a valid thread pool");
}
}
this.indexPattern = indexPattern;
this.primaryIndex = primaryIndex;
this.aliasName = aliasName;
final Automaton automaton = Operations.determinize(buildAutomaton(indexPattern, aliasName), DEFAULT_DETERMINIZE_WORK_LIMIT);
this.indexPatternAutomaton = new CharacterRunAutomaton(automaton);
if (primaryIndex != null && indexPatternAutomaton.run(primaryIndex) == false) {
throw new IllegalArgumentException("primary index does not match the index pattern!");
}
this.description = description;
this.mappings = mappings;
this.migrationScript = migrationScript;
settings = Objects.isNull(settings) ? Settings.EMPTY : settings;
if (settings.hasValue(IndexMetadata.SETTING_INDEX_HIDDEN) == false) {
settings = Settings.builder().put(settings).put(DEFAULT_SETTINGS).build();
}
if (settings.getAsBoolean(IndexMetadata.SETTING_INDEX_HIDDEN, false)) {
this.settings = settings;
} else {
throw new IllegalArgumentException("System indices must have " + IndexMetadata.SETTING_INDEX_HIDDEN + " set to true.");
}
this.indexFormat = indexFormat;
this.origin = origin;
this.type = type;
this.allowedElasticProductOrigins = allowedElasticProductOrigins;
this.hasDynamicMappings = this.mappings != null
&& findDynamicMapping(XContentHelper.convertToMap(JsonXContent.jsonXContent, mappings, false));
final List sortedPriorSystemIndexDescriptors;
if (priorSystemIndexDescriptors.isEmpty() || priorSystemIndexDescriptors.size() == 1) {
sortedPriorSystemIndexDescriptors = List.copyOf(priorSystemIndexDescriptors);
} else {
List copy = new ArrayList<>(priorSystemIndexDescriptors);
Collections.sort(copy);
sortedPriorSystemIndexDescriptors = List.copyOf(copy);
}
this.priorSystemIndexDescriptors = sortedPriorSystemIndexDescriptors;
this.executorNames = Objects.nonNull(executorNames) ? executorNames : ExecutorNames.DEFAULT_SYSTEM_INDEX_THREAD_POOLS;
this.isNetNew = isNetNew;
this.allowsTemplates = allowsTemplates;
}
/**
* @return The pattern of index names that this descriptor will be used for. Must start with a '.' character, must not
* overlap with any other descriptor patterns, and must allow a suffix (see note on
* {@link SystemIndexDescriptor#indexPattern} for details).
*/
@Override
public String getIndexPattern() {
return indexPattern;
}
/**
* @return The concrete name of an index being managed internally to Elasticsearch. Will be {@code null}
* for indices managed externally to Elasticsearch.
*/
public String getPrimaryIndex() {
assert isAutomaticallyManaged() : "Unmanaged indices should not have a primary index";
return primaryIndex;
}
/**
* Checks whether an index name matches the system index name pattern for this descriptor.
* @param index The index name to be checked against the index pattern given at construction time.
* @return True if the name matches the pattern, false otherwise.
*/
public boolean matchesIndexPattern(String index) {
return indexPatternAutomaton.run(index);
}
/**
* Retrieves a list of all indices which match this descriptor's pattern.
*
* This cannot be done via {@link org.elasticsearch.cluster.metadata.IndexNameExpressionResolver} because that class can only handle
* simple wildcard expressions, but system index name patterns may use full Lucene regular expression syntax,
*
* @param project The current project metadata to get the list of matching indices from
* @return A list of index names that match this descriptor
*/
@Override
public List getMatchingIndices(ProjectMetadata project) {
return project.indices().keySet().stream().filter(this::matchesIndexPattern).toList();
}
@Override
public String getDescription() {
return description;
}
@Override
public String toString() {
return "SystemIndexDescriptor[pattern=[" + indexPattern + "], description=[" + description + "], aliasName=[" + aliasName + "]]";
}
public String getMappings() {
assert isAutomaticallyManaged() : "Do not request mappings for unmanaged system indices";
return mappings;
}
public Settings getSettings() {
assert isAutomaticallyManaged() : "Do not request settings for unmanaged system indices";
return settings;
}
public String getAliasName() {
return aliasName;
}
public int getIndexFormat() {
assert isAutomaticallyManaged() : "Do not request index format for unmanaged system indices";
return this.indexFormat;
}
@Override
public boolean isAutomaticallyManaged() {
return type.isManaged();
}
@Override
public String getOrigin() {
// TODO[wrb]: most unmanaged system indices do not set origins; could we assert on that here?
return this.origin;
}
public boolean hasDynamicMappings() {
assert isAutomaticallyManaged() : "Do not check mapping properties for unmanaged system indices";
return this.hasDynamicMappings;
}
@Override
public boolean isExternal() {
return type.isExternal();
}
@Override
public List getAllowedElasticProductOrigins() {
return allowedElasticProductOrigins;
}
public boolean isNetNew() {
return isNetNew;
}
public boolean allowsTemplates() {
return allowsTemplates;
}
public MappingsVersion getMappingsVersion() {
if (isAutomaticallyManaged() == false) {
throw new IllegalStateException(this + " is not managed so there are no mappings or version");
}
return mappingsVersion;
}
/**
* Gets a standardized message when the node contains a data or master node whose mappings version is less
* than that of the minimum supported version of this descriptor and its prior descriptors.
*
* @param cause the action being attempted that triggered the check. Used in the error message.
* @return the standardized error message
*/
public String getMinimumMappingsVersionMessage(String cause, MappingsVersion requiredMinimumMappingVersion) {
Objects.requireNonNull(cause);
final MappingsVersion actualMinimumMappingsVersion = priorSystemIndexDescriptors.isEmpty()
? getMappingsVersion()
: priorSystemIndexDescriptors.get(priorSystemIndexDescriptors.size() - 1).mappingsVersion;
return Strings.format(
"[%s] failed - requested creation of system index [%s] with version [%s], while this cluster minimum supported version is "
+ "[%s]. For the cluster to support version [%s], ensure that the system index descriptor for [%s] includes a prior "
+ "definition for that version.",
cause,
this.getPrimaryIndex(),
requiredMinimumMappingVersion,
actualMinimumMappingsVersion,
requiredMinimumMappingVersion,
this.getPrimaryIndex()
);
}
/**
* Finds the descriptor that can be used within this cluster, by comparing the supplied minimum
* mappings version to this descriptor's minimum version and the prior descriptors minimum version.
*
* @param version the lower mappings version in the cluster
* @return null if the lowest mappings version is lower than the minimum version in this descriptor,
* or the appropriate descriptor if the supplied version is acceptable.
*/
public SystemIndexDescriptor getDescriptorCompatibleWith(MappingsVersion version) {
if (Objects.requireNonNull(version).version() >= mappingsVersion.version()) {
return this;
}
for (SystemIndexDescriptor prior : priorSystemIndexDescriptors) {
if (version.version() >= prior.mappingsVersion.version()) {
return prior;
}
}
return null;
}
/**
* @return The names of thread pools that should be used for operations on this system index.
*/
@Override
public ExecutorNames getThreadPoolNames() {
return this.executorNames;
}
public String getMigrationScript() {
return migrationScript;
}
public static Builder builder() {
return new Builder();
}
@Override
public int compareTo(SystemIndexDescriptor other) {
return mappingsVersion.compareTo(other.mappingsVersion) * -1;
}
/**
* The specific type of system index that this descriptor represents. System indices can be one of four defined types; the type is used
* to control behavior. Elasticsearch itself and plugins have system indices that are necessary for their features;
* these system indices are referred to as internal system indices. System indices can also belong to features outside of Elasticsearch
* that may be part of other Elastic stack components. These are external system indices as the intent is for these to be accessed via
* normal APIs with a special value.
*
* Within both internal and external system indices, there are two sub-types. The first are those that are managed by Elasticsearch and
* will have mappings/settings changed as the cluster itself is upgraded. The second are those managed by the owning applications code
* and for those Elasticsearch will not perform any updates.
*
* Internal system indices are almost always managed indices that Elasticsearch manages, but there are cases where the component of
* Elasticsearch will need to manage the system indices itself.
*/
public enum Type {
INTERNAL_MANAGED(false, true),
INTERNAL_UNMANAGED(false, false),
EXTERNAL_MANAGED(true, true),
EXTERNAL_UNMANAGED(true, false);
private final boolean external;
private final boolean managed;
Type(boolean external, boolean managed) {
this.external = external;
this.managed = managed;
}
public boolean isExternal() {
return external;
}
public boolean isManaged() {
return managed;
}
}
/**
* The version of the mapping, which should be stored as an int in a mapping metadata
* field. This will be used with prior index descriptors to determine which mappings
* should be used when creating a system index, and it will be used to determine when
* mappings should be updated with the latest mappings from the system index descriptor.
* See {@link SystemIndexMappingUpdateService}, {@link TransportCreateIndexAction},
* and {@link AutoCreateAction}.
*
* Version should be a non-negative integer for a managed index, or -1 for unmanaged indices.
* The hash is a hash of the system index descriptor's mappings so that we can warn
* in case of inconsistencies across nodes.
*/
public record MappingsVersion(int version, int hash) implements Writeable, ToXContent, Comparable {
public MappingsVersion(StreamInput in) throws IOException {
this(in.readVInt(), in.readInt());
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVInt(version);
out.writeInt(hash);
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field("version", version);
builder.field("hash", hash);
builder.endObject();
return builder;
}
@Override
public int compareTo(MappingsVersion o) {
Objects.requireNonNull(o, "Cannot compare null MappingsVersion");
return Integer.compare(this.version, o.version);
}
}
/**
* Provides a fluent API for building a {@link SystemIndexDescriptor}. Validation still happens in that class.
*/
public static class Builder {
private String indexPattern;
private String primaryIndex;
private String description;
private String mappings = null;
private Settings settings = null;
private String aliasName = null;
private int indexFormat = 0;
private String origin = null;
private String migrationScript;
private Type type = Type.INTERNAL_MANAGED;
private List allowedElasticProductOrigins = List.of();
private List priorSystemIndexDescriptors = List.of();
private ExecutorNames executorNames;
private boolean isNetNew = false;
private boolean allowsTemplates = false;
private Builder() {}
public Builder setIndexPattern(String indexPattern) {
this.indexPattern = indexPattern;
return this;
}
public Builder setPrimaryIndex(String primaryIndex) {
this.primaryIndex = primaryIndex;
return this;
}
public Builder setDescription(String description) {
this.description = description;
return this;
}
public Builder setMappings(XContentBuilder mappingsBuilder) {
mappings = mappingsBuilder == null ? null : Strings.toString(mappingsBuilder);
return this;
}
public Builder setMappings(String mappings) {
this.mappings = mappings;
return this;
}
public Builder setSettings(Settings settings) {
this.settings = settings;
return this;
}
public Builder setAliasName(String aliasName) {
this.aliasName = aliasName;
return this;
}
public Builder setIndexFormat(int indexFormat) {
this.indexFormat = indexFormat;
return this;
}
/**
* Sometimes a system operation will need to dispatch sub-actions. A product origin string will tell the system which component
* generated the sub-action. Internal system indices must not provide an origin, since they are supposed to reject access from
* outside the system. External system indices, on the other hand, must provide an origin. See
* {@link org.elasticsearch.client.internal.OriginSettingClient} for more information.
* @param origin the client origin to use when creating this index.
* @return a {@link Builder} object
*/
public Builder setOrigin(String origin) {
this.origin = origin;
return this;
}
public Builder setMigrationScript(String migrationScript) {
this.migrationScript = migrationScript;
return this;
}
public Builder setType(Type type) {
this.type = type;
return this;
}
public Builder setAllowedElasticProductOrigins(List allowedElasticProductOrigins) {
this.allowedElasticProductOrigins = allowedElasticProductOrigins;
return this;
}
public Builder setPriorSystemIndexDescriptors(List priorSystemIndexDescriptors) {
this.priorSystemIndexDescriptors = priorSystemIndexDescriptors;
return this;
}
public Builder setThreadPools(ExecutorNames threadPoolExecutorNames) {
this.executorNames = threadPoolExecutorNames;
return this;
}
public Builder setNetNew() {
this.isNetNew = true;
return this;
}
public Builder setAllowsTemplates() {
this.allowsTemplates = true;
return this;
}
/**
* Builds a {@link SystemIndexDescriptor} using the fields supplied to this builder.
* @return a populated descriptor.
*/
public SystemIndexDescriptor build() {
return new SystemIndexDescriptor(
indexPattern,
primaryIndex,
description,
mappings,
settings,
aliasName,
indexFormat,
origin,
migrationScript,
type,
allowedElasticProductOrigins,
priorSystemIndexDescriptors,
executorNames,
isNetNew,
allowsTemplates
);
}
}
/**
* Builds an automaton for matching index names against this descriptor's index pattern.
* If this descriptor has an alias name, the automaton will also try to match against
* the alias as well.
*/
static Automaton buildAutomaton(String pattern, String alias) {
final String patternAsRegex = patternToRegex(pattern);
final String aliasAsRegex = alias == null ? null : patternToRegex(alias);
final Automaton patternAutomaton = new RegExp(patternAsRegex, RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT).toAutomaton();
if (aliasAsRegex == null) {
return patternAutomaton;
}
final Automaton aliasAutomaton = new RegExp(aliasAsRegex, RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT).toAutomaton();
return Operations.determinize(Operations.union(patternAutomaton, aliasAutomaton), DEFAULT_DETERMINIZE_WORK_LIMIT);
}
/**
* Translate a simple string pattern into a regular expression, suitable for creating a
* {@link RegExp} instance. This exists because although
* {@link org.elasticsearch.common.regex.Regex#simpleMatchToAutomaton(String)} is useful
* for simple patterns, it doesn't support character ranges.
*
* @param input the string to translate
* @return the translate string
*/
private static String patternToRegex(String input) {
String output = input;
output = output.replace(".", "\\.");
output = output.replace("*", ".*");
return output;
}
/**
* Recursively searches for dynamic: true in the supplies mappings
* @param map a parsed fragment of an index's mappings
* @return whether the fragment contains a dynamic mapping
*/
@SuppressWarnings("unchecked")
static boolean findDynamicMapping(Map map) {
if (map == null) {
return false;
}
for (Map.Entry entry : map.entrySet()) {
final String key = entry.getKey();
final Object value = entry.getValue();
if (key.equals("dynamic") && (value instanceof Boolean) && ((Boolean) value)) {
return true;
}
if (value instanceof Map) {
if (findDynamicMapping((Map) value)) {
return true;
}
}
}
return false;
}
@SuppressWarnings("unchecked") // we do a lot of casting of maps
private static MappingsVersion extractVersionFromMappings(String mappings) {
final Map mappingsMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), mappings, true);
final Map doc = (Map) mappingsMap.get("_doc");
final Map meta;
final Map properties;
if (doc == null) {
meta = (Map) mappingsMap.get("_meta");
properties = (Map) mappingsMap.get("properties");
} else {
meta = (Map) doc.get("_meta");
properties = (Map) doc.get("properties");
}
if (meta == null) {
throw new IllegalStateException("mappings do not have _meta field");
}
final Integer value = (Integer) meta.get(VERSION_META_KEY);
if (value == null) {
throw new IllegalArgumentException("mappings do not have a version in _meta." + VERSION_META_KEY);
}
return new MappingsVersion(value, Objects.hash(properties));
}
}