From 76bba14592e039b28bd09c4ea70417512fc51bfc Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Wed, 19 Jul 2023 19:41:09 +0530 Subject: [PATCH] generalizing supported settings Signed-off-by: Sarthak Aggarwal --- .../org/opensearch/upgrades/IndexingIT.java | 3 +- .../index/codec/CodecCompressionLevelIT.java | 18 +++---- .../replication/SegmentReplicationIT.java | 4 +- .../org/opensearch/index/codec/CodecEnum.java | 53 +++++++++++++++++++ .../opensearch/index/codec/CodecService.java | 34 ++++++------ .../opensearch/index/engine/EngineConfig.java | 4 +- .../opensearch/index/shard/IndexShard.java | 3 +- .../opensearch/index/codec/CodecTests.java | 4 +- .../OngoingSegmentReplicationsTests.java | 3 +- .../test/OpenSearchIntegTestCase.java | 6 +-- 10 files changed, 94 insertions(+), 38 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/codec/CodecEnum.java diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java index b60ee09d39048..da0fe88ebc54b 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java @@ -40,6 +40,7 @@ import org.opensearch.common.Booleans; import org.opensearch.common.io.Streams; import org.opensearch.common.settings.Settings; +import org.opensearch.index.codec.CodecEnum; import org.opensearch.index.codec.CodecService; import org.opensearch.index.engine.EngineConfig; import org.opensearch.indices.replication.common.ReplicationType; @@ -267,7 +268,7 @@ public void testIndexingWithSegRep() throws Exception { .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put( EngineConfig.INDEX_CODEC_SETTING.getKey(), - randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC, CodecService.LUCENE_DEFAULT_CODEC) + randomFrom(CodecEnum.DEFAULT.getName(), CodecEnum.BEST_COMPRESSION.getName(), CodecEnum.LUCENE_DEFAULT.getName()) ) .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms"); createIndex(indexName, settings.build()); diff --git a/server/src/internalClusterTest/java/org/opensearch/index/codec/CodecCompressionLevelIT.java b/server/src/internalClusterTest/java/org/opensearch/index/codec/CodecCompressionLevelIT.java index 5f3e53f1454fc..e7761f8fc4d72 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/codec/CodecCompressionLevelIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/codec/CodecCompressionLevelIT.java @@ -34,7 +34,7 @@ public void testLuceneCodecsCreateIndexWithCompressionLevel() { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC)) + .put("index.codec", randomFrom(CodecEnum.DEFAULT.getName(), CodecEnum.BEST_COMPRESSION.getName())) .put("index.codec.compression_level", randomIntBetween(1, 6)) .build() ) @@ -45,7 +45,7 @@ public void testLuceneCodecsCreateIndexWithCompressionLevel() { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC)) + .put("index.codec", randomFrom(CodecEnum.DEFAULT.getName(), CodecEnum.BEST_COMPRESSION.getName())) .build() ); ensureGreen(index); @@ -62,7 +62,7 @@ public void testZStandardCodecsCreateIndexWithCompressionLevel() { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.codec", randomFrom(CodecService.ZSTD_CODEC, CodecService.ZSTD_NO_DICT_CODEC)) + .put("index.codec", randomFrom(CodecEnum.ZSTD.getName(), CodecEnum.ZSTD_NO_DICT.getName())) .put("index.codec.compression_level", randomIntBetween(1, 6)) .build() ); @@ -81,7 +81,7 @@ public void testZStandardToLuceneCodecsWithCompressionLevel() throws ExecutionEx Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.codec", randomFrom(CodecService.ZSTD_CODEC, CodecService.ZSTD_NO_DICT_CODEC)) + .put("index.codec", randomFrom(CodecEnum.ZSTD.getName(), CodecEnum.ZSTD_NO_DICT.getName())) .put("index.codec.compression_level", randomIntBetween(1, 6)) .build() ); @@ -95,7 +95,7 @@ public void testZStandardToLuceneCodecsWithCompressionLevel() throws ExecutionEx .indices() .updateSettings( new UpdateSettingsRequest(index).settings( - Settings.builder().put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC)) + Settings.builder().put("index.codec", randomFrom(CodecEnum.DEFAULT.getName(), CodecEnum.BEST_COMPRESSION.getName())) ) ) .get() @@ -111,7 +111,7 @@ public void testZStandardToLuceneCodecsWithCompressionLevel() throws ExecutionEx .updateSettings( new UpdateSettingsRequest(index).settings( Settings.builder() - .put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC)) + .put("index.codec", randomFrom(CodecEnum.DEFAULT.getName(), CodecEnum.BEST_COMPRESSION.getName())) .put("index.codec.compression_level", (String) null) ) ) @@ -133,7 +133,7 @@ public void testLuceneToZStandardCodecsWithCompressionLevel() throws ExecutionEx Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC)) + .put("index.codec", randomFrom(CodecEnum.DEFAULT.getName(), CodecEnum.BEST_COMPRESSION.getName())) .build() ); ensureGreen(index); @@ -147,7 +147,7 @@ public void testLuceneToZStandardCodecsWithCompressionLevel() throws ExecutionEx .updateSettings( new UpdateSettingsRequest(index).settings( Settings.builder() - .put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC)) + .put("index.codec", randomFrom(CodecEnum.DEFAULT.getName(), CodecEnum.BEST_COMPRESSION.getName())) .put("index.codec.compression_level", randomIntBetween(1, 6)) ) ) @@ -164,7 +164,7 @@ public void testLuceneToZStandardCodecsWithCompressionLevel() throws ExecutionEx .updateSettings( new UpdateSettingsRequest(index).settings( Settings.builder() - .put("index.codec", randomFrom(CodecService.ZSTD_CODEC, CodecService.ZSTD_NO_DICT_CODEC)) + .put("index.codec", randomFrom(CodecEnum.ZSTD.getName(), CodecEnum.ZSTD_NO_DICT.getName())) .put("index.codec.compression_level", randomIntBetween(1, 6)) ) ) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java index d345578c7de6e..feee6e42e7066 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java @@ -53,7 +53,7 @@ import org.opensearch.index.SegmentReplicationPerGroupStats; import org.opensearch.index.SegmentReplicationPressureService; import org.opensearch.index.SegmentReplicationShardStats; -import org.opensearch.index.codec.CodecService; +import org.opensearch.index.codec.CodecEnum; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.engine.NRTReplicationReaderManager; @@ -203,7 +203,7 @@ public void testReplicationAfterPrimaryRefreshAndFlush() throws Exception { .put(indexSettings()) .put( EngineConfig.INDEX_CODEC_SETTING.getKey(), - randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC, CodecService.LUCENE_DEFAULT_CODEC) + randomFrom(CodecEnum.DEFAULT.getName(), CodecEnum.BEST_COMPRESSION.getName(), CodecEnum.LUCENE_DEFAULT.getName()) ) .build(); createIndex(INDEX_NAME, settings); diff --git a/server/src/main/java/org/opensearch/index/codec/CodecEnum.java b/server/src/main/java/org/opensearch/index/codec/CodecEnum.java new file mode 100644 index 0000000000000..97bfcd317f78f --- /dev/null +++ b/server/src/main/java/org/opensearch/index/codec/CodecEnum.java @@ -0,0 +1,53 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec; + +import org.opensearch.common.settings.Setting; +import org.opensearch.index.engine.EngineConfig; + +import java.util.Set; + +/** + * A codec enum that maintains a list of compression codecs alongside their respective supported index settings. + * + * @opensearch.internal + */ +public enum CodecEnum { + + DEFAULT("default", Set.of()), + BEST_COMPRESSION("best_compression", Set.of()), + ZSTD("zstd", Set.of(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING)), + ZSTD_NO_DICT("zstd_no_dict", Set.of(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING)), + LUCENE_DEFAULT("lucene_default", Set.of()); + + private final String name; + private final Set supportedSettings; + + CodecEnum(String name, Set supportedSettings) { + this.name = name; + this.supportedSettings = supportedSettings; + } + + public static CodecEnum fromString(String codecName) { + for (CodecEnum codecEnum : CodecEnum.values()) { + if (codecEnum.name.equals(codecName)) { + return codecEnum; + } + } + throw new IllegalArgumentException("Codec is not valid. Codec must be one of [default, best_compression, zstd, zstd_no_dict]."); + } + + public Set getSupportedSettings() { + return supportedSettings; + } + + public String getName() { + return name; + } +} diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java index f847bd82161d2..ad8e2580c457e 100644 --- a/server/src/main/java/org/opensearch/index/codec/CodecService.java +++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java @@ -38,6 +38,7 @@ import org.apache.lucene.codecs.lucene95.Lucene95Codec.Mode; import org.opensearch.common.Nullable; import org.opensearch.common.collect.MapBuilder; +import org.opensearch.common.settings.Setting; import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.customcodecs.Lucene95CustomCodec; import org.opensearch.index.codec.customcodecs.ZstdCodec; @@ -60,36 +61,34 @@ public class CodecService { private final Map codecs; - - public static final String DEFAULT_CODEC = "default"; - public static final String BEST_COMPRESSION_CODEC = "best_compression"; /** * the raw unfiltered lucene default. useful for testing */ public static final String LUCENE_DEFAULT_CODEC = "lucene_default"; - public static final String ZSTD_CODEC = "zstd"; - public static final String ZSTD_NO_DICT_CODEC = "zstd_no_dict"; public CodecService(@Nullable MapperService mapperService, IndexSettings indexSettings, Logger logger) { final MapBuilder codecs = MapBuilder.newMapBuilder(); assert null != indexSettings; String codecName = indexSettings.getValue(INDEX_CODEC_SETTING); int compressionLevel = Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL; - if (isZStandardCodec(codecName)) { + if (isSupportedSetting(codecName, INDEX_CODEC_COMPRESSION_LEVEL_SETTING)) { compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING); } if (mapperService == null) { - codecs.put(DEFAULT_CODEC, new Lucene95Codec()); - codecs.put(BEST_COMPRESSION_CODEC, new Lucene95Codec(Mode.BEST_COMPRESSION)); - codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel)); + codecs.put(CodecEnum.DEFAULT.getName(), new Lucene95Codec()); + codecs.put(CodecEnum.BEST_COMPRESSION.getName(), new Lucene95Codec(Mode.BEST_COMPRESSION)); + codecs.put(CodecEnum.ZSTD.getName(), new ZstdCodec(compressionLevel)); + codecs.put(CodecEnum.ZSTD_NO_DICT.getName(), new ZstdNoDictCodec(compressionLevel)); } else { - codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); - codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger)); - codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel)); + codecs.put(CodecEnum.DEFAULT.getName(), new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); + codecs.put( + CodecEnum.BEST_COMPRESSION.getName(), + new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger) + ); + codecs.put(CodecEnum.ZSTD.getName(), new ZstdCodec(mapperService, logger, compressionLevel)); + codecs.put(CodecEnum.ZSTD_NO_DICT.getName(), new ZstdNoDictCodec(mapperService, logger, compressionLevel)); } - codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault()); + codecs.put(CodecEnum.LUCENE_DEFAULT.name(), Codec.getDefault()); for (String codec : Codec.availableCodecs()) { codecs.put(codec, Codec.forName(codec)); } @@ -111,8 +110,9 @@ public String[] availableCodecs() { return codecs.keySet().toArray(new String[0]); } - public static boolean isZStandardCodec(String codec) { - return codec.equals(ZSTD_CODEC) || codec.equals(ZSTD_NO_DICT_CODEC); + public static boolean isSupportedSetting(String codec, Setting setting) { + CodecEnum codecEnum = CodecEnum.fromString(codec); + return codecEnum.getSupportedSettings().contains(setting); } } diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java index 74e251e9eb036..81c181c3b3014 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java @@ -68,7 +68,7 @@ import java.util.function.LongSupplier; import java.util.function.Supplier; -import static org.opensearch.index.codec.CodecService.isZStandardCodec; +import static org.opensearch.index.codec.CodecService.isSupportedSetting; /** * Holds all the configuration that is used to create an {@link Engine}. @@ -178,7 +178,7 @@ public void validate(String key, Object value, Object dependency) { }; private static void doValidateCodecSettings(final String codec) { - if (!isZStandardCodec(codec)) { + if (!isSupportedSetting(codec, INDEX_CODEC_COMPRESSION_LEVEL_SETTING)) { throw new IllegalArgumentException( "Compression level cannot be set for the " + codec diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index fbb2df7f9640a..a97d31b9fc897 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -115,6 +115,7 @@ import org.opensearch.index.cache.IndexCache; import org.opensearch.index.cache.bitset.ShardBitsetFilterCache; import org.opensearch.index.cache.request.ShardRequestCache; +import org.opensearch.index.codec.CodecEnum; import org.opensearch.index.codec.CodecService; import org.opensearch.index.engine.CommitStats; import org.opensearch.index.engine.Engine; @@ -517,7 +518,7 @@ public boolean isSystem() { * Returns the name of the default codec in codecService */ public String getDefaultCodecName() { - return codecService.codec(CodecService.DEFAULT_CODEC).getName(); + return codecService.codec(CodecEnum.DEFAULT.getName()).getName(); } /** diff --git a/server/src/test/java/org/opensearch/index/codec/CodecTests.java b/server/src/test/java/org/opensearch/index/codec/CodecTests.java index 5f4590d1208c5..1b388cb81f814 100644 --- a/server/src/test/java/org/opensearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/opensearch/index/codec/CodecTests.java @@ -115,7 +115,7 @@ public void testZstdNoDictWithCompressionLevel() throws Exception { public void testBestCompressionWithCompressionLevel() { final Settings zstdSettings = Settings.builder() .put(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING.getKey(), randomIntBetween(1, 6)) - .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), randomFrom(CodecService.ZSTD_CODEC, CodecService.ZSTD_NO_DICT_CODEC)) + .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), randomFrom(CodecEnum.ZSTD.getName(), CodecEnum.ZSTD_NO_DICT.getName())) .build(); // able to validate zstd @@ -127,7 +127,7 @@ public void testBestCompressionWithCompressionLevel() { final Settings settings = Settings.builder() .put(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING.getKey(), randomIntBetween(1, 6)) - .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC)) + .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), randomFrom(CodecEnum.DEFAULT.getName(), CodecEnum.BEST_COMPRESSION.getName())) .build(); final IndexScopedSettings indexScopedSettings = new IndexScopedSettings(settings, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS); diff --git a/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java b/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java index 3b289114f5ca1..f1ab768a9cb8d 100644 --- a/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java @@ -17,6 +17,7 @@ import org.opensearch.common.util.CancellableThreads; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.IndexService; +import org.opensearch.index.codec.CodecEnum; import org.opensearch.index.codec.CodecService; import org.opensearch.index.engine.NRTReplicationEngineFactory; import org.opensearch.index.shard.IndexShard; @@ -75,7 +76,7 @@ public void setUp() throws Exception { ShardId testShardId = primary.shardId(); CodecService codecService = new CodecService(null, getEngine(primary).config().getIndexSettings(), null); - String defaultCodecName = codecService.codec(CodecService.DEFAULT_CODEC).getName(); + String defaultCodecName = codecService.codec(CodecEnum.DEFAULT.getName()).getName(); // This mirrors the creation of the ReplicationCheckpoint inside CopyState testCheckpoint = new ReplicationCheckpoint(testShardId, primary.getOperationPrimaryTerm(), 0L, 0L, defaultCodecName); diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 3564bd667ee2b..b6957056f0950 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -130,7 +130,7 @@ import org.opensearch.index.MergePolicyConfig; import org.opensearch.index.MergeSchedulerConfig; import org.opensearch.index.MockEngineFactoryPlugin; -import org.opensearch.index.codec.CodecService; +import org.opensearch.index.codec.CodecEnum; import org.opensearch.index.engine.Segment; import org.opensearch.index.mapper.MockFieldFilterPlugin; import org.opensearch.index.store.Store; @@ -427,9 +427,9 @@ protected void randomIndexTemplate() { // otherwise, use it, it has assertions and so on that can find bugs. SuppressCodecs annotation = getClass().getAnnotation(SuppressCodecs.class); if (annotation != null && annotation.value().length == 1 && "*".equals(annotation.value()[0])) { - randomSettingsBuilder.put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC)); + randomSettingsBuilder.put("index.codec", randomFrom(CodecEnum.DEFAULT.getName(), CodecEnum.BEST_COMPRESSION.getName())); } else { - randomSettingsBuilder.put("index.codec", CodecService.LUCENE_DEFAULT_CODEC); + randomSettingsBuilder.put("index.codec", CodecEnum.LUCENE_DEFAULT.getName()); } for (String setting : randomSettingsBuilder.keys()) {