generalizing supported settings

Signed-off-by: Sarthak Aggarwal <[email protected]>
sarthakaggarwal97 · Jul 20, 2023 · 0f5e0e9 · 0f5e0e9
1 parent 26bf31a
commit 0f5e0e9
Show file tree

Hide file tree

Showing 11 changed files with 99 additions and 45 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -74,7 +74,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Added
 - Add server version as REST response header [#6583](https://github.com/opensearch-project/OpenSearch/issues/6583)
 - Start replication checkpointTimers on primary before segments upload to remote store. ([#8221]()https://github.com/opensearch-project/OpenSearch/pull/8221)
-- Disallowing compression level to be set for default and best_compression index codecs ([#8737]()https://github.com/opensearch-project/OpenSearch/pull/8737)
+- Disallow compression level to be set for default and best_compression index codecs ([#8737]()https://github.com/opensearch-project/OpenSearch/pull/8737)
 ### Dependencies
 - Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307))
 - Bump `io.grpc:grpc-context` from 1.46.0 to 1.56.1 ([#8726](https://github.com/opensearch-project/OpenSearch/pull/8726))

diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java
@@ -40,7 +40,7 @@
 import org.opensearch.common.Booleans;
 import org.opensearch.common.io.Streams;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.index.codec.CodecService;
+import org.opensearch.index.codec.CodecType;
 import org.opensearch.index.engine.EngineConfig;
 import org.opensearch.indices.replication.common.ReplicationType;
 import org.opensearch.test.rest.yaml.ObjectPath;
@@ -267,7 +267,7 @@ public void testIndexingWithSegRep() throws Exception {
                     .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
                     .put(
                         EngineConfig.INDEX_CODEC_SETTING.getKey(),
-                        randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC, CodecService.LUCENE_DEFAULT_CODEC)
+                        randomFrom(CodecType.DEFAULT.getName(), CodecType.BEST_COMPRESSION.getName(), CodecType.LUCENE_DEFAULT.getName())
                     )
                     .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms");
                 createIndex(indexName, settings.build());

diff --git a/server/src/internalClusterTest/java/org/opensearch/index/codec/CodecCompressionLevelIT.java b/server/src/internalClusterTest/java/org/opensearch/index/codec/CodecCompressionLevelIT.java
@@ -34,7 +34,7 @@ public void testLuceneCodecsCreateIndexWithCompressionLevel() {
                 Settings.builder()
                     .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                     .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-                    .put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC))
+                    .put("index.codec", randomFrom(CodecType.DEFAULT.getName(), CodecType.BEST_COMPRESSION.getName()))
                     .put("index.codec.compression_level", randomIntBetween(1, 6))
                     .build()
             )
@@ -45,7 +45,7 @@ public void testLuceneCodecsCreateIndexWithCompressionLevel() {
             Settings.builder()
                 .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                 .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-                .put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC))
+                .put("index.codec", randomFrom(CodecType.DEFAULT.getName(), CodecType.BEST_COMPRESSION.getName()))
                 .build()
         );
         ensureGreen(index);
@@ -62,7 +62,7 @@ public void testZStandardCodecsCreateIndexWithCompressionLevel() {
             Settings.builder()
                 .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                 .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-                .put("index.codec", randomFrom(CodecService.ZSTD_CODEC, CodecService.ZSTD_NO_DICT_CODEC))
+                .put("index.codec", randomFrom(CodecType.ZSTD.getName(), CodecType.ZSTD_NO_DICT.getName()))
                 .put("index.codec.compression_level", randomIntBetween(1, 6))
                 .build()
         );
@@ -81,7 +81,7 @@ public void testZStandardToLuceneCodecsWithCompressionLevel() throws ExecutionEx
             Settings.builder()
                 .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                 .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-                .put("index.codec", randomFrom(CodecService.ZSTD_CODEC, CodecService.ZSTD_NO_DICT_CODEC))
+                .put("index.codec", randomFrom(CodecType.ZSTD.getName(), CodecType.ZSTD_NO_DICT.getName()))
                 .put("index.codec.compression_level", randomIntBetween(1, 6))
                 .build()
         );
@@ -95,7 +95,7 @@ public void testZStandardToLuceneCodecsWithCompressionLevel() throws ExecutionEx
                 .indices()
                 .updateSettings(
                     new UpdateSettingsRequest(index).settings(
-                        Settings.builder().put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC))
+                        Settings.builder().put("index.codec", randomFrom(CodecType.DEFAULT.getName(), CodecType.BEST_COMPRESSION.getName()))
                     )
                 )
                 .get()
@@ -111,7 +111,7 @@ public void testZStandardToLuceneCodecsWithCompressionLevel() throws ExecutionEx
                 .updateSettings(
                     new UpdateSettingsRequest(index).settings(
                         Settings.builder()
-                            .put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC))
+                            .put("index.codec", randomFrom(CodecType.DEFAULT.getName(), CodecType.BEST_COMPRESSION.getName()))
                             .put("index.codec.compression_level", (String) null)
                     )
                 )
@@ -133,7 +133,7 @@ public void testLuceneToZStandardCodecsWithCompressionLevel() throws ExecutionEx
             Settings.builder()
                 .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                 .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-                .put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC))
+                .put("index.codec", randomFrom(CodecType.DEFAULT.getName(), CodecType.BEST_COMPRESSION.getName()))
                 .build()
         );
         ensureGreen(index);
@@ -147,7 +147,7 @@ public void testLuceneToZStandardCodecsWithCompressionLevel() throws ExecutionEx
                 .updateSettings(
                     new UpdateSettingsRequest(index).settings(
                         Settings.builder()
-                            .put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC))
+                            .put("index.codec", randomFrom(CodecType.DEFAULT.getName(), CodecType.BEST_COMPRESSION.getName()))
                             .put("index.codec.compression_level", randomIntBetween(1, 6))
                     )
                 )
@@ -164,7 +164,7 @@ public void testLuceneToZStandardCodecsWithCompressionLevel() throws ExecutionEx
                 .updateSettings(
                     new UpdateSettingsRequest(index).settings(
                         Settings.builder()
-                            .put("index.codec", randomFrom(CodecService.ZSTD_CODEC, CodecService.ZSTD_NO_DICT_CODEC))
+                            .put("index.codec", randomFrom(CodecType.ZSTD.getName(), CodecType.ZSTD_NO_DICT.getName()))
                             .put("index.codec.compression_level", randomIntBetween(1, 6))
                     )
                 )

diff --git a/...src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/...src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java
@@ -53,7 +53,7 @@
 import org.opensearch.index.SegmentReplicationPerGroupStats;
 import org.opensearch.index.SegmentReplicationPressureService;
 import org.opensearch.index.SegmentReplicationShardStats;
-import org.opensearch.index.codec.CodecService;
+import org.opensearch.index.codec.CodecType;
 import org.opensearch.index.engine.Engine;
 import org.opensearch.index.engine.EngineConfig;
 import org.opensearch.index.engine.NRTReplicationReaderManager;
@@ -203,7 +203,7 @@ public void testReplicationAfterPrimaryRefreshAndFlush() throws Exception {
             .put(indexSettings())
             .put(
                 EngineConfig.INDEX_CODEC_SETTING.getKey(),
-                randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC, CodecService.LUCENE_DEFAULT_CODEC)
+                randomFrom(CodecType.DEFAULT.getName(), CodecType.BEST_COMPRESSION.getName(), CodecType.LUCENE_DEFAULT.getName())
             )
             .build();
         createIndex(INDEX_NAME, settings);

diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java
@@ -38,6 +38,7 @@
 import org.apache.lucene.codecs.lucene95.Lucene95Codec.Mode;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.collect.MapBuilder;
+import org.opensearch.common.settings.Setting;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.codec.customcodecs.Lucene95CustomCodec;
 import org.opensearch.index.codec.customcodecs.ZstdCodec;
@@ -61,35 +62,29 @@ public class CodecService {
 
     private final Map<String, Codec> codecs;
 
-    public static final String DEFAULT_CODEC = "default";
-    public static final String BEST_COMPRESSION_CODEC = "best_compression";
-    /**
-     * the raw unfiltered lucene default. useful for testing
-     */
-    public static final String LUCENE_DEFAULT_CODEC = "lucene_default";
-    public static final String ZSTD_CODEC = "zstd";
-    public static final String ZSTD_NO_DICT_CODEC = "zstd_no_dict";
-
     public CodecService(@Nullable MapperService mapperService, IndexSettings indexSettings, Logger logger) {
         final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
         assert null != indexSettings;
         String codecName = indexSettings.getValue(INDEX_CODEC_SETTING);
         int compressionLevel = Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL;
-        if (isZStandardCodec(codecName)) {
+        if (isSupportedSetting(codecName, INDEX_CODEC_COMPRESSION_LEVEL_SETTING)) {
             compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING);
         }
         if (mapperService == null) {
-            codecs.put(DEFAULT_CODEC, new Lucene95Codec());
-            codecs.put(BEST_COMPRESSION_CODEC, new Lucene95Codec(Mode.BEST_COMPRESSION));
-            codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel));
-            codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel));
+            codecs.put(CodecType.DEFAULT.getName(), new Lucene95Codec());
+            codecs.put(CodecType.BEST_COMPRESSION.getName(), new Lucene95Codec(Mode.BEST_COMPRESSION));
+            codecs.put(CodecType.ZSTD.getName(), new ZstdCodec(compressionLevel));
+            codecs.put(CodecType.ZSTD_NO_DICT.getName(), new ZstdNoDictCodec(compressionLevel));
         } else {
-            codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
-            codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
-            codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel));
-            codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel));
+            codecs.put(CodecType.DEFAULT.getName(), new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
+            codecs.put(
+                CodecType.BEST_COMPRESSION.getName(),
+                new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger)
+            );
+            codecs.put(CodecType.ZSTD.getName(), new ZstdCodec(mapperService, logger, compressionLevel));
+            codecs.put(CodecType.ZSTD_NO_DICT.getName(), new ZstdNoDictCodec(mapperService, logger, compressionLevel));
         }
-        codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
+        codecs.put(CodecType.LUCENE_DEFAULT.getName(), Codec.getDefault());
         for (String codec : Codec.availableCodecs()) {
             codecs.put(codec, Codec.forName(codec));
         }
@@ -111,8 +106,9 @@ public String[] availableCodecs() {
         return codecs.keySet().toArray(new String[0]);
     }
 
-    public static boolean isZStandardCodec(String codec) {
-        return codec.equals(ZSTD_CODEC) || codec.equals(ZSTD_NO_DICT_CODEC);
+    public static boolean isSupportedSetting(String codec, Setting setting) {
+        CodecType codecType = CodecType.fromString(codec);
+        return codecType.getSupportedSettings().contains(setting);
     }
 
 }
diff --git a/server/src/main/java/org/opensearch/index/codec/CodecType.java b/server/src/main/java/org/opensearch/index/codec/CodecType.java
@@ -0,0 +1,57 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec;
+
+import org.opensearch.common.settings.Setting;
+import org.opensearch.index.engine.EngineConfig;
+
+import java.util.Set;
+
+/**
+ * A codec enum that maintains a list of compression codecs alongside their respective supported index settings.
+ *
+ * @opensearch.internal
+ */
+public enum CodecType {
+
+    DEFAULT("default", Set.of()),
+    BEST_COMPRESSION("best_compression", Set.of()),
+    ZSTD("zstd", Set.of(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING)),
+    ZSTD_NO_DICT("zstd_no_dict", Set.of(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING)),
+    /**
+     * the lucene default codecs. useful for testing
+     */
+    LUCENE_DEFAULT("lucene_default", Set.of()),
+    ASSERTING("Asserting", Set.of());
+
+    private final String name;
+    private final Set<Setting> supportedSettings;
+
+    CodecType(String name, Set<Setting> supportedSettings) {
+        this.name = name;
+        this.supportedSettings = supportedSettings;
+    }
+
+    public static CodecType fromString(String codecName) {
+        for (CodecType codecType : CodecType.values()) {
+            if (codecType.name.equals(codecName)) {
+                return codecType;
+            }
+        }
+        throw new IllegalArgumentException("Codec is not valid. Codec must be one of [default, best_compression, zstd, zstd_no_dict].");
+    }
+
+    public Set<Setting> getSupportedSettings() {
+        return supportedSettings;
+    }
+
+    public String getName() {
+        return name;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java
@@ -68,7 +68,7 @@
 import java.util.function.LongSupplier;
 import java.util.function.Supplier;
 
-import static org.opensearch.index.codec.CodecService.isZStandardCodec;
+import static org.opensearch.index.codec.CodecService.isSupportedSetting;
 
 /**
  * Holds all the configuration that is used to create an {@link Engine}.
@@ -178,7 +178,7 @@ public void validate(String key, Object value, Object dependency) {
     };
 
     private static void doValidateCodecSettings(final String codec) {
-        if (!isZStandardCodec(codec)) {
+        if (!isSupportedSetting(codec, INDEX_CODEC_COMPRESSION_LEVEL_SETTING)) {
             throw new IllegalArgumentException(
                 "Compression level cannot be set for the "
                     + codec

diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
@@ -43,7 +43,6 @@
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryCachingPolicy;
 import org.apache.lucene.search.ReferenceManager;
@@ -115,6 +114,7 @@
 import org.opensearch.index.cache.IndexCache;
 import org.opensearch.index.cache.bitset.ShardBitsetFilterCache;
 import org.opensearch.index.cache.request.ShardRequestCache;
+import org.opensearch.index.codec.CodecType;
 import org.opensearch.index.codec.CodecService;
 import org.opensearch.index.engine.CommitStats;
 import org.opensearch.index.engine.Engine;
@@ -517,7 +517,7 @@ public boolean isSystem() {
      * Returns the name of the default codec in codecService
      */
     public String getDefaultCodecName() {
-        return codecService.codec(CodecService.DEFAULT_CODEC).getName();
+        return codecService.codec(CodecType.DEFAULT.getName()).getName();
     }
 
     /**

diff --git a/server/src/test/java/org/opensearch/index/codec/CodecTests.java b/server/src/test/java/org/opensearch/index/codec/CodecTests.java
@@ -115,7 +115,7 @@ public void testZstdNoDictWithCompressionLevel() throws Exception {
     public void testBestCompressionWithCompressionLevel() {
         final Settings zstdSettings = Settings.builder()
             .put(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING.getKey(), randomIntBetween(1, 6))
-            .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), randomFrom(CodecService.ZSTD_CODEC, CodecService.ZSTD_NO_DICT_CODEC))
+            .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), randomFrom(CodecType.ZSTD.getName(), CodecType.ZSTD_NO_DICT.getName()))
             .build();
 
         // able to validate zstd
@@ -127,7 +127,7 @@ public void testBestCompressionWithCompressionLevel() {
 
         final Settings settings = Settings.builder()
             .put(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING.getKey(), randomIntBetween(1, 6))
-            .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC))
+            .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), randomFrom(CodecType.DEFAULT.getName(), CodecType.BEST_COMPRESSION.getName()))
             .build();
         final IndexScopedSettings indexScopedSettings = new IndexScopedSettings(settings, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
 

diff --git a/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java b/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java
@@ -17,6 +17,7 @@
 import org.opensearch.common.util.CancellableThreads;
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.index.IndexService;
+import org.opensearch.index.codec.CodecType;
 import org.opensearch.index.codec.CodecService;
 import org.opensearch.index.engine.NRTReplicationEngineFactory;
 import org.opensearch.index.shard.IndexShard;
@@ -75,7 +76,7 @@ public void setUp() throws Exception {
         ShardId testShardId = primary.shardId();
 
         CodecService codecService = new CodecService(null, getEngine(primary).config().getIndexSettings(), null);
-        String defaultCodecName = codecService.codec(CodecService.DEFAULT_CODEC).getName();
+        String defaultCodecName = codecService.codec(CodecType.DEFAULT.getName()).getName();
 
         // This mirrors the creation of the ReplicationCheckpoint inside CopyState
         testCheckpoint = new ReplicationCheckpoint(testShardId, primary.getOperationPrimaryTerm(), 0L, 0L, defaultCodecName);

diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
@@ -130,7 +130,7 @@
 import org.opensearch.index.MergePolicyConfig;
 import org.opensearch.index.MergeSchedulerConfig;
 import org.opensearch.index.MockEngineFactoryPlugin;
-import org.opensearch.index.codec.CodecService;
+import org.opensearch.index.codec.CodecType;
 import org.opensearch.index.engine.Segment;
 import org.opensearch.index.mapper.MockFieldFilterPlugin;
 import org.opensearch.index.store.Store;
@@ -427,9 +427,9 @@ protected void randomIndexTemplate() {
             // otherwise, use it, it has assertions and so on that can find bugs.
             SuppressCodecs annotation = getClass().getAnnotation(SuppressCodecs.class);
             if (annotation != null && annotation.value().length == 1 && "*".equals(annotation.value()[0])) {
-                randomSettingsBuilder.put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC));
+                randomSettingsBuilder.put("index.codec", randomFrom(CodecType.DEFAULT.getName(), CodecType.BEST_COMPRESSION.getName()));
             } else {
-                randomSettingsBuilder.put("index.codec", CodecService.LUCENE_DEFAULT_CODEC);
+                randomSettingsBuilder.put("index.codec", CodecType.LUCENE_DEFAULT.getName());
             }
 
             for (String setting : randomSettingsBuilder.keys()) {