diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 3429a23..3d7930e 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -29,7 +29,7 @@ jobs: run: | # https://github.com/opensearch-project/opensearch-build/issues/4191 chown -R ci-runner:ci-runner `pwd` - su ci-runner -c "source /etc/profile.d/java_home.sh && ./gradlew check -Dorg.gradle.java.home=/opt/java/openjdk-${{ matrix.java }}" + su ci-runner -c "source /etc/profile.d/java_home.sh && ./gradlew test check -Dorg.gradle.java.home=/opt/java/openjdk-${{ matrix.java }}" - name: Run Gradle (assemble) run: | # https://github.com/opensearch-project/opensearch-build/issues/4191 @@ -53,7 +53,7 @@ jobs: cache: gradle - name: Run Gradle (check) run: | - ./gradlew check + ./gradlew test check - name: Run Gradle (assemble) run: | ./gradlew assemble diff --git a/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java b/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java index 1d9e0ae..df2ef08 100644 --- a/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java +++ b/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java @@ -26,7 +26,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; -import org.opensearch.index.codec.customcodecs.Lucene99QatCodec; +import org.opensearch.index.codec.customcodecs.Lucene912QatCodec; import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.test.rest.OpenSearchRestTestCase; @@ -103,7 +103,10 @@ public void testCreateIndexWithQatSPICodecWithQatHardwareUnavailable() throws IO Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.codec", randomFrom(Lucene99QatCodec.Mode.QAT_LZ4.getCodec(), Lucene99QatCodec.Mode.QAT_DEFLATE.getCodec())) + .put( + "index.codec", + randomFrom(Lucene912QatCodec.Mode.QAT_LZ4.getCodec(), Lucene912QatCodec.Mode.QAT_DEFLATE.getCodec()) + ) .put("index.codec.compression_level", randomIntBetween(1, 6)) .build() ) diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java index 0e45729..7015704 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java @@ -11,6 +11,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.CodecServiceFactory; +import org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec; import org.opensearch.index.engine.EngineConfig; import org.opensearch.plugins.EnginePlugin; import org.opensearch.plugins.Plugin; @@ -49,12 +50,11 @@ public Optional getCustomCodecServiceFactory(final IndexSet || codecName.equals(CustomCodecService.QAT_DEFLATE_CODEC)) { return Optional.of(new CustomCodecServiceFactory()); } else { - if (!QatZipperFactory.isQatAvailable() - && (codecName.equals(Lucene99QatCodec.Mode.QAT_LZ4.getCodec()) - || codecName.equals(Lucene99QatCodec.Mode.QAT_DEFLATE.getCodec()))) { - throw new IllegalArgumentException("QAT codecs are not supported. Please create indices with a different codec."); + if (!QatZipperFactory.isQatAvailable() && isQatCodec(codecName)) { + throw new IllegalArgumentException( + "QAT codecs are not supported (QAT is not available). Please create indices with a different codec." + ); } - } return Optional.empty(); } @@ -63,4 +63,11 @@ public Optional getCustomCodecServiceFactory(final IndexSet public List> getSettings() { return Arrays.asList(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING); } + + private static boolean isQatCodec(String codecName) { + return codecName.equals(Lucene99QatCodec.Mode.QAT_LZ4.getCodec()) + || codecName.equals(Lucene99QatCodec.Mode.QAT_DEFLATE.getCodec()) + || codecName.equals(Lucene912QatCodec.Mode.QAT_LZ4.getCodec()) + || codecName.equals(Lucene912QatCodec.Mode.QAT_DEFLATE.getCodec()); + } } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java index 93448c2..f953505 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java @@ -19,6 +19,7 @@ import java.util.Map; import java.util.stream.Stream; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING; import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; /** CustomCodecService provides ZSTD, ZSTD_NO_DICT, QAT_LZ4, and QAT_DEFLATE compression codecs. */ @@ -49,25 +50,26 @@ public CustomCodecService(MapperService mapperService, IndexSettings indexSettin int compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING); final MapBuilder codecs = MapBuilder.newMapBuilder(); if (mapperService == null) { - codecs.put(ZSTD_CODEC, new Zstd99Codec(compressionLevel)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict99Codec(compressionLevel)); + codecs.put(ZSTD_CODEC, new Zstd912Codec(compressionLevel)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict912Codec(compressionLevel)); if (QatZipperFactory.isQatAvailable()) { - codecs.put(QAT_LZ4_CODEC, new QatLz499Codec(compressionLevel, () -> { - return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING); - })); - codecs.put(QAT_DEFLATE_CODEC, new QatDeflate99Codec(compressionLevel, () -> { - return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING); + codecs.put( + QAT_LZ4_CODEC, + new QatLz4912Codec(compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); }) + ); + codecs.put(QAT_DEFLATE_CODEC, new QatDeflate912Codec(compressionLevel, () -> { + return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); })); } } else { - codecs.put(ZSTD_CODEC, new Zstd99Codec(mapperService, logger, compressionLevel)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict99Codec(mapperService, logger, compressionLevel)); + codecs.put(ZSTD_CODEC, new Zstd912Codec(mapperService, logger, compressionLevel)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict912Codec(mapperService, logger, compressionLevel)); if (QatZipperFactory.isQatAvailable()) { - codecs.put(QAT_LZ4_CODEC, new QatLz499Codec(mapperService, logger, compressionLevel, () -> { - return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING); + codecs.put(QAT_LZ4_CODEC, new QatLz4912Codec(mapperService, logger, compressionLevel, () -> { + return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); })); - codecs.put(QAT_DEFLATE_CODEC, new QatDeflate99Codec(mapperService, logger, compressionLevel, () -> { - return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING); + codecs.put(QAT_DEFLATE_CODEC, new QatDeflate912Codec(mapperService, logger, compressionLevel, () -> { + return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); })); } } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomCodec.java new file mode 100644 index 0000000..692360e --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomCodec.java @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; +import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; +import org.opensearch.index.mapper.MapperService; + +import java.util.Set; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + +/** + * + * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. + * Supports two modes zstd and zstd_no_dict. + * Uses Lucene912 as the delegate codec + * + * @opensearch.internal + */ +public abstract class Lucene912CustomCodec extends FilterCodec { + + /** Each mode represents a compression algorithm. */ + public enum Mode { + /** + * ZStandard mode with dictionary + */ + ZSTD("ZSTD912", Set.of("zstd")), + /** + * ZStandard mode without dictionary + */ + ZSTD_NO_DICT("ZSTDNODICT912", Set.of("zstd_no_dict")); + + private final String codec; + private final Set aliases; + + Mode(String codec, Set aliases) { + this.codec = codec; + this.aliases = aliases; + } + + /** + * Returns the Codec that is registered with Lucene + */ + public String getCodec() { + return codec; + } + + /** + * Returns the aliases of the Codec + */ + public Set getAliases() { + return aliases; + } + } + + private final StoredFieldsFormat storedFieldsFormat; + + /** + * Creates a new compression codec with the default compression level. + * + * @param mode The compression codec (ZSTD or ZSTDNODICT). + */ + public Lucene912CustomCodec(Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new compression codec with the given compression level. We use + * lowercase letters when registering the codec so that we remain consistent with + * the other compression codecs: default, lucene_default, and best_compression. + * + * @param mode The compression codec (ZSTD or ZSTDNODICT). + * @param compressionLevel The compression level. + */ + public Lucene912CustomCodec(Mode mode, int compressionLevel) { + super(mode.getCodec(), new Lucene912Codec()); + this.storedFieldsFormat = new Lucene912CustomStoredFieldsFormat(mode, compressionLevel); + } + + /** + * Creates a new compression codec with the given compression level. We use + * lowercase letters when registering the codec so that we remain consistent with + * the other compression codecs: default, lucene_default, and best_compression. + * + * @param mode The compression codec (ZSTD or ZSTDNODICT). + * @param compressionLevel The compression level. + * @param mapperService The mapper service. + * @param logger The logger. + */ + public Lucene912CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) { + super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene912Codec.Mode.BEST_SPEED, mapperService, logger)); + this.storedFieldsFormat = new Lucene912CustomStoredFieldsFormat(mode, compressionLevel); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormat.java new file mode 100644 index 0000000..86005d5 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormat.java @@ -0,0 +1,140 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.StoredFieldsReader; +import org.apache.lucene.codecs.StoredFieldsWriter; +import org.apache.lucene.codecs.compressing.CompressionMode; +import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsFormat; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; + +import java.io.IOException; +import java.util.Objects; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + +/** Stored field format used by pluggable codec */ +public class Lucene912CustomStoredFieldsFormat extends StoredFieldsFormat { + + /** A key that we use to map to a mode */ + public static final String MODE_KEY = Lucene912CustomStoredFieldsFormat.class.getSimpleName() + ".mode"; + + protected static final int ZSTD_BLOCK_LENGTH = 10 * 48 * 1024; + protected static final int ZSTD_MAX_DOCS_PER_BLOCK = 4096; + protected static final int ZSTD_BLOCK_SHIFT = 10; + + private final CompressionMode zstdCompressionMode; + private final CompressionMode zstdNoDictCompressionMode; + + private final Lucene912CustomCodec.Mode mode; + private final int compressionLevel; + + /** default constructor */ + public Lucene912CustomStoredFieldsFormat() { + this(Lucene912CustomCodec.Mode.ZSTD, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new instance. + * + * @param mode The mode represents ZSTD or ZSTDNODICT + */ + public Lucene912CustomStoredFieldsFormat(Lucene912CustomCodec.Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new instance with the specified mode and compression level. + * + * @param mode The mode represents ZSTD or ZSTDNODICT + * @param compressionLevel The compression level for the mode. + */ + public Lucene912CustomStoredFieldsFormat(Lucene912CustomCodec.Mode mode, int compressionLevel) { + this.mode = Objects.requireNonNull(mode); + this.compressionLevel = compressionLevel; + zstdCompressionMode = new ZstdCompressionMode(compressionLevel); + zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel); + } + + /** + * Returns a {@link StoredFieldsReader} to load stored fields. + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param fn The fieldInfos. + * @param context The IOContext that holds additional details on the merge/search context. + */ + @Override + public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException { + if (si.getAttribute(MODE_KEY) != null) { + String value = si.getAttribute(MODE_KEY); + Lucene912CustomCodec.Mode mode = Lucene912CustomCodec.Mode.valueOf(value); + return impl(mode).fieldsReader(directory, si, fn, context); + } else { + throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name); + } + } + + /** + * Returns a {@link StoredFieldsReader} to write stored fields. + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param context The IOContext that holds additional details on the merge/search context. + */ + @Override + public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException { + String previous = si.putAttribute(MODE_KEY, mode.name()); + if (previous != null && previous.equals(mode.name()) == false) { + throw new IllegalStateException( + "found existing value for " + MODE_KEY + " for segment: " + si.name + " old = " + previous + ", new = " + mode.name() + ); + } + return impl(mode).fieldsWriter(directory, si, context); + } + + StoredFieldsFormat impl(Lucene912CustomCodec.Mode mode) { + switch (mode) { + case ZSTD: + return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstd", this.zstdCompressionMode); + case ZSTD_NO_DICT: + return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstdNoDict", this.zstdNoDictCompressionMode); + default: + throw new IllegalStateException("Unsupported compression mode: " + mode); + } + } + + private StoredFieldsFormat getCustomCompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode) { + return new Lucene90CompressingStoredFieldsFormat( + formatName, + compressionMode, + ZSTD_BLOCK_LENGTH, + ZSTD_MAX_DOCS_PER_BLOCK, + ZSTD_BLOCK_SHIFT + ); + } + + public Lucene912CustomCodec.Mode getMode() { + return mode; + } + + /** + * Returns the compression level. + */ + public int getCompressionLevel() { + return compressionLevel; + } + + public CompressionMode getCompressionMode() { + return mode == Lucene912CustomCodec.Mode.ZSTD_NO_DICT ? zstdNoDictCompressionMode : zstdCompressionMode; + } + +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatCodec.java new file mode 100644 index 0000000..4010741 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatCodec.java @@ -0,0 +1,145 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; +import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; +import org.opensearch.index.mapper.MapperService; + +import java.util.Set; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; + +/** + * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. + * + * @opensearch.internal + */ +public abstract class Lucene912QatCodec extends FilterCodec { + /** Each mode represents a compression algorithm. */ + public enum Mode { + /** QAT lz4 mode. */ + QAT_LZ4("QATLZ4912", Set.of("qat_lz4")), + + /** QAT deflate mode. */ + QAT_DEFLATE("QATDEFLATE912", Set.of("qat_deflate")); + + private final String codec; + private final Set aliases; + + Mode(String codec, Set aliases) { + this.codec = codec; + this.aliases = aliases; + } + + /** Returns the Codec that is registered with Lucene */ + public String getCodec() { + return codec; + } + + /** Returns the aliases of the Codec */ + public Set getAliases() { + return aliases; + } + } + + /** The default compression mode. */ + public static final Mode DEFAULT_COMPRESSION_MODE = Mode.QAT_LZ4; + + private final StoredFieldsFormat storedFieldsFormat; + + /** + * Creates a new compression codec with the default compression level. + * + * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). + */ + public Lucene912QatCodec(Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). + * @param compressionLevel The compression level. + */ + public Lucene912QatCodec(Mode mode, int compressionLevel) { + super(mode.getCodec(), new Lucene912Codec()); + this.storedFieldsFormat = new Lucene912QatStoredFieldsFormat(mode, compressionLevel); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). + * @param compressionLevel The compression level. + * @param supplier supplier for QAT mode. + */ + public Lucene912QatCodec(Mode mode, int compressionLevel, Supplier supplier) { + super(mode.getCodec(), new Lucene912Codec()); + this.storedFieldsFormat = new Lucene912QatStoredFieldsFormat(mode, compressionLevel, supplier); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). + * @param compressionLevel The compression level. + * @param mapperService The mapper service. + * @param logger The logger. + */ + public Lucene912QatCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) { + super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene912Codec.Mode.BEST_SPEED, mapperService, logger)); + this.storedFieldsFormat = new Lucene912QatStoredFieldsFormat(mode, compressionLevel); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). + * @param compressionLevel The compression level. + * @param mapperService The mapper service. + * @param logger The logger. + * @param supplier supplier for QAT mode. + */ + public Lucene912QatCodec( + Mode mode, + int compressionLevel, + MapperService mapperService, + Logger logger, + Supplier supplier + ) { + super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene912Codec.Mode.BEST_SPEED, mapperService, logger)); + this.storedFieldsFormat = new Lucene912QatStoredFieldsFormat(mode, compressionLevel, supplier); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormat.java new file mode 100644 index 0000000..71d6b7e --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormat.java @@ -0,0 +1,188 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.StoredFieldsReader; +import org.apache.lucene.codecs.StoredFieldsWriter; +import org.apache.lucene.codecs.compressing.CompressionMode; +import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsFormat; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; + +import java.io.IOException; +import java.util.Objects; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_QAT_MODE; + +/** Stored field format used by pluggable codec */ +public class Lucene912QatStoredFieldsFormat extends StoredFieldsFormat { + + /** A key that we use to map to a mode */ + public static final String MODE_KEY = Lucene912QatStoredFieldsFormat.class.getSimpleName() + ".mode"; + + private static final int QAT_DEFLATE_BLOCK_LENGTH = 10 * 48 * 1024; + private static final int QAT_DEFLATE_MAX_DOCS_PER_BLOCK = 4096; + private static final int QAT_DEFLATE_BLOCK_SHIFT = 10; + + private static final int QAT_LZ4_BLOCK_LENGTH = 10 * 8 * 1024; + private static final int QAT_LZ4_MAX_DOCS_PER_BLOCK = 4096; + private static final int QAT_LZ4_BLOCK_SHIFT = 10; + + private final QatCompressionMode qatCompressionMode; + private final Lucene912QatCodec.Mode mode; + + /** default constructor */ + public Lucene912QatStoredFieldsFormat() { + this(Lucene912QatCodec.DEFAULT_COMPRESSION_MODE, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new instance. + * + * @param mode The mode represents QAT_LZ4 or QAT_DEFLATE + */ + public Lucene912QatStoredFieldsFormat(Lucene912QatCodec.Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new instance with the specified mode and compression level. + * + * @param mode The mode represents QAT_LZ4 or QAT_DEFLATE + * @param compressionLevel The compression level for the mode. + */ + public Lucene912QatStoredFieldsFormat(Lucene912QatCodec.Mode mode, int compressionLevel) { + this(mode, compressionLevel, () -> { return DEFAULT_QAT_MODE; }); + } + + /** + * Creates a new instance. + * + * @param mode The mode represents QAT_LZ4 or QAT_DEFLATE + * @param supplier a supplier for QAT acceleration mode. + */ + public Lucene912QatStoredFieldsFormat(Lucene912QatCodec.Mode mode, Supplier supplier) { + this(mode, DEFAULT_COMPRESSION_LEVEL, supplier); + } + + /** + * Creates a new instance with the specified mode and compression level. + * + * @param mode The mode represents QAT_LZ4 or QAT_DEFLATE + * @param compressionLevel The compression level for the mode. + * @param supplier a supplier for QAT acceleration mode. + */ + public Lucene912QatStoredFieldsFormat(Lucene912QatCodec.Mode mode, int compressionLevel, Supplier supplier) { + this.mode = Objects.requireNonNull(mode); + qatCompressionMode = new QatCompressionMode(getAlgorithm(mode), compressionLevel, supplier); + } + + /** + * Returns a {@link StoredFieldsReader} to load stored fields. + * + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param fn The fieldInfos. + * @param context The IOContext that holds additional details on the merge/search context. + */ + @Override + public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException { + if (si.getAttribute(MODE_KEY) != null) { + String value = si.getAttribute(MODE_KEY); + Lucene912QatCodec.Mode mode = Lucene912QatCodec.Mode.valueOf(value); + return impl(mode).fieldsReader(directory, si, fn, context); + } else { + throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name); + } + } + + /** + * Returns a {@link StoredFieldsReader} to write stored fields. + * + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param context The IOContext that holds additional details on the merge/search context. + */ + @Override + public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException { + String previous = si.putAttribute(MODE_KEY, mode.name()); + if (previous != null && previous.equals(mode.name()) == false) { + throw new IllegalStateException( + "found existing value for " + MODE_KEY + " for segment: " + si.name + " old = " + previous + ", new = " + mode.name() + ); + } + return impl(mode).fieldsWriter(directory, si, context); + } + + private StoredFieldsFormat impl(Lucene912QatCodec.Mode mode) { + switch (mode) { + case QAT_LZ4: + return getQatCompressingStoredFieldsFormat( + "QatStoredFieldsLz4", + qatCompressionMode, + QAT_LZ4_BLOCK_LENGTH, + QAT_LZ4_MAX_DOCS_PER_BLOCK, + QAT_LZ4_BLOCK_SHIFT + ); + case QAT_DEFLATE: + return getQatCompressingStoredFieldsFormat( + "QatStoredFieldsDeflate", + qatCompressionMode, + QAT_DEFLATE_BLOCK_LENGTH, + QAT_DEFLATE_MAX_DOCS_PER_BLOCK, + QAT_DEFLATE_BLOCK_SHIFT + ); + default: + throw new IllegalStateException("Unsupported compression mode: " + mode); + } + } + + private StoredFieldsFormat getQatCompressingStoredFieldsFormat( + String formatName, + CompressionMode compressionMode, + int blockSize, + int maxDocs, + int blockShift + ) { + return new Lucene90CompressingStoredFieldsFormat(formatName, compressionMode, blockSize, maxDocs, blockShift); + } + + /** + * Gets the mode of compression. + * + * @return either QAT_LZ4 or QAT_DEFLATE + */ + public Lucene912QatCodec.Mode getMode() { + return mode; + } + + /** + * + * @return the CompressionMode instance. + */ + public QatCompressionMode getCompressionMode() { + return qatCompressionMode; + } + + /** + * Returns {@link QatZipper.Algorithm} instance that corresponds codec's {@link Lucene912QatCodec.Mode mode} + * @param mode codec's {@link Lucene912QatCodec.Mode mode} + * @return the {@link QatZipper.Algorithm} instance that corresponds codec's {@link Lucene912QatCodec.Mode mode} + */ + private static QatZipper.Algorithm getAlgorithm(Lucene912QatCodec.Mode mode) { + return (mode == Lucene912QatCodec.Mode.QAT_LZ4) ? QatZipper.Algorithm.LZ4 : QatZipper.Algorithm.DEFLATE; + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java index 193c825..86046c6 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java @@ -22,6 +22,9 @@ import com.intel.qat.QatZipper; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_QAT_MODE; + /** QatCompressionMode offers QAT_LZ4 and QAT_DEFLATE compressors. */ public class QatCompressionMode extends CompressionMode { @@ -33,39 +36,37 @@ public class QatCompressionMode extends CompressionMode { /** default constructor */ protected QatCompressionMode() { - this(Lucene99QatCodec.DEFAULT_COMPRESSION_MODE, Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, () -> { - return Lucene99QatCodec.DEFAULT_QAT_MODE; - }); + this(QatZipper.Algorithm.LZ4, DEFAULT_COMPRESSION_LEVEL, () -> { return DEFAULT_QAT_MODE; }); } /** * Creates a new instance. * - * @param mode The compression mode (QAT_LZ4 or QAT_DEFLATE) + * @param algorithm The compression algorithm (LZ4 or DEFLATE) */ - protected QatCompressionMode(Lucene99QatCodec.Mode mode) { - this(mode, Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, () -> { return Lucene99QatCodec.DEFAULT_QAT_MODE; }); + protected QatCompressionMode(QatZipper.Algorithm algorithm) { + this(algorithm, DEFAULT_COMPRESSION_LEVEL, () -> { return DEFAULT_QAT_MODE; }); } /** * Creates a new instance. * - * @param mode The compression mode (QAT_LZ4 or QAT_DEFLATE) + * @param algorithm The compression algorithm (LZ4 or DEFLATE) * @param compressionLevel The compression level to use. */ - protected QatCompressionMode(Lucene99QatCodec.Mode mode, int compressionLevel) { - this(mode, compressionLevel, () -> { return Lucene99QatCodec.DEFAULT_QAT_MODE; }); + protected QatCompressionMode(QatZipper.Algorithm algorithm, int compressionLevel) { + this(algorithm, compressionLevel, () -> { return DEFAULT_QAT_MODE; }); } /** * Creates a new instance. * - * @param mode The compression mode (QAT_LZ4 or QAT_DEFLATE) + * @param algorithm The compression algorithm (LZ4 or DEFLATE) * @param compressionLevel The compression level to use. * @param supplier a supplier for QAT acceleration mode. */ - protected QatCompressionMode(Lucene99QatCodec.Mode mode, int compressionLevel, Supplier supplier) { - this.algorithm = mode == Lucene99QatCodec.Mode.QAT_LZ4 ? QatZipper.Algorithm.LZ4 : QatZipper.Algorithm.DEFLATE; + protected QatCompressionMode(QatZipper.Algorithm algorithm, int compressionLevel, Supplier supplier) { + this.algorithm = algorithm; this.compressionLevel = compressionLevel; this.supplier = supplier; } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate99Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate912Codec.java similarity index 67% rename from src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate99Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate912Codec.java index 3690c2c..62365ad 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate99Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate912Codec.java @@ -20,56 +20,58 @@ import com.intel.qat.QatZipper; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; + /** - * QatDeflate99Codec provides a DEFLATE compressor using the qat-java library. */ -public class QatDeflate99Codec extends Lucene99QatCodec implements CodecSettings, CodecAliases { +public class QatDeflate912Codec extends Lucene912QatCodec implements CodecSettings, CodecAliases { - /** Creates a new QatDeflate99Codec instance with the default compression level. */ - public QatDeflate99Codec() { + /** Creates a new QatDeflate912Codec instance with the default compression level. */ + public QatDeflate912Codec() { this(DEFAULT_COMPRESSION_LEVEL); } /** - * Creates a new QatDeflate99Codec instance. + * Creates a new QatDeflate912Codec instance. * * @param compressionLevel The compression level. */ - public QatDeflate99Codec(int compressionLevel) { + public QatDeflate912Codec(int compressionLevel) { super(Mode.QAT_DEFLATE, compressionLevel); } /** - * Creates a new QatDeflate99Codec instance with the default compression level. + * Creates a new QatDeflate912Codec instance with the default compression level. * * @param compressionLevel The compression level. * @param supplier supplier for QAT acceleration mode. */ - public QatDeflate99Codec(int compressionLevel, Supplier supplier) { + public QatDeflate912Codec(int compressionLevel, Supplier supplier) { super(Mode.QAT_DEFLATE, compressionLevel, supplier); } /** - * Creates a new QatDeflate99Codec instance. + * Creates a new QatDeflate912Codec instance. * * @param mapperService The mapper service. * @param logger The logger. * @param compressionLevel The compression level. */ - public QatDeflate99Codec(MapperService mapperService, Logger logger, int compressionLevel) { + public QatDeflate912Codec(MapperService mapperService, Logger logger, int compressionLevel) { super(Mode.QAT_DEFLATE, compressionLevel, mapperService, logger); } /** - * Creates a new QatDeflate99Codec instance. + * Creates a new QatDeflate912Codec instance. * * @param mapperService The mapper service. * @param logger The logger. * @param compressionLevel The compression level. * @param supplier supplier for QAT acceleration mode. */ - public QatDeflate99Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier supplier) { + public QatDeflate912Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier supplier) { super(Mode.QAT_DEFLATE, compressionLevel, mapperService, logger, supplier); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz499Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4912Codec.java similarity index 68% rename from src/main/java/org/opensearch/index/codec/customcodecs/QatLz499Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/QatLz4912Codec.java index 90c06fd..5a85aa4 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz499Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4912Codec.java @@ -20,56 +20,58 @@ import com.intel.qat.QatZipper; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; + /** - * QatLz499Codec provides an LZ4 compressor using the qat-java library. */ -public class QatLz499Codec extends Lucene99QatCodec implements CodecSettings, CodecAliases { +public class QatLz4912Codec extends Lucene912QatCodec implements CodecSettings, CodecAliases { - /** Creates a new QatLz499Codec instance with the default compression level. */ - public QatLz499Codec() { + /** Creates a new QatLz4912Codec instance with the default compression level. */ + public QatLz4912Codec() { this(DEFAULT_COMPRESSION_LEVEL); } /** - * Creates a new QatLz499Codec instance. + * Creates a new QatLz4912Codec instance. * * @param compressionLevel The compression level. */ - public QatLz499Codec(int compressionLevel) { + public QatLz4912Codec(int compressionLevel) { super(Mode.QAT_LZ4, compressionLevel); } /** - * Creates a new QatLz499Codec instance with the default compression level. + * Creates a new QatLz4912Codec instance with the default compression level. * * @param compressionLevel The compression level. * @param supplier supplier for QAT acceleration mode. */ - public QatLz499Codec(int compressionLevel, Supplier supplier) { + public QatLz4912Codec(int compressionLevel, Supplier supplier) { super(Mode.QAT_LZ4, compressionLevel, supplier); } /** - * Creates a new QatLz499Codec instance. + * Creates a new QatLz4912Codec instance. * * @param mapperService The mapper service. * @param logger The logger. * @param compressionLevel The compression level. */ - public QatLz499Codec(MapperService mapperService, Logger logger, int compressionLevel) { + public QatLz4912Codec(MapperService mapperService, Logger logger, int compressionLevel) { super(Mode.QAT_LZ4, compressionLevel, mapperService, logger); } /** - * Creates a new QatLz499Codec instance. + * Creates a new QatLz4912Codec instance. * * @param mapperService The mapper service. * @param logger The logger. * @param compressionLevel The compression level. * @param supplier supplier for QAT acceleration mode. */ - public QatLz499Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier supplier) { + public QatLz4912Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier supplier) { super(Mode.QAT_LZ4, compressionLevel, mapperService, logger, supplier); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Zstd99Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Zstd912Codec.java similarity index 80% rename from src/main/java/org/opensearch/index/codec/customcodecs/Zstd99Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/Zstd912Codec.java index 29b5f4b..6a643e7 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Zstd99Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Zstd912Codec.java @@ -17,14 +17,16 @@ import java.util.Set; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + /** * ZstdCodec provides ZSTD compressor using the zstd-jni library. */ -public class Zstd99Codec extends Lucene99CustomCodec implements CodecSettings, CodecAliases { +public class Zstd912Codec extends Lucene912CustomCodec implements CodecSettings, CodecAliases { /** Creates a new ZstdCodec instance with the default compression level. */ - public Zstd99Codec() { + public Zstd912Codec() { this(DEFAULT_COMPRESSION_LEVEL); } @@ -33,7 +35,7 @@ public Zstd99Codec() { * * @param compressionLevel The compression level. */ - public Zstd99Codec(int compressionLevel) { + public Zstd912Codec(int compressionLevel) { super(Mode.ZSTD, compressionLevel); } @@ -44,7 +46,7 @@ public Zstd99Codec(int compressionLevel) { * @param logger The logger. * @param compressionLevel The compression level. */ - public Zstd99Codec(MapperService mapperService, Logger logger, int compressionLevel) { + public Zstd912Codec(MapperService mapperService, Logger logger, int compressionLevel) { super(Mode.ZSTD, compressionLevel, mapperService, logger); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java index 67476ed..4fb53b7 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java @@ -25,6 +25,8 @@ import java.io.IOException; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + /** Zstandard Compression Mode */ public class ZstdCompressionMode extends CompressionMode { @@ -35,7 +37,7 @@ public class ZstdCompressionMode extends CompressionMode { /** default constructor */ protected ZstdCompressionMode() { - this.compressionLevel = Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + this.compressionLevel = DEFAULT_COMPRESSION_LEVEL; } /** diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict99Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict912Codec.java similarity index 79% rename from src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict99Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict912Codec.java index 76c3ce6..b256e4d 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict99Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict912Codec.java @@ -17,11 +17,13 @@ import java.util.Set; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + /** ZstdNoDictCodec provides ZSTD compressor without a dictionary support. */ -public class ZstdNoDict99Codec extends Lucene99CustomCodec implements CodecSettings, CodecAliases { +public class ZstdNoDict912Codec extends Lucene912CustomCodec implements CodecSettings, CodecAliases { /** Creates a new ZstdNoDictCodec instance with the default compression level. */ - public ZstdNoDict99Codec() { + public ZstdNoDict912Codec() { this(DEFAULT_COMPRESSION_LEVEL); } @@ -30,7 +32,7 @@ public ZstdNoDict99Codec() { * * @param compressionLevel The compression level. */ - public ZstdNoDict99Codec(int compressionLevel) { + public ZstdNoDict912Codec(int compressionLevel) { super(Mode.ZSTD_NO_DICT, compressionLevel); } @@ -41,7 +43,7 @@ public ZstdNoDict99Codec(int compressionLevel) { * @param logger The logger. * @param compressionLevel The compression level. */ - public ZstdNoDict99Codec(MapperService mapperService, Logger logger, int compressionLevel) { + public ZstdNoDict912Codec(MapperService mapperService, Logger logger, int compressionLevel) { super(Mode.ZSTD_NO_DICT, compressionLevel, mapperService, logger); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java index 2a44581..fdee56d 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java @@ -21,6 +21,8 @@ import java.io.IOException; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + /** ZSTD Compression Mode (without a dictionary support). */ public class ZstdNoDictCompressionMode extends CompressionMode { @@ -30,7 +32,7 @@ public class ZstdNoDictCompressionMode extends CompressionMode { /** default constructor */ protected ZstdNoDictCompressionMode() { - this.compressionLevel = Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + this.compressionLevel = DEFAULT_COMPRESSION_LEVEL; } /** diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomCodec.java similarity index 72% rename from src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomCodec.java index 6b31167..0dddcf7 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomCodec.java @@ -6,15 +6,12 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; -import org.apache.logging.log4j.Logger; +import org.apache.lucene.backward_codecs.lucene99.Lucene99Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.opensearch.common.settings.Settings; -import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; -import org.opensearch.index.mapper.MapperService; import java.util.Set; @@ -91,21 +88,6 @@ public Lucene99CustomCodec(Mode mode, int compressionLevel) { this.storedFieldsFormat = new Lucene99CustomStoredFieldsFormat(mode, compressionLevel); } - /** - * Creates a new compression codec with the given compression level. We use - * lowercase letters when registering the codec so that we remain consistent with - * the other compression codecs: default, lucene_default, and best_compression. - * - * @param mode The compression codec (ZSTD or ZSTDNODICT). - * @param compressionLevel The compression level. - * @param mapperService The mapper service. - * @param logger The logger. - */ - public Lucene99CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) { - super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger)); - this.storedFieldsFormat = new Lucene99CustomStoredFieldsFormat(mode, compressionLevel); - } - @Override public StoredFieldsFormat storedFieldsFormat() { return storedFieldsFormat; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomStoredFieldsFormat.java similarity index 95% rename from src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomStoredFieldsFormat.java index 271827b..637f454 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomStoredFieldsFormat.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.StoredFieldsReader; @@ -17,6 +17,8 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.opensearch.index.codec.customcodecs.ZstdCompressionMode; +import org.opensearch.index.codec.customcodecs.ZstdNoDictCompressionMode; import java.io.IOException; import java.util.Objects; @@ -60,8 +62,10 @@ public Lucene99CustomStoredFieldsFormat(Lucene99CustomCodec.Mode mode) { public Lucene99CustomStoredFieldsFormat(Lucene99CustomCodec.Mode mode, int compressionLevel) { this.mode = Objects.requireNonNull(mode); this.compressionLevel = compressionLevel; - zstdCompressionMode = new ZstdCompressionMode(compressionLevel); - zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel); + zstdCompressionMode = new ZstdCompressionMode(compressionLevel) { + }; + zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel) { + }; } /** diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatCodec.java similarity index 68% rename from src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatCodec.java index 20f265d..b22054f 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatCodec.java @@ -6,17 +6,14 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; -import org.apache.logging.log4j.Logger; +import org.apache.lucene.backward_codecs.lucene99.Lucene99Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; -import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; -import org.opensearch.index.mapper.MapperService; import java.util.Set; import java.util.function.Supplier; @@ -118,43 +115,6 @@ public Lucene99QatCodec(Mode mode, int compressionLevel, Supplier supplier - ) { - super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger)); - this.storedFieldsFormat = new Lucene99QatStoredFieldsFormat(mode, compressionLevel, supplier); - } - @Override public StoredFieldsFormat storedFieldsFormat() { return storedFieldsFormat; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatStoredFieldsFormat.java similarity index 89% rename from src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormat.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatStoredFieldsFormat.java index 9ea9dd9..079f297 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatStoredFieldsFormat.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.StoredFieldsReader; @@ -17,6 +17,7 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.opensearch.index.codec.customcodecs.QatCompressionMode; import java.io.IOException; import java.util.Objects; @@ -84,7 +85,8 @@ public Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode mode, Supplier supplier) { this.mode = Objects.requireNonNull(mode); - qatCompressionMode = new QatCompressionMode(mode, compressionLevel, supplier); + qatCompressionMode = new QatCompressionMode(getAlgorithm(mode), compressionLevel, supplier) { + }; } /** @@ -173,4 +175,13 @@ public Lucene99QatCodec.Mode getMode() { public QatCompressionMode getCompressionMode() { return qatCompressionMode; } + + /** + * Returns {@link QatZipper.Algorithm} instance that corresponds codec's {@link Lucene99QatCodec.Mode mode} + * @param mode codec's {@link Lucene99QatCodec.Mode mode} + * @return the {@link QatZipper.Algorithm} instance that corresponds codec's {@link Lucene99QatCodec.Mode mode} + */ + private static QatZipper.Algorithm getAlgorithm(Lucene99QatCodec.Mode mode) { + return (mode == Lucene99QatCodec.Mode.QAT_LZ4) ? QatZipper.Algorithm.LZ4 : QatZipper.Algorithm.DEFLATE; + } } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/QatDeflate99Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/QatDeflate99Codec.java new file mode 100644 index 0000000..6da8dba --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/QatDeflate99Codec.java @@ -0,0 +1,70 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; + +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; +import org.opensearch.index.engine.EngineConfig; + +import java.util.Set; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +/** + * QatDeflate99Codec provides a DEFLATE compressor using the qat-java library. + */ +public class QatDeflate99Codec extends Lucene99QatCodec implements CodecSettings, CodecAliases { + + /** Creates a new QatDeflate99Codec instance with the default compression level. */ + public QatDeflate99Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new QatDeflate99Codec instance. + * + * @param compressionLevel The compression level. + */ + public QatDeflate99Codec(int compressionLevel) { + super(Mode.QAT_DEFLATE, compressionLevel); + } + + /** + * Creates a new QatDeflate99Codec instance with the default compression level. + * + * @param compressionLevel The compression level. + * @param supplier supplier for QAT acceleration mode. + */ + public QatDeflate99Codec(int compressionLevel, Supplier supplier) { + super(Mode.QAT_DEFLATE, compressionLevel, supplier); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + if (!QatZipperFactory.isQatAvailable()) { + return Set.of(); + } + return Mode.QAT_DEFLATE.getAliases(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/QatLz499Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/QatLz499Codec.java new file mode 100644 index 0000000..4271ee6 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/QatLz499Codec.java @@ -0,0 +1,70 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; + +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; +import org.opensearch.index.engine.EngineConfig; + +import java.util.Set; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +/** + * QatLz499Codec provides an LZ4 compressor using the qat-java library. + */ +public class QatLz499Codec extends Lucene99QatCodec implements CodecSettings, CodecAliases { + + /** Creates a new QatLz499Codec instance with the default compression level. */ + public QatLz499Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new QatLz499Codec instance. + * + * @param compressionLevel The compression level. + */ + public QatLz499Codec(int compressionLevel) { + super(Mode.QAT_LZ4, compressionLevel); + } + + /** + * Creates a new QatLz499Codec instance with the default compression level. + * + * @param compressionLevel The compression level. + * @param supplier supplier for QAT acceleration mode. + */ + public QatLz499Codec(int compressionLevel, Supplier supplier) { + super(Mode.QAT_LZ4, compressionLevel, supplier); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + if (!QatZipperFactory.isQatAvailable()) { + return Set.of(); + } + return Mode.QAT_LZ4.getAliases(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Zstd99Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Zstd99Codec.java new file mode 100644 index 0000000..53eb8b4 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Zstd99Codec.java @@ -0,0 +1,53 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; + +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.engine.EngineConfig; + +import java.util.Set; + +/** + * ZstdCodec provides ZSTD compressor using the zstd-jni library. + */ +public class Zstd99Codec extends Lucene99CustomCodec implements CodecSettings, CodecAliases { + + /** Creates a new ZstdCodec instance with the default compression level. */ + public Zstd99Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new ZstdCodec instance. + * + * @param compressionLevel The compression level. + */ + public Zstd99Codec(int compressionLevel) { + super(Mode.ZSTD, compressionLevel); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + return Mode.ZSTD.getAliases(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/ZstdNoDict99Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/ZstdNoDict99Codec.java new file mode 100644 index 0000000..3bf42a4 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/ZstdNoDict99Codec.java @@ -0,0 +1,50 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; + +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.engine.EngineConfig; + +import java.util.Set; + +/** ZstdNoDictCodec provides ZSTD compressor without a dictionary support. */ +public class ZstdNoDict99Codec extends Lucene99CustomCodec implements CodecSettings, CodecAliases { + + /** Creates a new ZstdNoDictCodec instance with the default compression level. */ + public ZstdNoDict99Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new ZstdNoDictCodec instance. + * + * @param compressionLevel The compression level. + */ + public ZstdNoDict99Codec(int compressionLevel) { + super(Mode.ZSTD_NO_DICT, compressionLevel); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + return Mode.ZSTD_NO_DICT.getAliases(); + } +} diff --git a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 4e25917..4979cdf 100644 --- a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -1,7 +1,11 @@ org.opensearch.index.codec.customcodecs.backward_codecs.Zstd95Codec org.opensearch.index.codec.customcodecs.backward_codecs.ZstdNoDict95Codec org.opensearch.index.codec.customcodecs.backward_codecs.Zstd95DeprecatedCodec -org.opensearch.index.codec.customcodecs.Zstd99Codec -org.opensearch.index.codec.customcodecs.ZstdNoDict99Codec -org.opensearch.index.codec.customcodecs.QatDeflate99Codec -org.opensearch.index.codec.customcodecs.QatLz499Codec +org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Zstd99Codec +org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.ZstdNoDict99Codec +org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.QatDeflate99Codec +org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.QatLz499Codec +org.opensearch.index.codec.customcodecs.Zstd912Codec +org.opensearch.index.codec.customcodecs.ZstdNoDict912Codec +org.opensearch.index.codec.customcodecs.QatDeflate912Codec +org.opensearch.index.codec.customcodecs.QatLz4912Codec diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java index e995ead..692cbca 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java @@ -35,7 +35,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; @@ -69,6 +69,7 @@ import static org.opensearch.index.codec.customcodecs.CustomCodecService.QAT_LZ4_CODEC; import static org.opensearch.index.codec.customcodecs.CustomCodecService.ZSTD_CODEC; import static org.opensearch.index.codec.customcodecs.CustomCodecService.ZSTD_NO_DICT_CODEC; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; @SuppressCodecs("*") // we test against default codec so never get a random one here! @@ -83,16 +84,16 @@ public void setup() { public void testZstd() throws Exception { Codec codec = createCodecService(false).codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene99CustomCodec.Mode.ZSTD, codec); - Lucene99CustomStoredFieldsFormat storedFieldsFormat = (Lucene99CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD, codec); + Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDict() throws Exception { Codec codec = createCodecService(false).codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene99CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene99CustomStoredFieldsFormat storedFieldsFormat = (Lucene99CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testZstdDeprecatedCodec() { @@ -106,16 +107,16 @@ public void testZstdDeprecatedCodec() { public void testZstdWithCompressionLevel() throws Exception { int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "zstd").codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene99CustomCodec.Mode.ZSTD, codec); - Lucene99CustomStoredFieldsFormat storedFieldsFormat = (Lucene99CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD, codec); + Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDictWithCompressionLevel() throws Exception { int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "zstd_no_dict").codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene99CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene99CustomStoredFieldsFormat storedFieldsFormat = (Lucene99CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); } @@ -156,26 +157,26 @@ public void testZstandardCompressionLevelSupport() throws Exception { public void testDefaultMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("default"); - assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec); + assertStoredFieldsCompressionEquals(Lucene912Codec.Mode.BEST_SPEED, codec); } public void testBestCompressionMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("best_compression"); - assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsCompressionEquals(Lucene912Codec.Mode.BEST_COMPRESSION, codec); } public void testZstdMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene99CustomCodec.Mode.ZSTD, codec); - Lucene99CustomStoredFieldsFormat storedFieldsFormat = (Lucene99CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD, codec); + Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDictMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene99CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene99CustomStoredFieldsFormat storedFieldsFormat = (Lucene99CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testQatCodecsNotAvailable() throws IOException { @@ -183,11 +184,11 @@ public void testQatCodecsNotAvailable() throws IOException { assertThrows(IllegalArgumentException.class, () -> createCodecService(false).codec("qat_lz4")); assertThrows(IllegalArgumentException.class, () -> createCodecService(false).codec("qat_deflate")); - QatLz499Codec qatLz499Codec = new QatLz499Codec(); - assertTrue(qatLz499Codec.aliases().isEmpty()); + QatLz4912Codec qatLz4912Codec = new QatLz4912Codec(); + assertTrue(qatLz4912Codec.aliases().isEmpty()); - QatDeflate99Codec qatDeflate99Codec = new QatDeflate99Codec(); - assertTrue(qatDeflate99Codec.aliases().isEmpty()); + QatDeflate912Codec qatDeflate912Codec = new QatDeflate912Codec(); + assertTrue(qatDeflate912Codec.aliases().isEmpty()); } } @@ -203,18 +204,18 @@ public void testCodecServiceFactoryQatUnavailable() throws IOException { } // write some docs with it, inspect .si to see this was the used compression - private void assertStoredFieldsCompressionEquals(Lucene99Codec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene912Codec.Mode expected, Codec actual) throws Exception { SegmentReader sr = getSegmentReader(actual); String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene99Codec.Mode.valueOf(v)); + assertEquals(expected, Lucene912Codec.Mode.valueOf(v)); } - private void assertStoredFieldsCompressionEquals(Lucene99CustomCodec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode expected, Codec actual) throws Exception { SegmentReader sr = getSegmentReader(actual); - String v = sr.getSegmentInfo().info.getAttribute(Lucene99CustomStoredFieldsFormat.MODE_KEY); + String v = sr.getSegmentInfo().info.getAttribute(Lucene912CustomStoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene99CustomCodec.Mode.valueOf(v)); + assertEquals(expected, Lucene912CustomCodec.Mode.valueOf(v)); } private CodecService createCodecService(boolean isMapperServiceNull) throws IOException { diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormatTests.java new file mode 100644 index 0000000..1118cd1 --- /dev/null +++ b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormatTests.java @@ -0,0 +1,59 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.opensearch.test.OpenSearchTestCase; + +public class Lucene912CustomStoredFieldsFormatTests extends OpenSearchTestCase { + + public void testDefaultLucene912CustomCodecMode() { + Lucene912CustomStoredFieldsFormat lucene912CustomStoredFieldsFormat = new Lucene912CustomStoredFieldsFormat(); + assertEquals(Lucene912CustomCodec.Mode.ZSTD, lucene912CustomStoredFieldsFormat.getMode()); + } + + public void testZstdNoDictLucene912CustomCodecMode() { + Lucene912CustomStoredFieldsFormat lucene912CustomStoredFieldsFormat = new Lucene912CustomStoredFieldsFormat( + Lucene912CustomCodec.Mode.ZSTD_NO_DICT + ); + assertEquals(Lucene912CustomCodec.Mode.ZSTD_NO_DICT, lucene912CustomStoredFieldsFormat.getMode()); + } + + public void testZstdModeWithCompressionLevel() { + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene912CustomStoredFieldsFormat lucene912CustomStoredFieldsFormat = new Lucene912CustomStoredFieldsFormat( + Lucene912CustomCodec.Mode.ZSTD, + randomCompressionLevel + ); + assertEquals(Lucene912CustomCodec.Mode.ZSTD, lucene912CustomStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene912CustomStoredFieldsFormat.getCompressionLevel()); + } + + public void testZstdNoDictLucene912CustomCodecModeWithCompressionLevel() { + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene912CustomStoredFieldsFormat lucene912CustomStoredFieldsFormat = new Lucene912CustomStoredFieldsFormat( + Lucene912CustomCodec.Mode.ZSTD_NO_DICT, + randomCompressionLevel + ); + assertEquals(Lucene912CustomCodec.Mode.ZSTD_NO_DICT, lucene912CustomStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene912CustomStoredFieldsFormat.getCompressionLevel()); + } + + public void testCompressionModes() { + Lucene912CustomStoredFieldsFormat lucene912CustomStoredFieldsFormat = new Lucene912CustomStoredFieldsFormat(); + assertTrue(lucene912CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdCompressionMode); + } + + public void testZstdNoDictCompressionModes() { + Lucene912CustomStoredFieldsFormat lucene912CustomStoredFieldsFormat = new Lucene912CustomStoredFieldsFormat( + Lucene912CustomCodec.Mode.ZSTD_NO_DICT + ); + assertTrue(lucene912CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdNoDictCompressionMode); + } + +} diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormatTests.java new file mode 100644 index 0000000..936e712 --- /dev/null +++ b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormatTests.java @@ -0,0 +1,67 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.opensearch.test.OpenSearchTestCase; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assume.assumeThat; + +public class Lucene912QatStoredFieldsFormatTests extends OpenSearchTestCase { + + public void testLz4Lucene912QatCodecMode() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + Lucene912QatStoredFieldsFormat lucene912QatStoredFieldsFormat = new Lucene912QatStoredFieldsFormat(Lucene912QatCodec.Mode.QAT_LZ4); + assertEquals(Lucene912QatCodec.Mode.QAT_LZ4, lucene912QatStoredFieldsFormat.getMode()); + } + + public void testDeflateLucene912QatCodecMode() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + Lucene912QatStoredFieldsFormat lucene912QatStoredFieldsFormat = new Lucene912QatStoredFieldsFormat( + Lucene912QatCodec.Mode.QAT_DEFLATE + ); + assertEquals(Lucene912QatCodec.Mode.QAT_DEFLATE, lucene912QatStoredFieldsFormat.getMode()); + } + + public void testLz4Lucene912QatCodecModeWithCompressionLevel() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene912QatStoredFieldsFormat lucene912QatStoredFieldsFormat = new Lucene912QatStoredFieldsFormat( + Lucene912QatCodec.Mode.QAT_LZ4, + randomCompressionLevel + ); + assertEquals(Lucene912QatCodec.Mode.QAT_LZ4, lucene912QatStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene912QatStoredFieldsFormat.getCompressionMode().getCompressionLevel()); + } + + public void testDeflateLucene912QatCodecModeWithCompressionLevel() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene912QatStoredFieldsFormat lucene912QatStoredFieldsFormat = new Lucene912QatStoredFieldsFormat( + Lucene912QatCodec.Mode.QAT_DEFLATE, + randomCompressionLevel + ); + assertEquals(Lucene912QatCodec.Mode.QAT_DEFLATE, lucene912QatStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene912QatStoredFieldsFormat.getCompressionMode().getCompressionLevel()); + } + + public void testLz4CompressionModes() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + Lucene912QatStoredFieldsFormat lucene912QatStoredFieldsFormat = new Lucene912QatStoredFieldsFormat(Lucene912QatCodec.Mode.QAT_LZ4); + assertTrue(lucene912QatStoredFieldsFormat.getCompressionMode() instanceof QatCompressionMode); + } + + public void testDeflateCompressionModes() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + Lucene912QatStoredFieldsFormat lucene912QatStoredFieldsFormat = new Lucene912QatStoredFieldsFormat( + Lucene912QatCodec.Mode.QAT_DEFLATE + ); + assertTrue(lucene912QatStoredFieldsFormat.getCompressionMode() instanceof QatCompressionMode); + } +} diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java index e4220d3..2eb9a06 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java @@ -61,6 +61,8 @@ import java.util.Collections; import java.util.Optional; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING; import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; import static org.hamcrest.Matchers.is; import static org.junit.Assume.assumeThat; @@ -78,25 +80,25 @@ public void setup() { public void testQatLz4() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(false).codec("qat_lz4"); - assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_LZ4, codec); - Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); + assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_LZ4, codec); + Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } public void testQatDeflate() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(false).codec("qat_deflate"); - assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_DEFLATE, codec); - Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); + assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_DEFLATE, codec); + Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } public void testQatLz4WithCompressionLevel() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "qat_lz4").codec("qat_lz4"); - assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_LZ4, codec); - Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_LZ4, codec); + Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } @@ -104,8 +106,8 @@ public void testQatDeflateWithCompressionLevel() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "qat_deflate").codec("qat_deflate"); - assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_DEFLATE, codec); - Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_DEFLATE, codec); + Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } @@ -121,25 +123,25 @@ public void testQatCompressionLevelSupport() throws Exception { public void testQatLz4MapperServiceNull() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(true).codec("qat_lz4"); - assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_LZ4, codec); - Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); + assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_LZ4, codec); + Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } public void testQatDeflateMapperServiceNull() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(true).codec("qat_deflate"); - assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_DEFLATE, codec); - Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); + assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_DEFLATE, codec); + Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } - private void assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode expected, Codec actual) throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); SegmentReader sr = getSegmentReader(actual); - String v = sr.getSegmentInfo().info.getAttribute(Lucene99QatStoredFieldsFormat.MODE_KEY); + String v = sr.getSegmentInfo().info.getAttribute(Lucene912QatStoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene99QatCodec.Mode.valueOf(v)); + assertEquals(expected, Lucene912QatCodec.Mode.valueOf(v)); } private CodecService createCodecService(boolean isMapperServiceNull) throws IOException { @@ -147,7 +149,7 @@ private CodecService createCodecService(boolean isMapperServiceNull) throws IOEx if (isMapperServiceNull) { return new CustomCodecService( null, - IndexSettingsModule.newIndexSettings("_na", nodeSettings, Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING), + IndexSettingsModule.newIndexSettings("_na", nodeSettings, INDEX_CODEC_QAT_MODE_SETTING), LogManager.getLogger("test") ); } @@ -164,11 +166,7 @@ private CodecService createCodecService(int randomCompressionLevel, String codec } private CodecService buildCodecService(Settings nodeSettings) throws IOException { - IndexSettings indexSettings = IndexSettingsModule.newIndexSettings( - "_na", - nodeSettings, - Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING - ); + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("_na", nodeSettings, INDEX_CODEC_QAT_MODE_SETTING); SimilarityService similarityService = new SimilarityService(indexSettings, null, Collections.emptyMap()); IndexAnalyzers indexAnalyzers = createTestAnalysis(indexSettings, nodeSettings).indexAnalyzers; MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER); diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java index 1994f69..d06a7b7 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java @@ -12,6 +12,8 @@ import java.io.IOException; +import com.intel.qat.QatZipper; + import static org.hamcrest.Matchers.is; import static org.junit.Assume.assumeThat; @@ -21,13 +23,13 @@ public class QatDeflateCompressorTests extends AbstractCompressorTests { @Override Compressor compressor() { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); - return new QatCompressionMode(Lucene99QatCodec.Mode.QAT_DEFLATE).newCompressor(); + return new QatCompressionMode(QatZipper.Algorithm.DEFLATE).newCompressor(); } @Override Decompressor decompressor() { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); - return new QatCompressionMode(Lucene99QatCodec.Mode.QAT_DEFLATE).newDecompressor(); + return new QatCompressionMode(QatZipper.Algorithm.DEFLATE).newDecompressor(); } @Override diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java index 905773f..ef388a8 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java @@ -12,6 +12,8 @@ import java.io.IOException; +import com.intel.qat.QatZipper; + import static org.hamcrest.Matchers.is; import static org.junit.Assume.assumeThat; @@ -21,13 +23,13 @@ public class QatLz4CompressorTests extends AbstractCompressorTests { @Override Compressor compressor() { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); - return new QatCompressionMode(Lucene99QatCodec.Mode.QAT_LZ4).newCompressor(); + return new QatCompressionMode(QatZipper.Algorithm.LZ4).newCompressor(); } @Override Decompressor decompressor() { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); - return new QatCompressionMode(Lucene99QatCodec.Mode.QAT_LZ4).newDecompressor(); + return new QatCompressionMode(QatZipper.Algorithm.LZ4).newDecompressor(); } @Override diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomStoredFieldsFormatTests.java similarity index 92% rename from src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java rename to src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomStoredFieldsFormatTests.java index 4643246..7cd78d5 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomStoredFieldsFormatTests.java @@ -6,10 +6,10 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs; -import org.opensearch.index.codec.customcodecs.backward_codecs.Lucene95CustomCodec; -import org.opensearch.index.codec.customcodecs.backward_codecs.Lucene95CustomStoredFieldsFormat; +import org.opensearch.index.codec.customcodecs.ZstdCompressionMode; +import org.opensearch.index.codec.customcodecs.ZstdNoDictCompressionMode; import org.opensearch.test.OpenSearchTestCase; public class Lucene95CustomStoredFieldsFormatTests extends OpenSearchTestCase { diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomStoredFieldsFormatTests.java similarity index 92% rename from src/test/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormatTests.java rename to src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomStoredFieldsFormatTests.java index d0ab2ff..eec67e7 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomStoredFieldsFormatTests.java @@ -6,8 +6,10 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; +import org.opensearch.index.codec.customcodecs.ZstdCompressionMode; +import org.opensearch.index.codec.customcodecs.ZstdNoDictCompressionMode; import org.opensearch.test.OpenSearchTestCase; public class Lucene99CustomStoredFieldsFormatTests extends OpenSearchTestCase { diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatStoredFieldsFormatTests.java similarity index 94% rename from src/test/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormatTests.java rename to src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatStoredFieldsFormatTests.java index de3722b..0f0697a 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatStoredFieldsFormatTests.java @@ -6,8 +6,10 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; +import org.opensearch.index.codec.customcodecs.QatCompressionMode; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.test.OpenSearchTestCase; import static org.hamcrest.Matchers.is;