diff --git a/server/src/main/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducerWrapper.java b/server/src/main/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducerWrapper.java index 6e6aa125d1dba..884d1b3b48859 100644 --- a/server/src/main/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducerWrapper.java +++ b/server/src/main/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducerWrapper.java @@ -11,6 +11,7 @@ import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SortedNumericDocValues; +import org.opensearch.index.codec.composite.DocValuesProvider; import java.io.IOException; @@ -21,7 +22,7 @@ * * @opensearch.experimental */ -public class Lucene90DocValuesProducerWrapper { +public class Lucene90DocValuesProducerWrapper implements DocValuesProvider { private final Lucene90DocValuesProducer lucene90DocValuesProducer; private final SegmentReadState state; @@ -37,12 +38,15 @@ public Lucene90DocValuesProducerWrapper( this.state = state; } - // returns the doc id set iterator based on field name + // returns the field doc id set iterator based on field name + @Override public SortedNumericDocValues getSortedNumeric(String fieldName) throws IOException { return this.lucene90DocValuesProducer.getSortedNumeric(state.fieldInfos.fieldInfo(fieldName)); } - public Lucene90DocValuesProducer getLucene90DocValuesProducer() { + @Override + public DocValuesProducer getDocValuesProducer() { return lucene90DocValuesProducer; } + } diff --git a/server/src/main/java/org/opensearch/index/codec/composite/DocValuesProvider.java b/server/src/main/java/org/opensearch/index/codec/composite/DocValuesProvider.java new file mode 100644 index 0000000000000..229ebf3b52f73 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/codec/composite/DocValuesProvider.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.composite; + +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.SortedNumericDocValues; + +import java.io.IOException; + +/** + * An interface that provides access to document values for a specific field. + * + * @opensearch.experimental + */ +public interface DocValuesProvider { + + /** + * Returns the sorted numeric document values for the specified field. + * + * @param fieldName The name of the field for which to retrieve the sorted numeric document values. + * @return The sorted numeric document values for the specified field. + * @throws IOException If an error occurs while retrieving the sorted numeric document values. + */ + SortedNumericDocValues getSortedNumeric(String fieldName) throws IOException; + + /** + * Returns the DocValuesProducer instance. + * + * @return The DocValuesProducer instance. + */ + DocValuesProducer getDocValuesProducer(); +} diff --git a/server/src/main/java/org/opensearch/index/codec/composite/LuceneDocValuesConsumerFactory.java b/server/src/main/java/org/opensearch/index/codec/composite/LuceneDocValuesConsumerFactory.java new file mode 100644 index 0000000000000..a61135cfbb106 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/codec/composite/LuceneDocValuesConsumerFactory.java @@ -0,0 +1,46 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.composite; + +import org.apache.lucene.codecs.DocValuesConsumer; +import org.apache.lucene.codecs.lucene90.Lucene90DocValuesConsumerWrapper; +import org.apache.lucene.index.SegmentWriteState; + +import java.io.IOException; + +import static org.opensearch.index.codec.composite.composite99.Composite99Codec.COMPOSITE_INDEX_CODEC_NAME; + +/** + * A factory class that provides a factory method for creating {@link DocValuesConsumer} instances + * based on the specified composite codec. + * + * @opensearch.experimental + */ +public class LuceneDocValuesConsumerFactory { + + public static DocValuesConsumer getDocValuesConsumerForCompositeCodec( + String compositeCodec, + SegmentWriteState state, + String dataCodec, + String dataExtension, + String metaCodec, + String metaExtension + ) throws IOException { + + switch (compositeCodec) { + case COMPOSITE_INDEX_CODEC_NAME: + return new Lucene90DocValuesConsumerWrapper(state, dataCodec, dataExtension, metaCodec, metaExtension) + .getLucene90DocValuesConsumer(); + default: + throw new IllegalStateException("Invalid composite codec " + "[" + compositeCodec + "]"); + } + + } + +} diff --git a/server/src/main/java/org/opensearch/index/codec/composite/LuceneDocValuesProducerFactory.java b/server/src/main/java/org/opensearch/index/codec/composite/LuceneDocValuesProducerFactory.java new file mode 100644 index 0000000000000..a9287c610ffb1 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/codec/composite/LuceneDocValuesProducerFactory.java @@ -0,0 +1,44 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.composite; + +import org.apache.lucene.codecs.DocValuesConsumer; +import org.apache.lucene.codecs.lucene90.Lucene90DocValuesProducerWrapper; +import org.apache.lucene.index.SegmentReadState; +import org.opensearch.index.codec.composite.composite99.Composite99Codec; + +import java.io.IOException; + +/** + * A factory class that provides a factory method for creating {@link DocValuesConsumer} instances + * based on the specified composite codec. + * + * @opensearch.experimental + */ +public class LuceneDocValuesProducerFactory { + + public static DocValuesProvider getDocValuesProducerForCompositeCodec( + String compositeCodec, + SegmentReadState state, + String dataCodec, + String dataExtension, + String metaCodec, + String metaExtension + ) throws IOException { + + switch (compositeCodec) { + case Composite99Codec.COMPOSITE_INDEX_CODEC_NAME: + return new Lucene90DocValuesProducerWrapper(state, dataCodec, dataExtension, metaCodec, metaExtension); + default: + throw new IllegalStateException("Invalid composite codec " + "[" + compositeCodec + "]"); + } + + } + +} diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesReader.java b/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesReader.java index f8d221378935e..706f42629631e 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesReader.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesReader.java @@ -30,6 +30,8 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; import org.opensearch.index.codec.composite.CompositeIndexReader; +import org.opensearch.index.codec.composite.DocValuesProvider; +import org.opensearch.index.codec.composite.LuceneDocValuesProducerFactory; import org.opensearch.index.compositeindex.CompositeIndexMetadata; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.MetricEntry; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; @@ -185,13 +187,17 @@ public Composite99DocValuesReader(DocValuesProducer producer, SegmentReadState r ); // initialize star-tree doc values producer - compositeDocValuesProducer = new Lucene90DocValuesProducerWrapper( + + DocValuesProvider docValuesProvider = LuceneDocValuesProducerFactory.getDocValuesProducerForCompositeCodec( + Composite99Codec.COMPOSITE_INDEX_CODEC_NAME, segmentReadState, Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, Composite99DocValuesFormat.META_DOC_VALUES_CODEC, Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION ); + + compositeDocValuesProducer = (Lucene90DocValuesProducerWrapper) docValuesProvider; } catch (Throwable t) { priorE = t; } finally { @@ -242,7 +248,7 @@ public void close() throws IOException { boolean success = false; try { IOUtils.close(metaIn, dataIn); - IOUtils.close(compositeDocValuesProducer.getLucene90DocValuesProducer()); + IOUtils.close(compositeDocValuesProducer.getDocValuesProducer()); success = true; } finally { if (!success) { diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesWriter.java index 7d8636574e0a3..c0bde4388c241 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesWriter.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesWriter.java @@ -11,7 +11,6 @@ import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.codecs.lucene90.Lucene90DocValuesConsumerWrapper; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.EmptyDocValuesProducer; @@ -25,6 +24,7 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; import org.opensearch.index.codec.composite.CompositeIndexReader; +import org.opensearch.index.codec.composite.LuceneDocValuesConsumerFactory; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreesBuilder; import org.opensearch.index.compositeindex.datacube.startree.index.CompositeIndexValues; @@ -86,13 +86,14 @@ public Composite99DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState boolean success = false; try { - this.composite99DocValuesConsumer = new Lucene90DocValuesConsumerWrapper( + this.composite99DocValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + Composite99Codec.COMPOSITE_INDEX_CODEC_NAME, segmentWriteState, Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, Composite99DocValuesFormat.META_DOC_VALUES_CODEC, Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ).getLucene90DocValuesConsumer(); + ); String dataFileName = IndexFileNames.segmentFileName( segmentWriteState.segmentInfo.name, diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/StatelessDoubleValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/StatelessDoubleValueAggregator.java index 04c6bcc906ef2..27191847068a9 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/StatelessDoubleValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/StatelessDoubleValueAggregator.java @@ -57,7 +57,7 @@ public Double toStarTreeNumericTypeValue(Long value) { if (value == null) { return getIdentityMetricValue(); } - return starTreeNumericType.getDoubleValue(value); + return VALUE_AGGREGATOR_TYPE.getDoubleValue(value); } catch (Exception e) { throw new IllegalStateException("Cannot convert " + value + " to sortable aggregation type", e); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java index 91487dd901701..362a6e16563fb 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java @@ -96,7 +96,7 @@ public Double toStarTreeNumericTypeValue(Long value) { if (value == null) { return getIdentityMetricValue(); } - return starTreeNumericType.getDoubleValue(value); + return VALUE_AGGREGATOR_TYPE.getDoubleValue(value); } catch (Exception e) { throw new IllegalStateException("Cannot convert " + value + " to sortable aggregation type", e); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index 5584ec731d404..3c45f0b200b10 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -171,16 +171,16 @@ public List getMetricReaders(SegmentWriteState stat for (MetricStat metricStat : metric.getMetrics()) { SequentialDocValuesIterator metricReader; FieldInfo metricFieldInfo = state.fieldInfos.fieldInfo(metric.getField()); - if (metricStat != MetricStat.COUNT) { - if (metricFieldInfo == null) { - metricFieldInfo = StarTreeUtils.getFieldInfo(metric.getField(), 1); - } - metricReader = new SequentialDocValuesIterator( - fieldProducerMap.get(metricFieldInfo.name).getSortedNumeric(metricFieldInfo) - ); - } else { - metricReader = new SequentialDocValuesIterator(); + // if (metricStat != MetricStat.COUNT) { + if (metricFieldInfo == null) { + metricFieldInfo = StarTreeUtils.getFieldInfo(metric.getField(), 1); } + metricReader = new SequentialDocValuesIterator( + fieldProducerMap.get(metricFieldInfo.name).getSortedNumeric(metricFieldInfo) + ); + // } else { + // metricReader = new SequentialDocValuesIterator(); + // } metricReaders.add(metricReader); } @@ -238,9 +238,7 @@ public void build( ) throws IOException { int numSegmentStarTreeDocument = totalSegmentDocs; - while (starTreeDocumentIterator.hasNext()) { - appendToStarTree(starTreeDocumentIterator.next()); - } + appendDocumentsToStarTree(starTreeDocumentIterator); int numStarTreeDocument = numStarTreeDocs; logger.debug("Generated star tree docs : [{}] from segment docs : [{}]", numStarTreeDocument, numSegmentStarTreeDocument); @@ -269,6 +267,12 @@ public void build( serializeStarTree(numStarTreeDocument); } + void appendDocumentsToStarTree(Iterator starTreeDocumentIterator) throws IOException { + while (starTreeDocumentIterator.hasNext()) { + appendToStarTree(starTreeDocumentIterator.next()); + } + } + private void serializeStarTree(int numSegmentStarTreeDocument) throws IOException { // serialize the star tree data long dataFilePointer = dataOut.getFilePointer(); @@ -759,9 +763,7 @@ private InMemoryTreeNode constructStarNode(int startDocId, int endDocId, int dim starNode.nodeType = StarTreeNodeType.STAR.getValue(); starNode.startDocId = numStarTreeDocs; Iterator starTreeDocumentIterator = generateStarTreeDocumentsForStarNode(startDocId, endDocId, dimensionId); - while (starTreeDocumentIterator.hasNext()) { - appendToStarTree(starTreeDocumentIterator.next()); - } + appendDocumentsToStarTree(starTreeDocumentIterator); starNode.endDocId = numStarTreeDocs; return starNode; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java index 9341588334219..9e047df7f1aa0 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java @@ -22,7 +22,6 @@ import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.node.StarTree; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; -import org.opensearch.index.compositeindex.datacube.startree.node.Tree; import java.io.IOException; import java.util.ArrayList; @@ -102,7 +101,7 @@ public StarTreeValues( ) ); - Tree starTree = new StarTree(compositeIndexIn, starTreeMetadata); + StarTree starTree = new StarTree(compositeIndexIn, starTreeMetadata); this.root = starTree.getRoot(); // get doc id set iterators for metrics and dimensions diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTree.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTree.java index 49ddbf07a4259..4ed3c3ec9febe 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTree.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTree.java @@ -24,7 +24,7 @@ * * @opensearch.experimental */ -public class StarTree implements Tree { +public class StarTree { private static final Logger logger = LogManager.getLogger(StarTree.class); private final FixedLengthStarTreeNode root; private final Integer numNodes; @@ -49,7 +49,6 @@ public StarTree(IndexInput data, StarTreeMetadata starTreeMetadata) throws IOExc root = new FixedLengthStarTreeNode(in, 0); } - @Override public StarTreeNode getRoot() { return root; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java index 3c2acadff6a96..dd9d301096f44 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java @@ -85,9 +85,17 @@ public interface StarTreeNode { boolean isLeaf(); /** - * Checks if the current node is a star node, null node or a node with actual dimension value. + * Determines the type of the current node in the Star Tree index structure. * - * @return the node type value based on the star-tree node type + *

The node type can be one of the following: + *

    + *
  • Star Node: Represented by the value -2. + *
  • Null Node: Represented by the value -1. + *
  • Default Node: Represented by the value 0. + *
+ * @see StarTreeNodeType + * + * @return The type of the current node, represented by the corresponding integer value (-2, -1, or 0). * @throws IOException if an I/O error occurs while reading the node type */ byte getStarTreeNodeType() throws IOException; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNodeType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNodeType.java index 856fee072cd9e..2dcec37322778 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNodeType.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNodeType.java @@ -10,14 +10,27 @@ /** * Represents the different types of nodes in a StarTree data structure. + * *

* In order to handle different node types, we use a byte value to represent the node type. * This enum provides a convenient way to map byte values to their corresponding node types. + * *

* Star and Null Nodes are represented as special cases. Default is the general case. - * Star and Null nodes are represented with negative ordinals so that first node is Star, second node is Null Node - * and the rest of the default nodes are sorted based on dimension values. + * Star and null nodes are represented with negative ordinal values to ensure that they are + * sorted before the default nodes, which are sorted based on their dimension values. + * *

+ * The node type can be one of the following: + *

    + *
  • Star Node: Represented by the value -2. A star node is a special node that represents + * all possible values for a dimension.
    • + *
  • Null Node: Represented by the value -1. A null node indicates the absence of any value + * for a dimension.
    • + *
  • Default Node: Represented by the value 0. A default node represents a node with an + * actual dimension value.
    • + *
+ * * By default, we want to consider nodes as default node. * * @opensearch.experimental diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/Tree.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/Tree.java deleted file mode 100644 index 811150da64bec..0000000000000 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/Tree.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ -package org.opensearch.index.compositeindex.datacube.startree.node; - -/** - * Interface for star-tree. - * - * @opensearch.experimental - */ -public interface Tree { - - /** - * Fetches the root node of the star-tree. - * @return the root of the star-tree - */ - StarTreeNode getRoot(); - -} diff --git a/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java b/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java index 6d3a35bc2a259..66e3ee899b4ed 100644 --- a/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java +++ b/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java @@ -20,13 +20,11 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SegmentReader; -import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.BaseDocValuesFormatTestCase; import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.tests.util.TestUtil; -import org.apache.lucene.util.NumericUtils; import org.opensearch.Version; import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.metadata.IndexMetadata; @@ -47,10 +45,12 @@ import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.NumericDimension; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; -import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.StarTreeMapper; import org.opensearch.indices.IndicesModule; @@ -62,11 +62,10 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import static org.opensearch.common.util.FeatureFlags.STAR_TREE_INDEX; +import static org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils.assertStarTreeDocuments; /** * Star tree doc values Lucene tests @@ -148,15 +147,18 @@ public void testStarTreeDocValues() throws IOException { doc.add(new SortedNumericDocValuesField("dv", 1)); doc.add(new SortedNumericDocValuesField("field", 1)); iw.addDocument(doc); + doc = new Document(); doc.add(new SortedNumericDocValuesField("sndv", 1)); doc.add(new SortedNumericDocValuesField("dv", 1)); doc.add(new SortedNumericDocValuesField("field", 1)); iw.addDocument(doc); + doc = new Document(); iw.forceMerge(1); doc.add(new SortedNumericDocValuesField("sndv", 2)); doc.add(new SortedNumericDocValuesField("dv", 2)); doc.add(new SortedNumericDocValuesField("field", 2)); iw.addDocument(doc); + doc = new Document(); doc.add(new SortedNumericDocValuesField("sndv", 2)); doc.add(new SortedNumericDocValuesField("dv", 2)); doc.add(new SortedNumericDocValuesField("field", 2)); @@ -167,8 +169,13 @@ public void testStarTreeDocValues() throws IOException { DirectoryReader ir = maybeWrapWithMergingReader(DirectoryReader.open(directory)); TestUtil.checkReader(ir); assertEquals(1, ir.leaves().size()); - Map> expectedDimensionDocValues = getExpectedDimToValueMap(); - Map> expectedMetricsDocValues = getExpectedMetricsToValueMap(); + + StarTreeDocument[] expectedStarTreeDocuments = new StarTreeDocument[4]; + expectedStarTreeDocuments[0] = new StarTreeDocument(new Long[]{1L, 1L}, new Double[]{2.0, 2.0}); + expectedStarTreeDocuments[1] = new StarTreeDocument(new Long[]{2L, 2L}, new Double[]{4.0, 2.0}); + expectedStarTreeDocuments[2] = new StarTreeDocument(new Long[]{null, 1L}, new Double[]{2.0, 2.0}); + expectedStarTreeDocuments[3] = new StarTreeDocument(new Long[]{null, 2L}, new Double[]{4.0, 2.0}); + for (LeafReaderContext context : ir.leaves()) { SegmentReader reader = Lucene.segmentReader(context.reader()); CompositeIndexReader starTreeDocValuesReader = (CompositeIndexReader) reader.getDocValuesReader(); @@ -176,85 +183,25 @@ public void testStarTreeDocValues() throws IOException { for (CompositeIndexFieldInfo compositeIndexFieldInfo : compositeIndexFields) { StarTreeValues starTreeValues = (StarTreeValues) starTreeDocValuesReader.getCompositeIndexValues(compositeIndexFieldInfo); - - for (Map.Entry entry : starTreeValues.getDimensionDocValuesIteratorMap().entrySet()) { - List expectedDimDocValues = expectedDimensionDocValues.get(entry.getKey()); - SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(entry.getValue()); - assertNotNull(expectedDimDocValues); - for (int i = 0; i < expectedDimDocValues.size(); i++) { - int docId = sequentialDocValuesIterator.nextDoc(i); - if (docId < i) { - assertNull(expectedDimDocValues.get(i)); - } else if (docId == i) { - assertEquals(sequentialDocValuesIterator.value(docId), expectedDimDocValues.get(i)); - } - } - } - - for (Map.Entry entry : starTreeValues.getMetricDocValuesIteratorMap().entrySet()) { - List expectedMetricDocValues = expectedMetricsDocValues.get(entry.getKey()); - SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(entry.getValue()); - assertNotNull(expectedMetricDocValues); - for (int i = 0; i < expectedMetricDocValues.size(); i++) { - int docId = sequentialDocValuesIterator.nextDoc(i); - if (docId < i) { - assertNull(expectedMetricDocValues.get(i)); - } else if (docId == i) { - assertEquals(sequentialDocValuesIterator.value(docId), expectedMetricDocValues.get(i), 0); - } - } - } + StarTreeDocument[] starTreeDocuments = StarTreeTestUtils.getSegmentsStarTreeDocuments( + List.of(starTreeValues), + List.of(StarTreeNumericType.DOUBLE, StarTreeNumericType.LONG), + reader.maxDoc() + ); + assertStarTreeDocuments(starTreeDocuments, expectedStarTreeDocuments); } } ir.close(); directory.close(); } - private static Map> getExpectedDimToValueMap() { - Map> expectedDimToValueMap = new HashMap<>(); - List dimDocValues = new ArrayList<>(); - - dimDocValues.add(1L); - dimDocValues.add(1L); - dimDocValues.add(2L); - dimDocValues.add(2L); - expectedDimToValueMap.put("sndv", dimDocValues); - - dimDocValues = new ArrayList<>(); - - dimDocValues.add(1L); - dimDocValues.add(1L); - dimDocValues.add(2L); - dimDocValues.add(2L); - expectedDimToValueMap.put("dv", dimDocValues); - - return expectedDimToValueMap; - } - - private static Map> getExpectedMetricsToValueMap() { - Map> expectedDimToValueMap = new HashMap<>(); - List dimDocValues = new ArrayList<>(); - - dimDocValues.add(NumericUtils.doubleToSortableLong(2D)); - dimDocValues.add(NumericUtils.doubleToSortableLong(2D)); - expectedDimToValueMap.put("startree_field_count_metric", dimDocValues); - - dimDocValues = new ArrayList<>(); - - dimDocValues.add(NumericUtils.doubleToSortableLong(2D)); - dimDocValues.add(NumericUtils.doubleToSortableLong(4D)); - expectedDimToValueMap.put("startree_field_sum_metric", dimDocValues); - - return expectedDimToValueMap; - } - private XContentBuilder getExpandedMapping() throws IOException { return topMapping(b -> { b.startObject("composite"); b.startObject("startree"); b.field("type", "star_tree"); b.startObject("config"); - b.field("max_leaf_docs", 100); + b.field("max_leaf_docs", 1); b.startArray("ordered_dimensions"); b.startObject(); b.field("name", "sndv"); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java new file mode 100644 index 0000000000000..f2694f96d46b4 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java @@ -0,0 +1,259 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree; + +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.store.IndexInput; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.data.StarTreeDataWriter; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.MetricEntry; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; +import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; +import org.opensearch.index.compositeindex.datacube.startree.node.StarTree; +import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; +import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; +import org.opensearch.index.mapper.CompositeMappedFieldType; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Queue; + +import static org.opensearch.index.compositeindex.CompositeIndexConstants.COMPOSITE_FIELD_MARKER; +import static org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter.VERSION_CURRENT; +import static org.opensearch.index.mapper.CompositeMappedFieldType.CompositeFieldType.STAR_TREE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +public class StarTreeTestUtils { + + public static StarTreeDocument[] getSegmentsStarTreeDocuments( + List starTreeValuesSubs, + List starTreeNumericTypes, + int numDocs + ) throws IOException { + List starTreeDocuments = new ArrayList<>(); + for (StarTreeValues starTreeValues : starTreeValuesSubs) { + List dimensionsSplitOrder = starTreeValues.getStarTreeField().getDimensionsOrder(); + SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[dimensionsSplitOrder.size()]; + + for (int i = 0; i < dimensionsSplitOrder.size(); i++) { + String dimension = dimensionsSplitOrder.get(i).getField(); + dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocValuesIteratorMap().get(dimension)); + } + + List metricReaders = new ArrayList<>(); + for (Map.Entry metricDocValuesEntry : starTreeValues.getMetricDocValuesIteratorMap().entrySet()) { + metricReaders.add(new SequentialDocValuesIterator(metricDocValuesEntry.getValue())); + } + + int currentDocId = 0; + while (currentDocId < numDocs) { + starTreeDocuments.add(getStarTreeDocument(currentDocId, dimensionReaders, metricReaders, starTreeNumericTypes)); + currentDocId++; + } + } + StarTreeDocument[] starTreeDocumentsArr = new StarTreeDocument[starTreeDocuments.size()]; + return starTreeDocuments.toArray(starTreeDocumentsArr); + } + + public static StarTreeDocument getStarTreeDocument( + int currentDocId, + SequentialDocValuesIterator[] dimensionReaders, + List metricReaders, + List starTreeNumericTypes + ) throws IOException { + Long[] dims = new Long[dimensionReaders.length]; + int i = 0; + for (SequentialDocValuesIterator dimensionDocValueIterator : dimensionReaders) { + dimensionDocValueIterator.nextDoc(currentDocId); + Long val = dimensionDocValueIterator.value(currentDocId); + dims[i] = val; + i++; + } + i = 0; + Object[] metrics = new Object[metricReaders.size()]; + for (SequentialDocValuesIterator metricDocValuesIterator : metricReaders) { + metricDocValuesIterator.nextDoc(currentDocId); + metrics[i] = toStarTreeNumericTypeValue(metricDocValuesIterator.value(currentDocId), starTreeNumericTypes.get(i)); + i++; + } + return new StarTreeDocument(dims, metrics); + } + + public static Double toStarTreeNumericTypeValue(Long value, StarTreeNumericType starTreeNumericType) { + try { + return starTreeNumericType.getDoubleValue(value); + } catch (Exception e) { + throw new IllegalStateException("Cannot convert " + value + " to sortable aggregation type", e); + } + } + + public static void assertStarTreeDocuments(StarTreeDocument[] starTreeDocuments, StarTreeDocument[] expectedStarTreeDocuments) { + + assertNotNull(starTreeDocuments); + assertEquals(starTreeDocuments.length, expectedStarTreeDocuments.length); + + for (int i = 0; i < starTreeDocuments.length; i++) { + + StarTreeDocument resultStarTreeDocument = starTreeDocuments[i]; + StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocuments[i]; + + assertNotNull(resultStarTreeDocument.dimensions); + assertNotNull(resultStarTreeDocument.metrics); + + assertEquals(resultStarTreeDocument.dimensions.length, expectedStarTreeDocument.dimensions.length); + assertEquals(resultStarTreeDocument.metrics.length, expectedStarTreeDocument.metrics.length); + + for (int di = 0; di < resultStarTreeDocument.dimensions.length; di++) { + assertEquals(resultStarTreeDocument.dimensions[di], expectedStarTreeDocument.dimensions[di]); + } + + for (int mi = 0; mi < resultStarTreeDocument.metrics.length; mi++) { + assertEquals(resultStarTreeDocument.metrics[mi], expectedStarTreeDocument.metrics[mi]); + } + } + } + + + public static void validateFileFormats( + IndexInput dataIn, + IndexInput metaIn, + InMemoryTreeNode rootNode, + StarTreeMetadata expectedStarTreeMetadata + ) throws IOException { + long magicMarker = metaIn.readLong(); + assertEquals(COMPOSITE_FIELD_MARKER, magicMarker); + int version = metaIn.readVInt(); + assertEquals(VERSION_CURRENT, version); + + String compositeFieldName = metaIn.readString(); + assertEquals(expectedStarTreeMetadata.getStarTreeFieldName(), compositeFieldName); + CompositeMappedFieldType.CompositeFieldType compositeFieldType = CompositeMappedFieldType.CompositeFieldType.fromName( + metaIn.readString() + ); + assertEquals(STAR_TREE, compositeFieldType); + StarTreeMetadata resultStarTreeMetadata = new StarTreeMetadata(metaIn, compositeFieldName, compositeFieldType); + assertStarTreeMetadata(expectedStarTreeMetadata, resultStarTreeMetadata); + + IndexInput starTreeIndexInput = dataIn.slice( + "star-tree data slice for respective star-tree fields", + resultStarTreeMetadata.getDataStartFilePointer(), + resultStarTreeMetadata.getDataLength() + ); + StarTree starTree = new StarTree(starTreeIndexInput, resultStarTreeMetadata); + + StarTreeNode starTreeNode = starTree.getRoot(); + Queue expectedTreeNodeQueue = new ArrayDeque<>(); + Queue resultTreeNodeQueue = new ArrayDeque<>(); + + expectedTreeNodeQueue.add(starTreeNode); + resultTreeNodeQueue.add(rootNode); + + while ((starTreeNode = expectedTreeNodeQueue.poll()) != null && (rootNode = resultTreeNodeQueue.poll()) != null) { + + // verify the star node + assertStarTreeNode(starTreeNode, rootNode); + + Iterator expectedChildrenIterator = starTreeNode.getChildrenIterator(); + + List sortedChildren = new ArrayList<>(); + if (rootNode.children != null) { + sortedChildren = new ArrayList<>(rootNode.children.values()); + } + sortedChildren.sort( + Comparator.comparingInt(InMemoryTreeNode::getNodeType).thenComparingLong(InMemoryTreeNode::getDimensionValue) + ); + + if (starTreeNode.getChildDimensionId() != -1) { + assertFalse(sortedChildren.isEmpty()); + int childCount = 0; + while (expectedChildrenIterator.hasNext()) { + StarTreeNode child = expectedChildrenIterator.next(); + InMemoryTreeNode resultChildNode = sortedChildren.get(childCount); + + assertNotNull(resultChildNode); + assertNotNull(child); + assertStarTreeNode(child, resultChildNode); + + expectedTreeNodeQueue.add(child); + resultTreeNodeQueue.add(resultChildNode); + + childCount++; + } + assertEquals(childCount, rootNode.children.size()); + } else { + assertNull(rootNode.children); + } + } + + assertTrue(expectedTreeNodeQueue.isEmpty()); + assertTrue(resultTreeNodeQueue.isEmpty()); + + } + + public static void assertStarTreeNode(StarTreeNode starTreeNode, InMemoryTreeNode treeNode) throws IOException { + assertEquals(starTreeNode.getDimensionId(), treeNode.dimensionId); + assertEquals(starTreeNode.getDimensionValue(), treeNode.dimensionValue); + assertEquals(starTreeNode.getStartDocId(), treeNode.startDocId); + assertEquals(starTreeNode.getEndDocId(), treeNode.endDocId); + assertEquals(starTreeNode.getChildDimensionId(), treeNode.childDimensionId); + assertEquals(starTreeNode.getAggregatedDocId(), treeNode.aggregatedDocId); + + if (starTreeNode.getChildDimensionId() != -1) { + assertFalse(starTreeNode.isLeaf()); + if (treeNode.children != null) { + assertEquals(starTreeNode.getNumChildren(), treeNode.children.values().size()); + } + } else { + assertTrue(starTreeNode.isLeaf()); + } + + } + + public static void assertStarTreeMetadata(StarTreeMetadata expectedStarTreeMetadata, StarTreeMetadata resultStarTreeMetadata) { + + assertEquals(expectedStarTreeMetadata.getCompositeFieldName(), resultStarTreeMetadata.getCompositeFieldName()); + assertEquals(expectedStarTreeMetadata.getCompositeFieldType(), resultStarTreeMetadata.getCompositeFieldType()); + assertEquals(expectedStarTreeMetadata.getDimensionFields().size(), resultStarTreeMetadata.getDimensionFields().size()); + for (int i = 0; i < expectedStarTreeMetadata.getDimensionFields().size(); i++) { + assertEquals(expectedStarTreeMetadata.getDimensionFields().get(i), resultStarTreeMetadata.getDimensionFields().get(i)); + } + assertEquals(expectedStarTreeMetadata.getMetricEntries().size(), resultStarTreeMetadata.getMetricEntries().size()); + for (int i = 0; i < expectedStarTreeMetadata.getMetricEntries().size(); i++) { + MetricEntry expectedMetricEntry = expectedStarTreeMetadata.getMetricEntries().get(i); + MetricEntry resultMetricEntry = resultStarTreeMetadata.getMetricEntries().get(i); + assertEquals(expectedMetricEntry, resultMetricEntry); + } + + assertEquals(expectedStarTreeMetadata.getSegmentAggregatedDocCount(), resultStarTreeMetadata.getSegmentAggregatedDocCount()); + assertEquals(expectedStarTreeMetadata.getMaxLeafDocs(), resultStarTreeMetadata.getMaxLeafDocs()); + assertEquals( + expectedStarTreeMetadata.getSkipStarNodeCreationInDims().size(), + resultStarTreeMetadata.getSkipStarNodeCreationInDims().size() + ); + for (String skipDimension : expectedStarTreeMetadata.getSkipStarNodeCreationInDims()) { + assertTrue(resultStarTreeMetadata.getSkipStarNodeCreationInDims().contains(skipDimension)); + } + assertEquals(expectedStarTreeMetadata.getStarTreeBuildMode(), resultStarTreeMetadata.getStarTreeBuildMode()); + assertEquals(expectedStarTreeMetadata.getDataStartFilePointer(), resultStarTreeMetadata.getDataStartFilePointer()); + assertEquals(expectedStarTreeMetadata.getDataLength(), resultStarTreeMetadata.getDataLength()); + assertEquals(0, (resultStarTreeMetadata.getDataLength() - StarTreeDataWriter.computeStarTreeDataHeaderByteSize()) % 33); + } + +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/AbstractValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/AbstractValueAggregatorTests.java index f6adf442bb6ab..d6f9c72721eed 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/AbstractValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/AbstractValueAggregatorTests.java @@ -10,6 +10,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; @@ -21,7 +22,7 @@ public abstract class AbstractValueAggregatorTests extends OpenSearchTestCase { private ValueAggregator aggregator; - private StarTreeNumericType starTreeNumericType; + protected StarTreeNumericType starTreeNumericType; public AbstractValueAggregatorTests(StarTreeNumericType starTreeNumericType) { this.starTreeNumericType = starTreeNumericType; @@ -69,7 +70,7 @@ public void testGetInitialAggregatedValueForSegmentDocValue() { assertEquals(CountValueAggregator.DEFAULT_INITIAL_VALUE, aggregator.getInitialAggregatedValueForSegmentDocValue(randomLong())); } else { assertEquals( - aggregator.toStarTreeNumericTypeValue(randomLong), + StarTreeTestUtils.toStarTreeNumericTypeValue(randomLong, starTreeNumericType), aggregator.getInitialAggregatedValueForSegmentDocValue(randomLong) ); } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MaxValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MaxValueAggregatorTests.java index a4e01bb70492c..7581ce6d9e7eb 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MaxValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MaxValueAggregatorTests.java @@ -9,6 +9,7 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; import org.apache.lucene.util.NumericUtils; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; public class MaxValueAggregatorTests extends AbstractValueAggregatorTests { @@ -23,18 +24,18 @@ public void testMergeAggregatedValueAndSegmentValue() { Long randomLong = randomLong(); double randomDouble = randomDouble(); assertEquals( - Math.max(aggregator.toStarTreeNumericTypeValue(randomLong), randomDouble), + Math.max(StarTreeTestUtils.toStarTreeNumericTypeValue(randomLong, aggregator.starTreeNumericType), randomDouble), aggregator.mergeAggregatedValueAndSegmentValue(randomDouble, randomLong), 0.0 ); assertEquals( - aggregator.toStarTreeNumericTypeValue(randomLong), + StarTreeTestUtils.toStarTreeNumericTypeValue(randomLong, aggregator.starTreeNumericType), aggregator.mergeAggregatedValueAndSegmentValue(null, randomLong), 0.0 ); assertEquals(randomDouble, aggregator.mergeAggregatedValueAndSegmentValue(randomDouble, null), 0.0); assertEquals( - Math.max(2.0, aggregator.toStarTreeNumericTypeValue(3L)), + Math.max(2.0, StarTreeTestUtils.toStarTreeNumericTypeValue(3L, starTreeNumericType)), aggregator.mergeAggregatedValueAndSegmentValue(2.0, 3L), 0.0 ); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MinValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MinValueAggregatorTests.java index b8233b0fd7fe0..f971b2190852a 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MinValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MinValueAggregatorTests.java @@ -9,6 +9,7 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; import org.apache.lucene.util.NumericUtils; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; public class MinValueAggregatorTests extends AbstractValueAggregatorTests { @@ -22,18 +23,18 @@ public void testMergeAggregatedValueAndSegmentValue() { Long randomLong = randomLong(); double randomDouble = randomDouble(); assertEquals( - Math.min(aggregator.toStarTreeNumericTypeValue(randomLong), randomDouble), + Math.min(StarTreeTestUtils.toStarTreeNumericTypeValue(randomLong, aggregator.starTreeNumericType), randomDouble), aggregator.mergeAggregatedValueAndSegmentValue(randomDouble, randomLong), 0.0 ); assertEquals( - aggregator.toStarTreeNumericTypeValue(randomLong), + StarTreeTestUtils.toStarTreeNumericTypeValue(randomLong, starTreeNumericType), aggregator.mergeAggregatedValueAndSegmentValue(null, randomLong), 0.0 ); assertEquals(randomDouble, aggregator.mergeAggregatedValueAndSegmentValue(randomDouble, null), 0.0); assertEquals( - Math.min(2.0, aggregator.toStarTreeNumericTypeValue(3L)), + Math.min(2.0, StarTreeTestUtils.toStarTreeNumericTypeValue(3L, starTreeNumericType)), aggregator.mergeAggregatedValueAndSegmentValue(2.0, 3L), 0.0 ); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java index 3a85357da5ffe..6dfbceb613448 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java @@ -8,6 +8,7 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; public class SumValueAggregatorTests extends AbstractValueAggregatorTests { @@ -29,7 +30,7 @@ public void testMergeAggregatedValueAndSegmentValue() { Long randomLong = randomLong(); aggregator.getInitialAggregatedValue(randomDouble); assertEquals( - randomDouble + aggregator.toStarTreeNumericTypeValue(randomLong), + randomDouble + StarTreeTestUtils.toStarTreeNumericTypeValue(randomLong, starTreeNumericType), aggregator.mergeAggregatedValueAndSegmentValue(randomDouble, randomLong), 0.0 ); @@ -41,7 +42,7 @@ public void testMergeAggregatedValueAndSegmentValue_nullSegmentDocValue() { aggregator.getInitialAggregatedValue(randomDouble1); assertEquals(randomDouble1, aggregator.mergeAggregatedValueAndSegmentValue(randomDouble1, null), 0.0); assertEquals( - randomDouble1 + aggregator.toStarTreeNumericTypeValue(randomLong), + randomDouble1 + StarTreeTestUtils.toStarTreeNumericTypeValue(randomLong, starTreeNumericType), aggregator.mergeAggregatedValueAndSegmentValue(randomDouble1, randomLong), 0.0 ); @@ -51,7 +52,7 @@ public void testMergeAggregatedValueAndSegmentValue_nullInitialDocValue() { Long randomLong = randomLong(); aggregator.getInitialAggregatedValue(null); assertEquals( - aggregator.toStarTreeNumericTypeValue(randomLong), + StarTreeTestUtils.toStarTreeNumericTypeValue(randomLong, starTreeNumericType), aggregator.mergeAggregatedValueAndSegmentValue(null, randomLong), 0.0 ); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java index 219119ceb551d..a3d6fd0ddf329 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java @@ -43,16 +43,12 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; -import org.opensearch.index.compositeindex.datacube.startree.fileformats.data.StarTreeDataWriter; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.MetricEntry; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; -import org.opensearch.index.compositeindex.datacube.startree.node.StarTree; -import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; -import org.opensearch.index.mapper.CompositeMappedFieldType; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.Mapper; @@ -67,7 +63,6 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -80,8 +75,7 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; -import static org.opensearch.index.compositeindex.CompositeIndexConstants.COMPOSITE_FIELD_MARKER; -import static org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter.VERSION_CURRENT; +import static org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils.validateFileFormats; import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.ALL; import static org.opensearch.index.mapper.CompositeMappedFieldType.CompositeFieldType.STAR_TREE; import static org.mockito.Mockito.mock; @@ -1635,131 +1629,6 @@ private void validateStarTreeFileFormats(InMemoryTreeNode rootNode, int numDocs, metaIn.close(); } - private void validateFileFormats( - IndexInput dataIn, - IndexInput metaIn, - InMemoryTreeNode rootNode, - StarTreeMetadata expectedStarTreeMetadata - ) throws IOException { - long magicMarker = metaIn.readLong(); - assertEquals(COMPOSITE_FIELD_MARKER, magicMarker); - int version = metaIn.readVInt(); - assertEquals(VERSION_CURRENT, version); - - String compositeFieldName = metaIn.readString(); - assertEquals(expectedStarTreeMetadata.getStarTreeFieldName(), compositeFieldName); - CompositeMappedFieldType.CompositeFieldType compositeFieldType = CompositeMappedFieldType.CompositeFieldType.fromName( - metaIn.readString() - ); - assertEquals(STAR_TREE, compositeFieldType); - StarTreeMetadata resultStarTreeMetadata = new StarTreeMetadata(metaIn, compositeFieldName, compositeFieldType); - assertStarTreeMetadata(expectedStarTreeMetadata, resultStarTreeMetadata); - - IndexInput starTreeIndexInput = dataIn.slice( - "star-tree data slice for respective star-tree fields", - resultStarTreeMetadata.getDataStartFilePointer(), - resultStarTreeMetadata.getDataLength() - ); - StarTree starTree = new StarTree(starTreeIndexInput, resultStarTreeMetadata); - - StarTreeNode starTreeNode = starTree.getRoot(); - Queue expectedTreeNodeQueue = new ArrayDeque<>(); - Queue resultTreeNodeQueue = new ArrayDeque<>(); - - expectedTreeNodeQueue.add(starTreeNode); - resultTreeNodeQueue.add(rootNode); - - while ((starTreeNode = expectedTreeNodeQueue.poll()) != null && (rootNode = resultTreeNodeQueue.poll()) != null) { - - // verify the star node - assertStarTreeNode(starTreeNode, rootNode); - - Iterator expectedChildrenIterator = starTreeNode.getChildrenIterator(); - - List sortedChildren = new ArrayList<>(); - if (rootNode.children != null) { - sortedChildren = new ArrayList<>(rootNode.children.values()); - } - sortedChildren.sort( - Comparator.comparingInt(InMemoryTreeNode::getNodeType).thenComparingLong(InMemoryTreeNode::getDimensionValue) - ); - - if (starTreeNode.getChildDimensionId() != -1) { - assertFalse(sortedChildren.isEmpty()); - int childCount = 0; - while (expectedChildrenIterator.hasNext()) { - StarTreeNode child = expectedChildrenIterator.next(); - InMemoryTreeNode resultChildNode = sortedChildren.get(childCount); - - assertNotNull(resultChildNode); - assertNotNull(child); - assertStarTreeNode(child, resultChildNode); - - expectedTreeNodeQueue.add(child); - resultTreeNodeQueue.add(resultChildNode); - - childCount++; - } - assertEquals(childCount, rootNode.children.size()); - } else { - assertNull(rootNode.children); - } - } - - assertTrue(expectedTreeNodeQueue.isEmpty()); - assertTrue(resultTreeNodeQueue.isEmpty()); - - } - - private void assertStarTreeNode(StarTreeNode starTreeNode, InMemoryTreeNode treeNode) throws IOException { - assertEquals(starTreeNode.getDimensionId(), treeNode.dimensionId); - assertEquals(starTreeNode.getDimensionValue(), treeNode.dimensionValue); - assertEquals(starTreeNode.getStartDocId(), treeNode.startDocId); - assertEquals(starTreeNode.getEndDocId(), treeNode.endDocId); - assertEquals(starTreeNode.getChildDimensionId(), treeNode.childDimensionId); - assertEquals(starTreeNode.getAggregatedDocId(), treeNode.aggregatedDocId); - - if (starTreeNode.getChildDimensionId() != -1) { - assertFalse(starTreeNode.isLeaf()); - if (treeNode.children != null) { - assertEquals(starTreeNode.getNumChildren(), treeNode.children.values().size()); - } - } else { - assertTrue(starTreeNode.isLeaf()); - } - - } - - private void assertStarTreeMetadata(StarTreeMetadata expectedStarTreeMetadata, StarTreeMetadata resultStarTreeMetadata) { - - assertEquals(expectedStarTreeMetadata.getCompositeFieldName(), resultStarTreeMetadata.getCompositeFieldName()); - assertEquals(expectedStarTreeMetadata.getCompositeFieldType(), resultStarTreeMetadata.getCompositeFieldType()); - assertEquals(expectedStarTreeMetadata.getDimensionFields().size(), resultStarTreeMetadata.getDimensionFields().size()); - for (int i = 0; i < expectedStarTreeMetadata.getDimensionFields().size(); i++) { - assertEquals(expectedStarTreeMetadata.getDimensionFields().get(i), resultStarTreeMetadata.getDimensionFields().get(i)); - } - assertEquals(expectedStarTreeMetadata.getMetricEntries().size(), resultStarTreeMetadata.getMetricEntries().size()); - for (int i = 0; i < expectedStarTreeMetadata.getMetricEntries().size(); i++) { - MetricEntry expectedMetricEntry = expectedStarTreeMetadata.getMetricEntries().get(i); - MetricEntry resultMetricEntry = resultStarTreeMetadata.getMetricEntries().get(i); - assertEquals(expectedMetricEntry, resultMetricEntry); - } - - assertEquals(expectedStarTreeMetadata.getSegmentAggregatedDocCount(), resultStarTreeMetadata.getSegmentAggregatedDocCount()); - assertEquals(expectedStarTreeMetadata.getMaxLeafDocs(), resultStarTreeMetadata.getMaxLeafDocs()); - assertEquals( - expectedStarTreeMetadata.getSkipStarNodeCreationInDims().size(), - resultStarTreeMetadata.getSkipStarNodeCreationInDims().size() - ); - for (String skipDimension : expectedStarTreeMetadata.getSkipStarNodeCreationInDims()) { - assertTrue(resultStarTreeMetadata.getSkipStarNodeCreationInDims().contains(skipDimension)); - } - assertEquals(expectedStarTreeMetadata.getStarTreeBuildMode(), resultStarTreeMetadata.getStarTreeBuildMode()); - assertEquals(expectedStarTreeMetadata.getDataStartFilePointer(), resultStarTreeMetadata.getDataStartFilePointer()); - assertEquals(expectedStarTreeMetadata.getDataLength(), resultStarTreeMetadata.getDataLength()); - assertEquals(0, (resultStarTreeMetadata.getDataLength() - StarTreeDataWriter.computeStarTreeDataHeaderByteSize()) % 33); - } - private static Map> getExpectedDimToValueMap() { Map> expectedDimToValueMap = new HashMap<>(); Map dimValueMap = new HashMap<>(); @@ -1932,6 +1801,8 @@ public void testFlushFlowDimsReverse() throws IOException { [5, 5] | [50.0, 1] [null, 0] | [0.0, 1] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(6, builder.getStarTreeDocuments().size()); builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); int count = 0; for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { @@ -2115,6 +1986,8 @@ public void testMergeFlowWithSum() throws IOException { * ------------------ We only take non star docs * [6,-1] | [120.0] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(6, builder.getStarTreeDocuments().size()); builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); int count = 0; for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { @@ -2187,6 +2060,8 @@ public void testMergeFlowWithCount() throws IOException { --------------- [6,-1] | [12] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(6, builder.getStarTreeDocuments().size()); builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); int count = 0; for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { @@ -2288,6 +2163,8 @@ public void testMergeFlowWithDifferentDocsFromSegments() throws IOException { [null, 2] | [2] [null, 7] | [7] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(9, builder.getStarTreeDocuments().size()); builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); int count = 0; for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { @@ -2430,6 +2307,8 @@ public void testMergeFlowWithMissingDocs() throws IOException { [null, 5] | [5] [null, 7] | [7] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(10, builder.getStarTreeDocuments().size()); builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); int count = 0; for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { @@ -2508,6 +2387,8 @@ public void testMergeFlowWithMissingDocsWithZero() throws IOException { [null, 7] | [7] [null, null] | [12] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(6, builder.getStarTreeDocuments().size()); builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); int count = 0; for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { @@ -2592,6 +2473,8 @@ public void testMergeFlowWithMissingDocsWithZeroComplexCase() throws IOException [null, 7] | [7] [null, null] | [19] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(7, builder.getStarTreeDocuments().size()); builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); int count = 0; for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { @@ -2682,6 +2565,8 @@ public void testMergeFlowWithMissingDocsInSecondDim() throws IOException { [8, 8] | [8] [null, 7] | [7] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(10, builder.getStarTreeDocuments().size()); builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); int count = 0; for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { @@ -2765,6 +2650,8 @@ public void testMergeFlowWithDocsMissingAtTheEnd() throws IOException { [null, 5] | [5] [null, 7] | [7] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(10, builder.getStarTreeDocuments().size()); builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); int count = 0; for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { @@ -2835,6 +2722,8 @@ public void testMergeFlowWithEmptyFieldsInOneSegment() throws IOException { [4, 4] | [4] [null, 5] | [5] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(6, builder.getStarTreeDocuments().size()); builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); int count = 0; for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) {