diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java index 3115dce6c10a5..244449172a4e9 100644 --- a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java @@ -74,7 +74,8 @@ import java.util.function.Supplier; /** A {@link FieldMapper} for scaled floats. Values are internally multiplied - * by a scaling factor and rounded to the closest long. */ + * by a scaling factor and rounded to the closest long. + */ public class ScaledFloatFieldMapper extends ParametrizedFieldMapper { public static final String CONTENT_TYPE = "scaled_float"; @@ -162,11 +163,21 @@ public ScaledFloatFieldMapper build(BuilderContext context) { ); return new ScaledFloatFieldMapper(name, type, multiFieldsBuilder.build(this, context), copyTo.build(), this); } + + @Override + public boolean isDataCubeDimensionSupported() { + return true; + } + + @Override + public boolean isDataCubeMetricSupported() { + return true; + } } public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getSettings())); - public static final class ScaledFloatFieldType extends SimpleMappedFieldType implements NumericPointEncoder { + public static final class ScaledFloatFieldType extends SimpleMappedFieldType implements NumericPointEncoder, NumericFieldConverter { private final double scalingFactor; private final Double nullValue; @@ -340,6 +351,12 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { private double scale(Object input) { return new BigDecimal(Double.toString(parse(input))).multiply(BigDecimal.valueOf(scalingFactor)).doubleValue(); } + + @Override + public double toDoubleValue(Long value) { + double inverseScalingFactor = 1d / scalingFactor; + return value * inverseScalingFactor; + } } private final Explicit ignoreMalformed; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java index e79abe0f170b3..e7760f7ee6869 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java @@ -7,22 +7,22 @@ */ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.NumberFieldMapper; /** * Count value aggregator for star tree * * @opensearch.experimental */ -class CountValueAggregator implements ValueAggregator { +public class CountValueAggregator implements ValueAggregator { public static final long DEFAULT_INITIAL_VALUE = 1L; - private static final StarTreeNumericType VALUE_AGGREGATOR_TYPE = StarTreeNumericType.LONG; + private static final NumberFieldMapper.NumberType VALUE_AGGREGATOR_TYPE = NumberFieldMapper.NumberType.LONG; public CountValueAggregator() {} @Override - public StarTreeNumericType getAggregatedValueType() { + public NumberFieldMapper.NumberType getAggregatedValueType() { return VALUE_AGGREGATOR_TYPE; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/DocCountAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/DocCountAggregator.java index 0896fa54e9f46..a327eaf4147ce 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/DocCountAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/DocCountAggregator.java @@ -8,7 +8,7 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.NumberFieldMapper; /** * Aggregator to handle '_doc_count' field @@ -17,12 +17,12 @@ */ public class DocCountAggregator implements ValueAggregator { - private static final StarTreeNumericType VALUE_AGGREGATOR_TYPE = StarTreeNumericType.LONG; + private static final NumberFieldMapper.NumberType VALUE_AGGREGATOR_TYPE = NumberFieldMapper.NumberType.LONG; public DocCountAggregator() {} @Override - public StarTreeNumericType getAggregatedValueType() { + public NumberFieldMapper.NumberType getAggregatedValueType() { return VALUE_AGGREGATOR_TYPE; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MaxValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MaxValueAggregator.java index 8e7bb44238e40..908fb90dd1d36 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MaxValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MaxValueAggregator.java @@ -7,7 +7,7 @@ */ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.NumericFieldConverter; /** * Max value aggregator for star tree @@ -16,8 +16,8 @@ */ class MaxValueAggregator extends StatelessDoubleValueAggregator { - public MaxValueAggregator(StarTreeNumericType starTreeNumericType) { - super(starTreeNumericType, null); + public MaxValueAggregator(NumericFieldConverter numericFieldConverter) { + super(numericFieldConverter, null); } @Override diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java index 8b6db2a183bf8..32a96f070ccfc 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java @@ -8,8 +8,7 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; -import org.opensearch.index.fielddata.IndexNumericFieldData; +import org.opensearch.index.mapper.NumericFieldConverter; import java.util.Comparator; import java.util.Objects; @@ -27,15 +26,15 @@ public class MetricAggregatorInfo implements Comparable { private final MetricStat metricStat; private final String field; private final ValueAggregator valueAggregators; - private final StarTreeNumericType starTreeNumericType; + private final NumericFieldConverter numericFieldConverter; /** * Constructor for MetricAggregatorInfo */ - public MetricAggregatorInfo(MetricStat metricStat, String field, String starFieldName, IndexNumericFieldData.NumericType numericType) { + public MetricAggregatorInfo(MetricStat metricStat, String field, String starFieldName, NumericFieldConverter numericFieldConverter) { this.metricStat = metricStat; - this.starTreeNumericType = StarTreeNumericType.fromNumericType(numericType); - this.valueAggregators = ValueAggregatorFactory.getValueAggregator(metricStat, this.starTreeNumericType); + this.numericFieldConverter = numericFieldConverter; + this.valueAggregators = ValueAggregatorFactory.getValueAggregator(metricStat, this.numericFieldConverter); this.field = field; this.starFieldName = starFieldName; this.metric = toFieldName(); @@ -72,8 +71,8 @@ public ValueAggregator getValueAggregators() { /** * @return star tree aggregated value type */ - public StarTreeNumericType getAggregatedValueType() { - return starTreeNumericType; + public NumericFieldConverter getNumericFieldConverter() { + return numericFieldConverter; } /** diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MinValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MinValueAggregator.java index 46e9188b5dc2f..4fb4089f82125 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MinValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MinValueAggregator.java @@ -7,7 +7,7 @@ */ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.NumericFieldConverter; /** * Min value aggregator for star tree @@ -16,8 +16,8 @@ */ class MinValueAggregator extends StatelessDoubleValueAggregator { - public MinValueAggregator(StarTreeNumericType starTreeNumericType) { - super(starTreeNumericType, null); + public MinValueAggregator(NumericFieldConverter numericFieldConverter) { + super(numericFieldConverter, null); } @Override diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/StatelessDoubleValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/StatelessDoubleValueAggregator.java index 30a1c47c0ee9b..4ff8cef37f4bd 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/StatelessDoubleValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/StatelessDoubleValueAggregator.java @@ -7,7 +7,8 @@ */ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.index.mapper.NumericFieldConverter; /** * This is an abstract class that defines the common methods for all double value aggregators @@ -17,17 +18,17 @@ */ abstract class StatelessDoubleValueAggregator implements ValueAggregator { - protected final StarTreeNumericType starTreeNumericType; + protected final NumericFieldConverter numericFieldConverter; protected final Double identityValue; - private static final StarTreeNumericType VALUE_AGGREGATOR_TYPE = StarTreeNumericType.DOUBLE; + private static final NumberFieldMapper.NumberType VALUE_AGGREGATOR_TYPE = NumberFieldMapper.NumberType.DOUBLE; - public StatelessDoubleValueAggregator(StarTreeNumericType starTreeNumericType, Double identityValue) { - this.starTreeNumericType = starTreeNumericType; + public StatelessDoubleValueAggregator(NumericFieldConverter numericFieldConverter, Double identityValue) { + this.numericFieldConverter = numericFieldConverter; this.identityValue = identityValue; } @Override - public StarTreeNumericType getAggregatedValueType() { + public NumberFieldMapper.NumberType getAggregatedValueType() { return VALUE_AGGREGATOR_TYPE; } @@ -36,7 +37,7 @@ public Double getInitialAggregatedValueForSegmentDocValue(Long segmentDocValue) if (segmentDocValue == null) { return getIdentityMetricValue(); } - return starTreeNumericType.getDoubleValue(segmentDocValue); + return numericFieldConverter.toDoubleValue(segmentDocValue); } @Override @@ -57,7 +58,7 @@ public Double toAggregatedValueType(Long value) { if (value == null) { return getIdentityMetricValue(); } - return VALUE_AGGREGATOR_TYPE.getDoubleValue(value); + return VALUE_AGGREGATOR_TYPE.toDoubleValue(value); } catch (Exception e) { throw new IllegalStateException("Cannot convert " + value + " to sortable aggregation type", e); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java index ef97a9b603df3..2ff82283004bc 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java @@ -7,7 +7,8 @@ */ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.index.mapper.NumericFieldConverter; import org.opensearch.search.aggregations.metrics.CompensatedSum; /** @@ -22,17 +23,17 @@ */ class SumValueAggregator implements ValueAggregator { - private final StarTreeNumericType starTreeNumericType; - private static final StarTreeNumericType VALUE_AGGREGATOR_TYPE = StarTreeNumericType.DOUBLE; + private final NumericFieldConverter numericFieldConverter; + private static final NumberFieldMapper.NumberType VALUE_AGGREGATOR_TYPE = NumberFieldMapper.NumberType.DOUBLE; private CompensatedSum kahanSummation = new CompensatedSum(0, 0); - public SumValueAggregator(StarTreeNumericType starTreeNumericType) { - this.starTreeNumericType = starTreeNumericType; + public SumValueAggregator(NumericFieldConverter numericFieldConverter) { + this.numericFieldConverter = numericFieldConverter; } @Override - public StarTreeNumericType getAggregatedValueType() { + public NumberFieldMapper.NumberType getAggregatedValueType() { return VALUE_AGGREGATOR_TYPE; } @@ -41,7 +42,7 @@ public Double getInitialAggregatedValueForSegmentDocValue(Long segmentDocValue) kahanSummation.reset(0, 0); // add takes care of the sum and compensation internally if (segmentDocValue != null) { - kahanSummation.add(starTreeNumericType.getDoubleValue(segmentDocValue)); + kahanSummation.add(numericFieldConverter.toDoubleValue(segmentDocValue)); } else { kahanSummation.add(getIdentityMetricValue()); } @@ -55,7 +56,7 @@ public Double mergeAggregatedValueAndSegmentValue(Double value, Long segmentDocV assert value == null || kahanSummation.value() == value; // add takes care of the sum and compensation internally if (segmentDocValue != null) { - kahanSummation.add(starTreeNumericType.getDoubleValue(segmentDocValue)); + kahanSummation.add(numericFieldConverter.toDoubleValue(segmentDocValue)); } else { kahanSummation.add(getIdentityMetricValue()); } @@ -92,7 +93,7 @@ public Double toAggregatedValueType(Long value) { if (value == null) { return getIdentityMetricValue(); } - return VALUE_AGGREGATOR_TYPE.getDoubleValue(value); + return VALUE_AGGREGATOR_TYPE.toDoubleValue(value); } catch (Exception e) { throw new IllegalStateException("Cannot convert " + value + " to sortable aggregation type", e); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java index d5ca7f3493087..b328140b6f519 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java @@ -7,7 +7,7 @@ */ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.NumberFieldMapper; /** * A value aggregator that pre-aggregates on the input values for a specific type of aggregation. @@ -19,7 +19,7 @@ public interface ValueAggregator { /** * Returns the data type of the aggregated value. */ - StarTreeNumericType getAggregatedValueType(); + NumberFieldMapper.NumberType getAggregatedValueType(); /** * Returns the initial aggregated value. diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java index bdc381110365d..6120d7b4b9d60 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java @@ -8,7 +8,7 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.NumericFieldConverter; /** * Value aggregator factory for a given aggregation type @@ -22,20 +22,20 @@ private ValueAggregatorFactory() {} * Returns a new instance of value aggregator for the given aggregation type. * * @param aggregationType Aggregation type - * @param starTreeNumericType Numeric type associated with star tree field ( as specified in index mapping ) + * @param numericFieldConverter Numeric type converter associated with star tree field ( as specified in index mapping ) * @return Value aggregator */ - public static ValueAggregator getValueAggregator(MetricStat aggregationType, StarTreeNumericType starTreeNumericType) { + public static ValueAggregator getValueAggregator(MetricStat aggregationType, NumericFieldConverter numericFieldConverter) { switch (aggregationType) { // avg aggregator will be covered in the part of query (using count and sum) case SUM: - return new SumValueAggregator(starTreeNumericType); + return new SumValueAggregator(numericFieldConverter); case VALUE_COUNT: return new CountValueAggregator(); case MIN: - return new MinValueAggregator(starTreeNumericType); + return new MinValueAggregator(numericFieldConverter); case MAX: - return new MaxValueAggregator(starTreeNumericType); + return new MaxValueAggregator(numericFieldConverter); case DOC_COUNT: return new DocCountAggregator(); default: diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java deleted file mode 100644 index 57fe573a6a93c..0000000000000 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype; - -import org.opensearch.index.fielddata.IndexNumericFieldData; - -import java.util.function.Function; - -/** - * Enum to map Star Tree Numeric Types to Lucene's Numeric Type - * - * @opensearch.experimental - */ -public enum StarTreeNumericType { - - // TODO: Handle scaled floats - HALF_FLOAT(IndexNumericFieldData.NumericType.HALF_FLOAT, StarTreeNumericTypeConverters::halfFloatPointToDouble), - FLOAT(IndexNumericFieldData.NumericType.FLOAT, StarTreeNumericTypeConverters::floatPointToDouble), - LONG(IndexNumericFieldData.NumericType.LONG, StarTreeNumericTypeConverters::longToDouble), - DOUBLE(IndexNumericFieldData.NumericType.DOUBLE, StarTreeNumericTypeConverters::sortableLongtoDouble), - INT(IndexNumericFieldData.NumericType.INT, StarTreeNumericTypeConverters::intToDouble), - SHORT(IndexNumericFieldData.NumericType.SHORT, StarTreeNumericTypeConverters::shortToDouble), - BYTE(IndexNumericFieldData.NumericType.BYTE, StarTreeNumericTypeConverters::bytesToDouble), - UNSIGNED_LONG(IndexNumericFieldData.NumericType.UNSIGNED_LONG, StarTreeNumericTypeConverters::unsignedlongToDouble); - - final IndexNumericFieldData.NumericType numericType; - final Function converter; - - StarTreeNumericType(IndexNumericFieldData.NumericType numericType, Function converter) { - this.numericType = numericType; - this.converter = converter; - } - - public double getDoubleValue(long rawValue) { - return this.converter.apply(rawValue); - } - - public static StarTreeNumericType fromNumericType(IndexNumericFieldData.NumericType numericType) { - switch (numericType) { - case HALF_FLOAT: - return StarTreeNumericType.HALF_FLOAT; - case FLOAT: - return StarTreeNumericType.FLOAT; - case LONG: - return StarTreeNumericType.LONG; - case DOUBLE: - return StarTreeNumericType.DOUBLE; - case INT: - return StarTreeNumericType.INT; - case SHORT: - return StarTreeNumericType.SHORT; - case UNSIGNED_LONG: - return StarTreeNumericType.UNSIGNED_LONG; - case BYTE: - return StarTreeNumericType.BYTE; - default: - throw new UnsupportedOperationException("Unknown numeric type [" + numericType + "]"); - } - } -} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java deleted file mode 100644 index eb7647c4f9851..0000000000000 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype; - -import org.apache.lucene.sandbox.document.HalfFloatPoint; -import org.apache.lucene.util.NumericUtils; -import org.opensearch.common.Numbers; -import org.opensearch.common.annotation.ExperimentalApi; - -/** - * Numeric converters used during aggregations of metric values - * - * @opensearch.experimental - */ -@ExperimentalApi -public class StarTreeNumericTypeConverters { - - public static double halfFloatPointToDouble(Long value) { - return HalfFloatPoint.sortableShortToHalfFloat((short) value.longValue()); - } - - public static double floatPointToDouble(Long value) { - return NumericUtils.sortableIntToFloat((int) value.longValue()); - } - - public static double longToDouble(Long value) { - return (double) value; - } - - public static double intToDouble(Long value) { - return (double) value; - } - - public static double shortToDouble(Long value) { - return (double) value; - } - - public static Double sortableLongtoDouble(Long value) { - return NumericUtils.sortableLongToDouble(value); - } - - public static double unsignedlongToDouble(Long value) { - return Numbers.unsignedLongToDouble(value); - } - - public static double bytesToDouble(Long value) { - byte[] bytes = new byte[8]; - NumericUtils.longToSortableBytes(value, bytes, 0); - return NumericUtils.sortableLongToDouble(NumericUtils.sortableBytesToLong(bytes, 0)); - } - -} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java deleted file mode 100644 index fe5c2a7ceb254..0000000000000 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java +++ /dev/null @@ -1,14 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/** - * Numeric Types for Composite Index Star Tree - * - * @opensearch.experimental - */ -package org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractDocumentsFileManager.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractDocumentsFileManager.java index 4214a46b2fc1c..f3b8573be62df 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractDocumentsFileManager.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractDocumentsFileManager.java @@ -19,8 +19,8 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericTypeConverters; import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeDocumentBitSetUtil; +import org.opensearch.index.mapper.NumberFieldMapper; import java.io.Closeable; import java.io.IOException; @@ -165,7 +165,7 @@ protected long readMetrics(RandomAccessInput input, long offset, int numMetrics, case DOUBLE: long val = input.readLong(offset); if (isAggregatedDoc) { - metrics[i] = StarTreeNumericTypeConverters.sortableLongtoDouble(val); + metrics[i] = NumberFieldMapper.NumberType.DOUBLE.toDoubleValue(val); } else { metrics[i] = val; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index ddcf02cc6291a..4d68839014a6a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -28,11 +28,12 @@ import org.opensearch.index.compositeindex.datacube.startree.aggregators.ValueAggregator; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.TreeNode; -import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.mapper.DocCountFieldMapper; import org.opensearch.index.mapper.Mapper; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.index.mapper.NumericFieldConverter; +import org.opensearch.index.mapper.ParametrizedFieldMapper; import java.io.IOException; import java.util.ArrayList; @@ -124,7 +125,7 @@ public List generateMetricAggregatorInfos(MapperService ma MetricStat.DOC_COUNT, metric.getField(), starTreeField.getName(), - IndexNumericFieldData.NumericType.LONG + NumberFieldMapper.NumberType.LONG ); metricAggregatorInfos.add(metricAggregatorInfo); continue; @@ -133,10 +134,11 @@ public List generateMetricAggregatorInfos(MapperService ma if (metricStat.isDerivedMetric()) { continue; } - IndexNumericFieldData.NumericType numericType; + NumericFieldConverter numericFieldConverter; Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper(metric.getField()); - if (fieldMapper instanceof NumberFieldMapper) { - numericType = ((NumberFieldMapper) fieldMapper).fieldType().numericType(); + if (fieldMapper instanceof ParametrizedFieldMapper + && ((ParametrizedFieldMapper) fieldMapper).fieldType() instanceof NumericFieldConverter) { + numericFieldConverter = (NumericFieldConverter) ((ParametrizedFieldMapper) fieldMapper).fieldType(); } else { logger.error("unsupported mapper type"); throw new IllegalStateException("unsupported mapper type"); @@ -146,7 +148,7 @@ public List generateMetricAggregatorInfos(MapperService ma metricStat, metric.getField(), starTreeField.getName(), - numericType + numericFieldConverter ); metricAggregatorInfos.add(metricAggregatorInfo); } diff --git a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java index cf8703209fb37..c729a47ca4a09 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java @@ -331,6 +331,11 @@ public DateFieldMapper build(BuilderContext context) { Long nullTimestamp = parseNullValue(ft); return new DateFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), nullTimestamp, resolution, this); } + + @Override + public boolean isDataCubeDimensionSupported() { + return true; + } } public static final TypeParser MILLIS_PARSER = new TypeParser((n, c) -> { diff --git a/server/src/main/java/org/opensearch/index/mapper/Mapper.java b/server/src/main/java/org/opensearch/index/mapper/Mapper.java index 46a5050d4fc18..ddd18edb71e5f 100644 --- a/server/src/main/java/org/opensearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/Mapper.java @@ -100,7 +100,7 @@ public Version indexCreatedVersionOrDefault(@Nullable Version defaultValue) { * @opensearch.api */ @PublicApi(since = "1.0.0") - public abstract static class Builder { + public abstract static class Builder implements MapperBuilderSettings { public String name; diff --git a/server/src/main/java/org/opensearch/index/mapper/MapperBuilderSettings.java b/server/src/main/java/org/opensearch/index/mapper/MapperBuilderSettings.java new file mode 100644 index 0000000000000..cd4df866ab75b --- /dev/null +++ b/server/src/main/java/org/opensearch/index/mapper/MapperBuilderSettings.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +/** + * An interface that defines settings for MapperBuilder implementations. + * + * @opensearch.experimental + */ +public interface MapperBuilderSettings { + + /** + * Indicates whether the implementation supports data cube dimensions. + * + * @return true if data cube dimensions are supported, false otherwise + */ + default boolean isDataCubeDimensionSupported() { + return false; + } + + /** + * Indicates whether the implementation supports data cube metrics. + * + * @return true if data cube metrics are supported, false otherwise + */ + default boolean isDataCubeMetricSupported() { + return false; + } + +} diff --git a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java index 9286b5c64b5f2..5566a2cc44a1a 100644 --- a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java @@ -172,6 +172,16 @@ public NumberFieldMapper build(BuilderContext context) { MappedFieldType ft = new NumberFieldType(buildFullName(context), this); return new NumberFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), this); } + + @Override + public boolean isDataCubeDimensionSupported() { + return true; + } + + @Override + public boolean isDataCubeMetricSupported() { + return true; + } } /** @@ -179,7 +189,7 @@ public NumberFieldMapper build(BuilderContext context) { * * @opensearch.internal */ - public enum NumberType implements NumericPointEncoder { + public enum NumberType implements NumericPointEncoder, NumericFieldConverter { HALF_FLOAT("half_float", NumericType.HALF_FLOAT) { @Override public Float parse(Object value, boolean coerce) { @@ -209,6 +219,11 @@ public byte[] encodePoint(Number value) { return point; } + @Override + public double toDoubleValue(Long value) { + return HalfFloatPoint.sortableShortToHalfFloat((short) value.longValue()); + } + @Override public Float parse(XContentParser parser, boolean coerce) throws IOException { float parsed = parser.floatValue(coerce); @@ -353,6 +368,11 @@ public byte[] encodePoint(Number value) { return point; } + @Override + public double toDoubleValue(Long value) { + return NumericUtils.sortableIntToFloat((int) value.longValue()); + } + @Override public Float parse(XContentParser parser, boolean coerce) throws IOException { float parsed = parser.floatValue(coerce); @@ -486,6 +506,11 @@ public byte[] encodePoint(Number value) { return point; } + @Override + public double toDoubleValue(Long value) { + return NumericUtils.sortableLongToDouble(value); + } + @Override public Double parse(XContentParser parser, boolean coerce) throws IOException { double parsed = parser.doubleValue(coerce); @@ -618,6 +643,13 @@ public byte[] encodePoint(Number value) { return point; } + @Override + public double toDoubleValue(Long value) { + byte[] bytes = new byte[8]; + NumericUtils.longToSortableBytes(value, bytes, 0); + return NumericUtils.sortableLongToDouble(NumericUtils.sortableBytesToLong(bytes, 0)); + } + @Override public Short parse(XContentParser parser, boolean coerce) throws IOException { int value = parser.intValue(coerce); @@ -697,6 +729,11 @@ public byte[] encodePoint(Number value) { return point; } + @Override + public double toDoubleValue(Long value) { + return (double) value; + } + @Override public Short parse(XContentParser parser, boolean coerce) throws IOException { return parser.shortValue(coerce); @@ -772,6 +809,11 @@ public byte[] encodePoint(Number value) { return point; } + @Override + public double toDoubleValue(Long value) { + return (double) value; + } + @Override public Integer parse(XContentParser parser, boolean coerce) throws IOException { return parser.intValue(coerce); @@ -943,6 +985,11 @@ public byte[] encodePoint(Number value) { return point; } + @Override + public double toDoubleValue(Long value) { + return (double) value; + } + @Override public Long parse(XContentParser parser, boolean coerce) throws IOException { return parser.longValue(coerce); @@ -1070,6 +1117,11 @@ public byte[] encodePoint(Number value) { return point; } + @Override + public double toDoubleValue(Long value) { + return Numbers.unsignedLongToDouble(value); + } + @Override public BigInteger parse(XContentParser parser, boolean coerce) throws IOException { return parser.bigIntegerValue(coerce); @@ -1486,7 +1538,7 @@ protected String toString(byte[] value) { * * @opensearch.internal */ - public static class NumberFieldType extends SimpleMappedFieldType implements NumericPointEncoder { + public static class NumberFieldType extends SimpleMappedFieldType implements NumericPointEncoder, NumericFieldConverter { private final NumberType type; private final boolean coerce; @@ -1652,6 +1704,11 @@ public Number parsePoint(byte[] value) { public byte[] encodePoint(Number value) { return type.encodePoint(value); } + + @Override + public double toDoubleValue(Long value) { + return type.toDoubleValue(value); + } } private final NumberType type; diff --git a/server/src/main/java/org/opensearch/index/mapper/NumericFieldConverter.java b/server/src/main/java/org/opensearch/index/mapper/NumericFieldConverter.java new file mode 100644 index 0000000000000..a9048f9264f51 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/mapper/NumericFieldConverter.java @@ -0,0 +1,30 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +/** + *

+ * Implementations of this interface should define the conversion logic + * from a sortable long value to a double value, taking into account any necessary + * scaling, normalization, or other transformations required by the specific + * field type. + * + * @opensearch.experimental + */ +public interface NumericFieldConverter { + + /** + * Converts the given long value to a double value. + * + * @param value the long value to be converted + * @return the corresponding double value + */ + double toDoubleValue(Long value); + +} diff --git a/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java b/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java index e52d6a621e4e8..5497b468675f3 100644 --- a/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java @@ -318,11 +318,11 @@ protected List> getParameters() { } private static boolean isBuilderAllowedForDimension(Mapper.Builder builder) { - return ALLOWED_DIMENSION_MAPPER_BUILDERS.stream().anyMatch(allowedType -> allowedType.isInstance(builder)); + return builder.isDataCubeDimensionSupported(); } private static boolean isBuilderAllowedForMetric(Mapper.Builder builder) { - return ALLOWED_METRIC_MAPPER_BUILDERS.stream().anyMatch(allowedType -> allowedType.isInstance(builder)); + return builder.isDataCubeMetricSupported(); } private Optional findMapperBuilderByName(String field, List mappersBuilders) { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/AbstractValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/AbstractValueAggregatorTests.java index 36f75834abba8..5bd9033a99157 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/AbstractValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/AbstractValueAggregatorTests.java @@ -10,7 +10,8 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.index.mapper.NumericFieldConverter; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; @@ -21,27 +22,27 @@ public abstract class AbstractValueAggregatorTests extends OpenSearchTestCase { private ValueAggregator aggregator; - protected StarTreeNumericType starTreeNumericType; + protected NumericFieldConverter numericFieldConverter; - public AbstractValueAggregatorTests(StarTreeNumericType starTreeNumericType) { - this.starTreeNumericType = starTreeNumericType; + public AbstractValueAggregatorTests(NumericFieldConverter numericFieldConverter) { + this.numericFieldConverter = numericFieldConverter; } @Before public void setup() { - aggregator = getValueAggregator(starTreeNumericType); + aggregator = getValueAggregator(numericFieldConverter); } @ParametersFactory public static Collection parameters() { List parameters = new ArrayList<>(); - for (StarTreeNumericType starTreeNumericType : StarTreeNumericType.values()) { - parameters.add(new Object[] { starTreeNumericType }); + for (NumericFieldConverter numericFieldConverter : NumberFieldMapper.NumberType.values()) { + parameters.add(new Object[] { numericFieldConverter }); } return parameters; } - public abstract ValueAggregator getValueAggregator(StarTreeNumericType starTreeNumericType); + public abstract ValueAggregator getValueAggregator(NumericFieldConverter numericFieldConverter); public void testGetInitialAggregatedValueForSegmentDocNullValue() { assertEquals(aggregator.getIdentityMetricValue(), aggregator.getInitialAggregatedValueForSegmentDocValue(null)); @@ -61,6 +62,13 @@ public void testGetInitialAggregatedNullValue() { public void testGetInitialAggregatedValueForSegmentDocValue() { long randomLong = randomLong(); - assertEquals(starTreeNumericType.getDoubleValue(randomLong), aggregator.getInitialAggregatedValueForSegmentDocValue(randomLong)); + if (aggregator instanceof CountValueAggregator) { + assertEquals(CountValueAggregator.DEFAULT_INITIAL_VALUE, aggregator.getInitialAggregatedValueForSegmentDocValue(randomLong())); + } else { + assertEquals( + numericFieldConverter.toDoubleValue(randomLong), + aggregator.getInitialAggregatedValueForSegmentDocValue(randomLong) + ); + } } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java index b270c1b1bc26c..cbd48cc41897a 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java @@ -8,14 +8,14 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.NumericFieldConverter; public class CountValueAggregatorTests extends AbstractValueAggregatorTests { private CountValueAggregator aggregator; - public CountValueAggregatorTests(StarTreeNumericType starTreeNumericType) { - super(starTreeNumericType); + public CountValueAggregatorTests(NumericFieldConverter numericFieldConverter) { + super(numericFieldConverter); } public void testMergeAggregatedValueAndSegmentValue() { @@ -52,7 +52,7 @@ public void testIdentityMetricValue() { } @Override - public ValueAggregator getValueAggregator(StarTreeNumericType starTreeNumericType) { + public ValueAggregator getValueAggregator(NumericFieldConverter numericFieldConverter) { aggregator = new CountValueAggregator(); return aggregator; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/DocCountAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/DocCountAggregatorTests.java index 2765629aa5950..16540d75acbce 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/DocCountAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/DocCountAggregatorTests.java @@ -8,7 +8,7 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.NumericFieldConverter; /** * Unit tests for {@link DocCountAggregator}. @@ -17,8 +17,8 @@ public class DocCountAggregatorTests extends AbstractValueAggregatorTests { private DocCountAggregator aggregator; - public DocCountAggregatorTests(StarTreeNumericType starTreeNumericType) { - super(starTreeNumericType); + public DocCountAggregatorTests(NumericFieldConverter numericFieldConverter) { + super(numericFieldConverter); } public void testMergeAggregatedValueAndSegmentValue() { @@ -62,7 +62,7 @@ public void testIdentityMetricValue() { } @Override - public ValueAggregator getValueAggregator(StarTreeNumericType starTreeNumericType) { + public ValueAggregator getValueAggregator(NumericFieldConverter numericFieldConverter) { aggregator = new DocCountAggregator(); return aggregator; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MaxValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MaxValueAggregatorTests.java index b103416251c46..6c703dbc2f40f 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MaxValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MaxValueAggregatorTests.java @@ -9,27 +9,32 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; import org.apache.lucene.util.NumericUtils; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.index.mapper.NumericFieldConverter; public class MaxValueAggregatorTests extends AbstractValueAggregatorTests { private MaxValueAggregator aggregator; - public MaxValueAggregatorTests(StarTreeNumericType starTreeNumericType) { - super(starTreeNumericType); + public MaxValueAggregatorTests(NumericFieldConverter numericFieldConverter) { + super(numericFieldConverter); } public void testMergeAggregatedValueAndSegmentValue() { Long randomLong = randomLong(); double randomDouble = randomDouble(); assertEquals( - Math.max(starTreeNumericType.getDoubleValue(randomLong), randomDouble), + Math.max(numericFieldConverter.toDoubleValue(randomLong), randomDouble), aggregator.mergeAggregatedValueAndSegmentValue(randomDouble, randomLong), 0.0 ); - assertEquals(starTreeNumericType.getDoubleValue(randomLong), aggregator.mergeAggregatedValueAndSegmentValue(null, randomLong), 0.0); + assertEquals( + numericFieldConverter.toDoubleValue(randomLong), + aggregator.mergeAggregatedValueAndSegmentValue(null, randomLong), + 0.0 + ); assertEquals(randomDouble, aggregator.mergeAggregatedValueAndSegmentValue(randomDouble, null), 0.0); - assertEquals(Math.max(2.0, starTreeNumericType.getDoubleValue(3L)), aggregator.mergeAggregatedValueAndSegmentValue(2.0, 3L), 0.0); + assertEquals(Math.max(2.0, numericFieldConverter.toDoubleValue(3L)), aggregator.mergeAggregatedValueAndSegmentValue(2.0, 3L), 0.0); } public void testMergeAggregatedValues() { @@ -46,7 +51,7 @@ public void testGetInitialAggregatedValue() { } public void testToAggregatedValueType() { - MaxValueAggregator aggregator = new MaxValueAggregator(StarTreeNumericType.DOUBLE); + MaxValueAggregator aggregator = new MaxValueAggregator(NumberFieldMapper.NumberType.DOUBLE); long randomLong = randomLong(); assertEquals(NumericUtils.sortableLongToDouble(randomLong), aggregator.toAggregatedValueType(randomLong), 0.0); } @@ -56,8 +61,8 @@ public void testIdentityMetricValue() { } @Override - public ValueAggregator getValueAggregator(StarTreeNumericType starTreeNumericType) { - aggregator = new MaxValueAggregator(starTreeNumericType); + public ValueAggregator getValueAggregator(NumericFieldConverter numericFieldConverter) { + aggregator = new MaxValueAggregator(numericFieldConverter); return aggregator; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java index 62671ffa03b82..5588cc1b366c9 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java @@ -9,7 +9,7 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.index.fielddata.IndexNumericFieldData; +import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.test.OpenSearchTestCase; public class MetricAggregatorInfoTests extends OpenSearchTestCase { @@ -19,7 +19,7 @@ public void testConstructor() { MetricStat.SUM, "column1", "star_tree_field", - IndexNumericFieldData.NumericType.DOUBLE + NumberFieldMapper.NumberType.DOUBLE ); assertEquals(MetricStat.SUM, pair.getMetricStat()); assertEquals("column1", pair.getField()); @@ -30,7 +30,7 @@ public void testCountStarConstructor() { MetricStat.VALUE_COUNT, "anything", "star_tree_field", - IndexNumericFieldData.NumericType.DOUBLE + NumberFieldMapper.NumberType.DOUBLE ); assertEquals(MetricStat.VALUE_COUNT, pair.getMetricStat()); assertEquals("anything", pair.getField()); @@ -41,7 +41,7 @@ public void testToFieldName() { MetricStat.SUM, "column2", "star_tree_field", - IndexNumericFieldData.NumericType.DOUBLE + NumberFieldMapper.NumberType.DOUBLE ); assertEquals("star_tree_field_column2_sum", pair.toFieldName()); } @@ -51,23 +51,20 @@ public void testEquals() { MetricStat.SUM, "column1", "star_tree_field", - IndexNumericFieldData.NumericType.DOUBLE + NumberFieldMapper.NumberType.DOUBLE ); MetricAggregatorInfo pair2 = new MetricAggregatorInfo( MetricStat.SUM, "column1", "star_tree_field", - IndexNumericFieldData.NumericType.DOUBLE + NumberFieldMapper.NumberType.DOUBLE ); assertEquals(pair1, pair2); assertNotEquals( pair1, - new MetricAggregatorInfo(MetricStat.VALUE_COUNT, "column1", "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE) - ); - assertNotEquals( - pair1, - new MetricAggregatorInfo(MetricStat.SUM, "column2", "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE) + new MetricAggregatorInfo(MetricStat.VALUE_COUNT, "column1", "star_tree_field", NumberFieldMapper.NumberType.DOUBLE) ); + assertNotEquals(pair1, new MetricAggregatorInfo(MetricStat.SUM, "column2", "star_tree_field", NumberFieldMapper.NumberType.DOUBLE)); } public void testHashCode() { @@ -75,13 +72,13 @@ public void testHashCode() { MetricStat.SUM, "column1", "star_tree_field", - IndexNumericFieldData.NumericType.DOUBLE + NumberFieldMapper.NumberType.DOUBLE ); MetricAggregatorInfo pair2 = new MetricAggregatorInfo( MetricStat.SUM, "column1", "star_tree_field", - IndexNumericFieldData.NumericType.DOUBLE + NumberFieldMapper.NumberType.DOUBLE ); assertEquals(pair1.hashCode(), pair2.hashCode()); } @@ -91,19 +88,19 @@ public void testCompareTo() { MetricStat.SUM, "column1", "star_tree_field", - IndexNumericFieldData.NumericType.DOUBLE + NumberFieldMapper.NumberType.DOUBLE ); MetricAggregatorInfo pair2 = new MetricAggregatorInfo( MetricStat.SUM, "column2", "star_tree_field", - IndexNumericFieldData.NumericType.DOUBLE + NumberFieldMapper.NumberType.DOUBLE ); MetricAggregatorInfo pair3 = new MetricAggregatorInfo( MetricStat.VALUE_COUNT, "column1", "star_tree_field", - IndexNumericFieldData.NumericType.DOUBLE + NumberFieldMapper.NumberType.DOUBLE ); assertTrue(pair1.compareTo(pair2) < 0); assertTrue(pair2.compareTo(pair1) > 0); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MinValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MinValueAggregatorTests.java index 013c60d8a1b91..f997d33005c9f 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MinValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MinValueAggregatorTests.java @@ -9,26 +9,31 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; import org.apache.lucene.util.NumericUtils; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.index.mapper.NumericFieldConverter; public class MinValueAggregatorTests extends AbstractValueAggregatorTests { private MinValueAggregator aggregator; - public MinValueAggregatorTests(StarTreeNumericType starTreeNumericType) { - super(starTreeNumericType); + public MinValueAggregatorTests(NumericFieldConverter numericFieldConverter) { + super(numericFieldConverter); } public void testMergeAggregatedValueAndSegmentValue() { Long randomLong = randomLong(); double randomDouble = randomDouble(); assertEquals( - Math.min(starTreeNumericType.getDoubleValue(randomLong), randomDouble), + Math.min(numericFieldConverter.toDoubleValue(randomLong), randomDouble), aggregator.mergeAggregatedValueAndSegmentValue(randomDouble, randomLong), 0.0 ); - assertEquals(starTreeNumericType.getDoubleValue(randomLong), aggregator.mergeAggregatedValueAndSegmentValue(null, randomLong), 0.0); + assertEquals( + numericFieldConverter.toDoubleValue(randomLong), + aggregator.mergeAggregatedValueAndSegmentValue(null, randomLong), + 0.0 + ); assertEquals(randomDouble, aggregator.mergeAggregatedValueAndSegmentValue(randomDouble, null), 0.0); - assertEquals(Math.min(2.0, starTreeNumericType.getDoubleValue(3L)), aggregator.mergeAggregatedValueAndSegmentValue(2.0, 3L), 0.0); + assertEquals(Math.min(2.0, numericFieldConverter.toDoubleValue(3L)), aggregator.mergeAggregatedValueAndSegmentValue(2.0, 3L), 0.0); } public void testMergeAggregatedValues() { @@ -45,7 +50,7 @@ public void testGetInitialAggregatedValue() { } public void testToAggregatedValueType() { - MinValueAggregator aggregator = new MinValueAggregator(StarTreeNumericType.DOUBLE); + MinValueAggregator aggregator = new MinValueAggregator(NumberFieldMapper.NumberType.DOUBLE); long randomLong = randomLong(); assertEquals(NumericUtils.sortableLongToDouble(randomLong), aggregator.toAggregatedValueType(randomLong), 0.0); } @@ -55,8 +60,8 @@ public void testIdentityMetricValue() { } @Override - public ValueAggregator getValueAggregator(StarTreeNumericType starTreeNumericType) { - aggregator = new MinValueAggregator(starTreeNumericType); + public ValueAggregator getValueAggregator(NumericFieldConverter numericFieldConverter) { + aggregator = new MinValueAggregator(numericFieldConverter); return aggregator; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/StaticValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/StaticValueAggregatorTests.java index 487bd54602503..2ba29a1165dc8 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/StaticValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/StaticValueAggregatorTests.java @@ -9,7 +9,7 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; import org.apache.lucene.util.NumericUtils; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.search.aggregations.metrics.CompensatedSum; import org.opensearch.test.OpenSearchTestCase; @@ -39,7 +39,7 @@ public void testKahanSummation() { private static double getAggregatedValue(double[] numbers) { // explicitly took double to test for most precision // hard to run similar tests for different data types dynamically as inputs and precision vary - SumValueAggregator aggregator = new SumValueAggregator(StarTreeNumericType.DOUBLE); + SumValueAggregator aggregator = new SumValueAggregator(NumberFieldMapper.NumberType.DOUBLE); double aggregatedValue = aggregator.getInitialAggregatedValueForSegmentDocValue(NumericUtils.doubleToSortableLong(numbers[0])); aggregatedValue = aggregator.mergeAggregatedValueAndSegmentValue(aggregatedValue, NumericUtils.doubleToSortableLong(numbers[1])); aggregatedValue = aggregator.mergeAggregatedValueAndSegmentValue(aggregatedValue, NumericUtils.doubleToSortableLong(numbers[2])); @@ -65,7 +65,7 @@ private double normalSum(double[] numbers) { public void testMaxAggregatorExtremeValues() { double[] numbers = { Double.MAX_VALUE, Double.MIN_VALUE, 0.0, Double.MAX_VALUE + 1 }; double expected = Double.MAX_VALUE + 1; - MaxValueAggregator aggregator = new MaxValueAggregator(StarTreeNumericType.DOUBLE); + MaxValueAggregator aggregator = new MaxValueAggregator(NumberFieldMapper.NumberType.DOUBLE); double aggregatedValue = aggregator.getInitialAggregatedValueForSegmentDocValue(NumericUtils.doubleToSortableLong(numbers[0])); for (int i = 1; i < numbers.length; i++) { aggregatedValue = aggregator.mergeAggregatedValueAndSegmentValue( @@ -85,7 +85,7 @@ public void testMaxAggregatorExtremeValues_Infinity() { Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY }; double expected = Double.POSITIVE_INFINITY; - MaxValueAggregator aggregator = new MaxValueAggregator(StarTreeNumericType.DOUBLE); + MaxValueAggregator aggregator = new MaxValueAggregator(NumberFieldMapper.NumberType.DOUBLE); double aggregatedValue = aggregator.getInitialAggregatedValueForSegmentDocValue(NumericUtils.doubleToSortableLong(numbers[0])); for (int i = 1; i < numbers.length; i++) { aggregatedValue = aggregator.mergeAggregatedValueAndSegmentValue( @@ -99,7 +99,7 @@ public void testMaxAggregatorExtremeValues_Infinity() { public void testMinAggregatorExtremeValues() { double[] numbers = { Double.MAX_VALUE, Double.MIN_VALUE - 1, 0.0, Double.MAX_VALUE + 1 }; double expected = Double.MIN_VALUE - 1; - MinValueAggregator aggregator = new MinValueAggregator(StarTreeNumericType.DOUBLE); + MinValueAggregator aggregator = new MinValueAggregator(NumberFieldMapper.NumberType.DOUBLE); double aggregatedValue = aggregator.getInitialAggregatedValueForSegmentDocValue(NumericUtils.doubleToSortableLong(numbers[0])); for (int i = 1; i < numbers.length; i++) { aggregatedValue = aggregator.mergeAggregatedValueAndSegmentValue( @@ -119,7 +119,7 @@ public void testMinAggregatorExtremeValues_Infinity() { Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY }; double expected = Double.NEGATIVE_INFINITY; - MinValueAggregator aggregator = new MinValueAggregator(StarTreeNumericType.DOUBLE); + MinValueAggregator aggregator = new MinValueAggregator(NumberFieldMapper.NumberType.DOUBLE); double aggregatedValue = aggregator.getInitialAggregatedValueForSegmentDocValue(NumericUtils.doubleToSortableLong(numbers[0])); for (int i = 1; i < numbers.length; i++) { aggregatedValue = aggregator.mergeAggregatedValueAndSegmentValue( diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java index 44c7f17a276b4..e5dfdd9a78058 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java @@ -8,19 +8,19 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.NumericFieldConverter; public class SumValueAggregatorTests extends AbstractValueAggregatorTests { private SumValueAggregator aggregator; - public SumValueAggregatorTests(StarTreeNumericType starTreeNumericType) { - super(starTreeNumericType); + public SumValueAggregatorTests(NumericFieldConverter numericFieldConverter) { + super(numericFieldConverter); } @Override - public ValueAggregator getValueAggregator(StarTreeNumericType starTreeNumericType) { - aggregator = new SumValueAggregator(starTreeNumericType); + public ValueAggregator getValueAggregator(NumericFieldConverter numericFieldConverter) { + aggregator = new SumValueAggregator(numericFieldConverter); return aggregator; } @@ -29,7 +29,7 @@ public void testMergeAggregatedValueAndSegmentValue() { Long randomLong = randomLong(); aggregator.getInitialAggregatedValue(randomDouble); assertEquals( - randomDouble + starTreeNumericType.getDoubleValue(randomLong), + randomDouble + numericFieldConverter.toDoubleValue(randomLong), aggregator.mergeAggregatedValueAndSegmentValue(randomDouble, randomLong), 0.0 ); @@ -41,7 +41,7 @@ public void testMergeAggregatedValueAndSegmentValue_nullSegmentDocValue() { aggregator.getInitialAggregatedValue(randomDouble1); assertEquals(randomDouble1, aggregator.mergeAggregatedValueAndSegmentValue(randomDouble1, null), 0.0); assertEquals( - randomDouble1 + starTreeNumericType.getDoubleValue(randomLong), + randomDouble1 + numericFieldConverter.toDoubleValue(randomLong), aggregator.mergeAggregatedValueAndSegmentValue(randomDouble1, randomLong), 0.0 ); @@ -50,7 +50,11 @@ public void testMergeAggregatedValueAndSegmentValue_nullSegmentDocValue() { public void testMergeAggregatedValueAndSegmentValue_nullInitialDocValue() { Long randomLong = randomLong(); aggregator.getInitialAggregatedValue(null); - assertEquals(starTreeNumericType.getDoubleValue(randomLong), aggregator.mergeAggregatedValueAndSegmentValue(null, randomLong), 0.0); + assertEquals( + numericFieldConverter.toDoubleValue(randomLong), + aggregator.mergeAggregatedValueAndSegmentValue(null, randomLong), + 0.0 + ); } public void testMergeAggregatedValues() { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java index 6572d75d5b738..96ab1b3a52e8b 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java @@ -9,31 +9,31 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.test.OpenSearchTestCase; public class ValueAggregatorFactoryTests extends OpenSearchTestCase { public void testGetValueAggregatorForSumType() { - ValueAggregator aggregator = ValueAggregatorFactory.getValueAggregator(MetricStat.SUM, StarTreeNumericType.LONG); + ValueAggregator aggregator = ValueAggregatorFactory.getValueAggregator(MetricStat.SUM, NumberFieldMapper.NumberType.LONG); assertNotNull(aggregator); assertEquals(SumValueAggregator.class, aggregator.getClass()); } public void testGetValueAggregatorForMinType() { - ValueAggregator aggregator = ValueAggregatorFactory.getValueAggregator(MetricStat.MIN, StarTreeNumericType.LONG); + ValueAggregator aggregator = ValueAggregatorFactory.getValueAggregator(MetricStat.MIN, NumberFieldMapper.NumberType.LONG); assertNotNull(aggregator); assertEquals(MinValueAggregator.class, aggregator.getClass()); } public void testGetValueAggregatorForMaxType() { - ValueAggregator aggregator = ValueAggregatorFactory.getValueAggregator(MetricStat.MAX, StarTreeNumericType.LONG); + ValueAggregator aggregator = ValueAggregatorFactory.getValueAggregator(MetricStat.MAX, NumberFieldMapper.NumberType.LONG); assertNotNull(aggregator); assertEquals(MaxValueAggregator.class, aggregator.getClass()); } public void testGetValueAggregatorForCountType() { - ValueAggregator aggregator = ValueAggregatorFactory.getValueAggregator(MetricStat.VALUE_COUNT, StarTreeNumericType.LONG); + ValueAggregator aggregator = ValueAggregatorFactory.getValueAggregator(MetricStat.VALUE_COUNT, NumberFieldMapper.NumberType.LONG); assertNotNull(aggregator); assertEquals(CountValueAggregator.class, aggregator.getClass()); } @@ -41,7 +41,7 @@ public void testGetValueAggregatorForCountType() { public void testGetValueAggregatorForAvgType() { assertThrows( IllegalStateException.class, - () -> ValueAggregatorFactory.getValueAggregator(MetricStat.AVG, StarTreeNumericType.LONG) + () -> ValueAggregatorFactory.getValueAggregator(MetricStat.AVG, NumberFieldMapper.NumberType.LONG) ); } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java index 51ebc02ea8243..292264e2f9be7 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java @@ -32,7 +32,6 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; -import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.Mapper; @@ -203,8 +202,8 @@ Iterator mergeStarTrees(List starTreeValues) t public void test_generateMetricAggregatorInfos() throws IOException { List metricAggregatorInfos = builder.generateMetricAggregatorInfos(mapperService); List expectedMetricAggregatorInfos = List.of( - new MetricAggregatorInfo(MetricStat.SUM, "field2", starTreeField.getName(), IndexNumericFieldData.NumericType.DOUBLE), - new MetricAggregatorInfo(MetricStat.SUM, "field4", starTreeField.getName(), IndexNumericFieldData.NumericType.DOUBLE) + new MetricAggregatorInfo(MetricStat.SUM, "field2", starTreeField.getName(), NumberFieldMapper.NumberType.DOUBLE), + new MetricAggregatorInfo(MetricStat.SUM, "field4", starTreeField.getName(), NumberFieldMapper.NumberType.DOUBLE) ); assertEquals(metricAggregatorInfos, expectedMetricAggregatorInfos); }