diff --git a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java index 8cfb710679137..0d69b762ab4f2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java @@ -12,7 +12,6 @@ import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.common.Rounding; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.unit.ByteSizeUnit; @@ -23,7 +22,6 @@ import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.compositeindex.CompositeIndexSettings; -import org.opensearch.index.compositeindex.datacube.DateDimension; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; @@ -62,7 +60,10 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool .startObject("config") .startArray("ordered_dimensions") .startObject() - .field("name", "timestamp") + .field("name", "numeric_dv_1") + .endObject() + .startObject() + .field("name", "numeric_dv_2") .endObject() .startObject() .field("name", getDim(invalidDim, keywordDim)) @@ -84,6 +85,14 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool .field("type", "integer") .field("doc_values", true) .endObject() + .startObject("numeric_dv_1") + .field("type", "integer") + .field("doc_values", true) + .endObject() + .startObject("numeric_dv_2") + .field("type", "integer") + .field("doc_values", true) + .endObject() .startObject("numeric") .field("type", "integer") .field("doc_values", false) @@ -112,11 +121,7 @@ private static XContentBuilder createMaxDimTestMapping() { .startObject("config") .startArray("ordered_dimensions") .startObject() - .field("name", "timestamp") - .startArray("calendar_intervals") - .value("day") - .value("month") - .endArray() + .field("name", "dim4") .endObject() .startObject() .field("name", "dim2") @@ -201,7 +206,7 @@ private static XContentBuilder createUpdateTestMapping(boolean changeDim, boolea .startObject("config") .startArray("ordered_dimensions") .startObject() - .field("name", "timestamp") + .field("name", "numeric_dv1") .endObject() .startObject() .field("name", changeDim ? "numeric_new" : getDim(false, false)) @@ -223,6 +228,10 @@ private static XContentBuilder createUpdateTestMapping(boolean changeDim, boolea .field("type", "integer") .field("doc_values", true) .endObject() + .startObject("numeric_dv1") + .field("type", "integer") + .field("doc_values", true) + .endObject() .startObject("numeric") .field("type", "integer") .field("doc_values", false) @@ -256,7 +265,7 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo .startObject("config") .startArray("ordered_dimensions") .startObject() - .field("name", "timestamp") + .field("name", "numeric_dv2") .endObject() .startObject() .field("name", "numeric_dv") @@ -284,6 +293,10 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo .field("type", "integer") .field("doc_values", true) .endObject() + .startObject("numeric_dv2") + .field("type", "integer") + .field("doc_values", true) + .endObject() .startObject("numeric_dv1") .field("type", "integer") .field("doc_values", true) @@ -328,15 +341,8 @@ public void testValidCompositeIndex() { for (CompositeMappedFieldType ft : fts) { assertTrue(ft instanceof StarTreeMapper.StarTreeFieldType); StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) ft; - assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField()); - assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); - DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); - List expectedTimeUnits = Arrays.asList( - Rounding.DateTimeUnit.MINUTES_OF_HOUR, - Rounding.DateTimeUnit.HOUR_OF_DAY - ); - assertEquals(expectedTimeUnits, dateDim.getIntervals()); - assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField()); + assertEquals("numeric_dv_1", starTreeFieldType.getDimensions().get(0).getField()); + assertEquals("numeric_dv_2", starTreeFieldType.getDimensions().get(1).getField()); assertEquals(2, starTreeFieldType.getMetrics().size()); assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField()); @@ -496,15 +502,8 @@ public void testUpdateIndexWhenMappingIsSame() { for (CompositeMappedFieldType ft : fts) { assertTrue(ft instanceof StarTreeMapper.StarTreeFieldType); StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) ft; - assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField()); - assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); - DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); - List expectedTimeUnits = Arrays.asList( - Rounding.DateTimeUnit.MINUTES_OF_HOUR, - Rounding.DateTimeUnit.HOUR_OF_DAY - ); - assertEquals(expectedTimeUnits, dateDim.getIntervals()); - assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField()); + assertEquals("numeric_dv_1", starTreeFieldType.getDimensions().get(0).getField()); + assertEquals("numeric_dv_2", starTreeFieldType.getDimensions().get(1).getField()); assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField()); // Assert default metrics @@ -570,24 +569,6 @@ public void testMaxMetricsCompositeIndex() { ); } - public void testMaxCalendarIntervalsCompositeIndex() { - MapperParsingException ex = expectThrows( - MapperParsingException.class, - () -> prepareCreate(TEST_INDEX).setMapping(createMaxDimTestMapping()) - .setSettings( - Settings.builder() - .put(StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING.getKey(), 1) - .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) - .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB)) - ) - .get() - ); - assertEquals( - "Failed to parse mapping [_doc]: At most [1] calendar intervals are allowed in dimension [timestamp]", - ex.getMessage() - ); - } - public void testUnsupportedDim() { MapperParsingException ex = expectThrows( MapperParsingException.class, diff --git a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java index 4698440f08036..bc50f507010c4 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java @@ -54,7 +54,6 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.LocaleUtils; -import org.opensearch.index.compositeindex.datacube.DimensionType; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexNumericFieldData.NumericType; import org.opensearch.index.fielddata.plain.SortedNumericIndexFieldData; @@ -77,7 +76,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Optional; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.LongSupplier; @@ -352,10 +350,6 @@ public DateFieldMapper build(BuilderContext context) { return new DateFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), nullTimestamp, resolution, this); } - @Override - public Optional getSupportedDataCubeDimensionType() { - return Optional.of(DimensionType.DATE); - } } public static final TypeParser MILLIS_PARSER = new TypeParser((n, c) -> { diff --git a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java index e06d9889ec905..daa9fda7a5a3b 100644 --- a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java @@ -74,14 +74,7 @@ public void testValidStarTree() throws IOException { for (CompositeMappedFieldType type : compositeFieldTypes) { StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type; assertEquals(2, starTreeFieldType.getDimensions().size()); - assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField()); - assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); - DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); - List expectedTimeUnits = Arrays.asList( - Rounding.DateTimeUnit.DAY_OF_MONTH, - Rounding.DateTimeUnit.MONTH_OF_YEAR - ); - assertEquals(expectedTimeUnits, dateDim.getIntervals()); + assertEquals("node", starTreeFieldType.getDimensions().get(0).getField()); assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); assertEquals(2, starTreeFieldType.getMetrics().size()); assertEquals("size", starTreeFieldType.getMetrics().get(0).getField()); @@ -92,7 +85,7 @@ public void testValidStarTree() throws IOException { assertEquals(100, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode()); assertEquals( - new HashSet<>(Arrays.asList("@timestamp", "status")), + new HashSet<>(Arrays.asList("node", "status")), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims() ); } @@ -137,14 +130,7 @@ public void testMetricsWithJustSum() throws IOException { Set compositeFieldTypes = mapperService.getCompositeFieldTypes(); for (CompositeMappedFieldType type : compositeFieldTypes) { StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type; - assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField()); - assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); - DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); - List expectedTimeUnits = Arrays.asList( - Rounding.DateTimeUnit.DAY_OF_MONTH, - Rounding.DateTimeUnit.MONTH_OF_YEAR - ); - assertEquals(expectedTimeUnits, dateDim.getIntervals()); + assertEquals("node", starTreeFieldType.getDimensions().get(0).getField()); assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); assertEquals("size", starTreeFieldType.getMetrics().get(0).getField()); @@ -154,7 +140,7 @@ public void testMetricsWithJustSum() throws IOException { assertEquals(100, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode()); assertEquals( - new HashSet<>(Arrays.asList("@timestamp", "status")), + new HashSet<>(Arrays.asList("node", "status")), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims() ); } @@ -165,14 +151,7 @@ public void testMetricsWithCountAndSum() throws IOException { Set compositeFieldTypes = mapperService.getCompositeFieldTypes(); for (CompositeMappedFieldType type : compositeFieldTypes) { StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type; - assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField()); - assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); - DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); - List expectedTimeUnits = Arrays.asList( - Rounding.DateTimeUnit.DAY_OF_MONTH, - Rounding.DateTimeUnit.MONTH_OF_YEAR - ); - assertEquals(expectedTimeUnits, dateDim.getIntervals()); + assertEquals("node", starTreeFieldType.getDimensions().get(0).getField()); assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); assertEquals("size", starTreeFieldType.getMetrics().get(0).getField()); @@ -187,7 +166,7 @@ public void testMetricsWithCountAndSum() throws IOException { assertEquals(100, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode()); assertEquals( - new HashSet<>(Arrays.asList("@timestamp", "status")), + new HashSet<>(Arrays.asList("node", "status")), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims() ); } @@ -198,14 +177,7 @@ public void testValidStarTreeDefaults() throws IOException { Set compositeFieldTypes = mapperService.getCompositeFieldTypes(); for (CompositeMappedFieldType type : compositeFieldTypes) { StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type; - assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField()); - assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); - DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); - List expectedTimeUnits = Arrays.asList( - Rounding.DateTimeUnit.MINUTES_OF_HOUR, - Rounding.DateTimeUnit.HOUR_OF_DAY - ); - assertEquals(expectedTimeUnits, dateDim.getIntervals()); + assertEquals("node", starTreeFieldType.getDimensions().get(0).getField()); assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); assertEquals(3, starTreeFieldType.getMetrics().size()); assertEquals("status", starTreeFieldType.getMetrics().get(0).getField()); @@ -278,7 +250,7 @@ public void testMissingDims() { MapperParsingException.class, () -> createMapperService(getMinMapping(false, false, true, false)) ); - assertThat(ex.getMessage(), containsString("Failed to parse mapping [_doc]: unknown dimension field [@timestamp]")); + assertThat(ex.getMessage(), containsString("Failed to parse mapping [_doc]: unknown dimension field [node]")); } public void testMissingMetrics() { @@ -314,7 +286,7 @@ public void testInvalidDimType() { () -> createMapperService(getInvalidMapping(false, false, true, false)) ); assertEquals( - "Failed to parse mapping [_doc]: unsupported field type associated with dimension [@timestamp] as part of star tree field [startree]", + "Failed to parse mapping [_doc]: unsupported field type associated with dimension [node] as part of star tree field [startree]", ex.getMessage() ); } @@ -511,7 +483,7 @@ public void testValidations() throws IOException { ) ); assertEquals( - "Aggregations not supported for the dimension field [@timestamp] with field type [date] as part of star tree field", + "Aggregations not supported for the dimension field [node] with field type [integer] as part of star tree field", ex.getMessage() ); @@ -534,17 +506,13 @@ private XContentBuilder getExpandedMappingWithJustAvg(String dim, String metric) b.field("max_leaf_docs", 100); b.startArray("skip_star_node_creation_for_dimensions"); { - b.value("@timestamp"); + b.value("node"); b.value("status"); } b.endArray(); b.startArray("ordered_dimensions"); b.startObject(); - b.field("name", "@timestamp"); - b.startArray("calendar_intervals"); - b.value("day"); - b.value("month"); - b.endArray(); + b.field("name", "node"); b.endObject(); b.startObject(); b.field("name", dim); @@ -562,8 +530,8 @@ private XContentBuilder getExpandedMappingWithJustAvg(String dim, String metric) b.endObject(); b.endObject(); b.startObject("properties"); - b.startObject("@timestamp"); - b.field("type", "date"); + b.startObject("node"); + b.field("type", "integer"); b.endObject(); b.startObject("status"); b.field("type", "integer"); @@ -585,7 +553,7 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo .startObject("config") .startArray("ordered_dimensions") .startObject() - .field("name", "timestamp") + .field("name", "node") .endObject() .startObject() .field("name", "numeric_dv") @@ -606,8 +574,8 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo .endObject() .endObject() .startObject("properties") - .startObject("timestamp") - .field("type", "date") + .startObject("node") + .field("type", "integer") .endObject() .startObject("numeric_dv") .field("type", "integer") @@ -634,17 +602,13 @@ private XContentBuilder getExpandedMappingWithJustSum(String dim, String metric) b.field("max_leaf_docs", 100); b.startArray("skip_star_node_creation_for_dimensions"); { - b.value("@timestamp"); + b.value("node"); b.value("status"); } b.endArray(); b.startArray("ordered_dimensions"); b.startObject(); - b.field("name", "@timestamp"); - b.startArray("calendar_intervals"); - b.value("day"); - b.value("month"); - b.endArray(); + b.field("name", "node"); b.endObject(); b.startObject(); b.field("name", dim); @@ -662,8 +626,8 @@ private XContentBuilder getExpandedMappingWithJustSum(String dim, String metric) b.endObject(); b.endObject(); b.startObject("properties"); - b.startObject("@timestamp"); - b.field("type", "date"); + b.startObject("node"); + b.field("type", "integer"); b.endObject(); b.startObject("status"); b.field("type", "integer"); @@ -684,17 +648,13 @@ private XContentBuilder getExpandedMappingWithSumAndCount(String dim, String met b.field("max_leaf_docs", 100); b.startArray("skip_star_node_creation_for_dimensions"); { - b.value("@timestamp"); + b.value("node"); b.value("status"); } b.endArray(); b.startArray("ordered_dimensions"); b.startObject(); - b.field("name", "@timestamp"); - b.startArray("calendar_intervals"); - b.value("day"); - b.value("month"); - b.endArray(); + b.field("name", "node"); b.endObject(); b.startObject(); b.field("name", dim); @@ -713,8 +673,8 @@ private XContentBuilder getExpandedMappingWithSumAndCount(String dim, String met b.endObject(); b.endObject(); b.startObject("properties"); - b.startObject("@timestamp"); - b.field("type", "date"); + b.startObject("node"); + b.field("type", "integer"); b.endObject(); b.startObject("status"); b.field("type", "integer"); @@ -740,7 +700,7 @@ private XContentBuilder getMinMapping(boolean isEmptyDims, boolean isEmptyMetric if (!isEmptyDims) { b.startArray("ordered_dimensions"); b.startObject(); - b.field("name", "@timestamp"); + b.field("name", "node"); b.endObject(); b.startObject(); b.field("name", "status"); @@ -762,8 +722,8 @@ private XContentBuilder getMinMapping(boolean isEmptyDims, boolean isEmptyMetric b.endObject(); b.startObject("properties"); if (!missingDim) { - b.startObject("@timestamp"); - b.field("type", "date"); + b.startObject("node"); + b.field("type", "integer"); b.endObject(); } b.startObject("status"); @@ -876,7 +836,7 @@ private XContentBuilder getInvalidMapping( b.startArray("ordered_dimensions"); if (!singleDim) { b.startObject(); - b.field("name", "@timestamp"); + b.field("name", "node"); b.endObject(); } b.startObject(); @@ -901,9 +861,9 @@ private XContentBuilder getInvalidMapping( b.endObject(); b.endObject(); b.startObject("properties"); - b.startObject("@timestamp"); + b.startObject("node"); if (!invalidDimType) { - b.field("type", "date"); + b.field("type", "integer"); } else { b.field("type", "keyword"); } @@ -946,7 +906,7 @@ private XContentBuilder getInvalidMappingWithDv( b.startArray("ordered_dimensions"); if (!singleDim) { b.startObject(); - b.field("name", "@timestamp"); + b.field("name", "node"); b.endObject(); } b.startObject(); @@ -965,12 +925,12 @@ private XContentBuilder getInvalidMappingWithDv( b.endObject(); b.endObject(); b.startObject("properties"); - b.startObject("@timestamp"); + b.startObject("node"); if (!invalidDimType) { - b.field("type", "date"); + b.field("type", "integer"); b.field("doc_values", "true"); } else { - b.field("type", "date"); + b.field("type", "integer"); b.field("doc_values", "false"); } b.endObject();