removing data header

Signed-off-by: Sarthak Aggarwal <[email protected]>

server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/StarTreeWriter.java

@@ -37,7 +37,7 @@ public StarTreeWriter() {}
      *
      * @param dataOut  data index output
      * @param rootNode root star-tree node
-     * @param numNodes number of nodes in the tree
+     * @param numNodes number of nodes in the star tree
      * @param name     name of the star-tree field
      * @return total size of the three
      * @throws IOException when star-tree data serialization fails
@@ -52,6 +52,7 @@ public long writeStarTree(IndexOutput dataOut, InMemoryTreeNode rootNode, int nu
      * @param metaOut                meta index output
      * @param starTreeField          star tree field
      * @param metricAggregatorInfos  metric aggregator infos
+     * @param numNodes               number of nodes in the star tree
      * @param segmentAggregatedCount segment aggregated count
      * @param dataFilePointer        data file pointer
      * @param dataFileLength         data file length
@@ -61,6 +62,7 @@ public void writeStarTreeMetadata(
         IndexOutput metaOut,
         StarTreeField starTreeField,
         List<MetricAggregatorInfo> metricAggregatorInfos,
+        Integer numNodes,
         Integer segmentAggregatedCount,
         long dataFilePointer,
         long dataFileLength
@@ -69,6 +71,7 @@ public void writeStarTreeMetadata(
             metaOut,
             starTreeField,
             metricAggregatorInfos,
+            numNodes,
             segmentAggregatedCount,
             dataFilePointer,
             dataFileLength

server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeDataWriter.java

@@ -20,8 +20,6 @@
 import java.util.List;
 import java.util.Queue;
 
-import static org.opensearch.index.compositeindex.CompositeIndexConstants.COMPOSITE_FIELD_MARKER;
-import static org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter.VERSION_CURRENT;
 import static org.opensearch.index.compositeindex.datacube.startree.node.FixedLengthStarTreeNode.SERIALIZABLE_DATA_SIZE_IN_BYTES;
 import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.ALL;
 
@@ -45,44 +43,14 @@ public class StarTreeDataWriter {
      * @throws IOException if an I/O error occurs while writing the star-tree data
      */
     public static long writeStarTree(IndexOutput indexOutput, InMemoryTreeNode rootNode, int numNodes, String name) throws IOException {
-        long totalSizeInBytes = 0L;
-        totalSizeInBytes += computeStarTreeDataHeaderByteSize();
-        totalSizeInBytes += (long) numNodes * SERIALIZABLE_DATA_SIZE_IN_BYTES;
+        long totalSizeInBytes = (long) numNodes * SERIALIZABLE_DATA_SIZE_IN_BYTES;
 
         logger.debug("Star tree data size in bytes : {} for star-tree field {}", totalSizeInBytes, name);
 
-        writeStarTreeHeader(indexOutput, numNodes);
         writeStarTreeNodes(indexOutput, rootNode);
         return totalSizeInBytes;
     }
 
-    /**
-     * Computes the byte size of the star-tree data header.
-     *
-     * @return the byte size of the star-tree data header
-     */
-    public static int computeStarTreeDataHeaderByteSize() {
-        // Magic marker (8), version (4)
-        int headerSizeInBytes = 12;
-
-        // For number of nodes.
-        headerSizeInBytes += Integer.BYTES;
-        return headerSizeInBytes;
-    }
-
-    /**
-     * Writes the star-tree data header.
-     *
-     * @param output   the IndexOutput to write the header
-     * @param numNodes the total number of nodes in the star-tree
-     * @throws IOException if an I/O error occurs while writing the header
-     */
-    private static void writeStarTreeHeader(IndexOutput output, int numNodes) throws IOException {
-        output.writeLong(COMPOSITE_FIELD_MARKER);
-        output.writeInt(VERSION_CURRENT);
-        output.writeInt(numNodes);
-    }
-
     /**
      * Writes the star-tree nodes in a breadth-first order.
      *

server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetaWriter.java

@@ -40,6 +40,7 @@ public class StarTreeMetaWriter {
      * @param starTreeField          the star-tree field
      * @param metricAggregatorInfos  the list of metric aggregator information
      * @param segmentAggregatedCount the aggregated document count for the segment
+     * @param numNodes               number of nodes in the star tree
      * @param dataFilePointer        the file pointer to the start of the star tree data
      * @param dataFileLength         the length of the star tree data file
      * @throws IOException if an I/O error occurs while serializing the metadata
@@ -48,6 +49,7 @@ public static void writeStarTreeMetadata(
         IndexOutput metaOut,
         StarTreeField starTreeField,
         List<MetricAggregatorInfo> metricAggregatorInfos,
+        Integer numNodes,
         Integer segmentAggregatedCount,
         long dataFilePointer,
         long dataFileLength
@@ -56,7 +58,7 @@ public static void writeStarTreeMetadata(
         long initialMetaFilePointer = metaOut.getFilePointer();
 
         writeMetaHeader(metaOut, CompositeMappedFieldType.CompositeFieldType.STAR_TREE, starTreeField.getName());
-        writeMeta(metaOut, metricAggregatorInfos, starTreeField, segmentAggregatedCount, dataFilePointer, dataFileLength);
+        writeMeta(metaOut, metricAggregatorInfos, starTreeField, numNodes, segmentAggregatedCount, dataFilePointer, dataFileLength);
 
         logger.debug(
             "Star tree meta size in bytes : {} for star-tree field {}",
@@ -97,6 +99,7 @@ private static void writeMetaHeader(
      * @param metaOut                   the IndexOutput to write the metadata
      * @param metricAggregatorInfos     the list of metric aggregator information
      * @param starTreeField             the star tree field
+     * @param numNodes                  number of nodes in the star tree
      * @param segmentAggregatedDocCount the aggregated document count for the segment
      * @param dataFilePointer           the file pointer to the start of the star-tree data
      * @param dataFileLength            the length of the star-tree data file
@@ -106,11 +109,15 @@ private static void writeMeta(
         IndexOutput metaOut,
         List<MetricAggregatorInfo> metricAggregatorInfos,
         StarTreeField starTreeField,
+        int numNodes,
         Integer segmentAggregatedDocCount,
         long dataFilePointer,
         long dataFileLength
     ) throws IOException {
 
+        // number of nodes
+        metaOut.writeInt(numNodes);
+
         // number of dimensions
         metaOut.writeVInt(starTreeField.getDimensionsOrder().size());
 

server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java

@@ -35,6 +35,16 @@ public class StarTreeMetadata extends CompositeIndexMetadata {
      */
     private final IndexInput meta;
 
+    /**
+     * The version of the star tree stored in the segments.
+     */
+    private final int version;
+
+    /**
+     * The number of the nodes in the respective star tree
+     */
+    private final int numberOfNodes;
+
     /**
      * The name of the star-tree field, used to identify the star-tree.
      */
@@ -92,15 +102,22 @@ public class StarTreeMetadata extends CompositeIndexMetadata {
      * @param metaIn             an index input to read star-tree meta
      * @param compositeFieldName name of the composite field. Here, name of the star-tree field.
      * @param compositeFieldType type of the composite field. Here, STAR_TREE field.
+     * @param version The version of the star tree stored in the segments.
      * @throws IOException if unable to read star-tree metadata from the file
      */
-    public StarTreeMetadata(IndexInput metaIn, String compositeFieldName, CompositeMappedFieldType.CompositeFieldType compositeFieldType)
-        throws IOException {
+    public StarTreeMetadata(
+        IndexInput metaIn,
+        String compositeFieldName,
+        CompositeMappedFieldType.CompositeFieldType compositeFieldType,
+        Integer version
+    ) throws IOException {
         super(compositeFieldName, compositeFieldType);
         this.meta = metaIn;
         try {
             this.starTreeFieldName = this.getCompositeFieldName();
             this.starTreeFieldType = this.getCompositeFieldType().getName();
+            this.version = version;
+            this.numberOfNodes = readNumberOfNodes();
             this.dimensionFields = readStarTreeDimensions();
             this.metricEntries = readMetricEntries();
             this.segmentAggregatedDocCount = readSegmentAggregatedDocCount();
@@ -122,6 +139,7 @@ public StarTreeMetadata(IndexInput metaIn, String compositeFieldName, CompositeM
      * @param meta                       an index input to read star-tree meta
      * @param compositeFieldName         name of the composite field. Here, name of the star-tree field.
      * @param compositeFieldType         type of the composite field. Here, STAR_TREE field.
+     * @param version The version of the star tree stored in the segments.
      * @param dimensionFields            list of dimension fields
      * @param metricEntries              list of metric entries
      * @param segmentAggregatedDocCount  segment aggregated doc count
@@ -135,6 +153,8 @@ public StarTreeMetadata(
         String compositeFieldName,
         CompositeMappedFieldType.CompositeFieldType compositeFieldType,
         IndexInput meta,
+        Integer version,
+        Integer numberOfNodes,
         List<String> dimensionFields,
         List<MetricEntry> metricEntries,
         Integer segmentAggregatedDocCount,
@@ -148,6 +168,8 @@ public StarTreeMetadata(
         this.meta = meta;
         this.starTreeFieldName = compositeFieldName;
         this.starTreeFieldType = compositeFieldType.getName();
+        this.version = version;
+        this.numberOfNodes = numberOfNodes;
         this.dimensionFields = dimensionFields;
         this.metricEntries = metricEntries;
         this.segmentAggregatedDocCount = segmentAggregatedDocCount;
@@ -158,6 +180,10 @@ public StarTreeMetadata(
         this.dataLength = dataLength;
     }
 
+    private int readNumberOfNodes() throws IOException {
+        return meta.readInt();
+    }
+
     private int readDimensionsCount() throws IOException {
         return meta.readVInt();
     }
@@ -313,4 +339,20 @@ public long getDataStartFilePointer() {
     public long getDataLength() {
         return dataLength;
     }
+
+    /**
+     * Returns the version with which the star tree is stored in the segments
+     * @return star-tree version
+     */
+    public int getVersion() {
+        return version;
+    }
+
+    /**
+     * Returns the number of nodes in the star tree
+     * @return number of nodes in the star tree
+     */
+    public int getNumberOfNodes() {
+        return numberOfNodes;
+    }
 }

server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTree.java

@@ -7,59 +7,27 @@
  */
 package org.opensearch.index.compositeindex.datacube.startree.node;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.RandomAccessInput;
-import org.opensearch.index.compositeindex.datacube.startree.fileformats.data.StarTreeDataWriter;
 import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata;
 
 import java.io.IOException;
 
-import static org.opensearch.index.compositeindex.CompositeIndexConstants.COMPOSITE_FIELD_MARKER;
-import static org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter.VERSION_CURRENT;
-
 /**
  * Off heap implementation of the star-tree.
  *
  * @opensearch.experimental
  */
 public class StarTree {
-    private static final Logger logger = LogManager.getLogger(StarTree.class);
     private final FixedLengthStarTreeNode root;
-    private final Integer numNodes;
 
     public StarTree(IndexInput data, StarTreeMetadata starTreeMetadata) throws IOException {
-        long magicMarker = data.readLong();
-        if (COMPOSITE_FIELD_MARKER != magicMarker) {
-            logger.error("Invalid magic marker");
-            throw new IOException("Invalid magic marker");
-        }
-        int version = data.readInt();
-        if (VERSION_CURRENT != version) {
-            logger.error("Invalid star tree version");
-            throw new IOException("Invalid version");
-        }
-        numNodes = data.readInt(); // num nodes
-
-        RandomAccessInput in = data.randomAccessSlice(
-            StarTreeDataWriter.computeStarTreeDataHeaderByteSize(),
-            starTreeMetadata.getDataLength() - StarTreeDataWriter.computeStarTreeDataHeaderByteSize()
-        );
+        RandomAccessInput in = data.randomAccessSlice(0, starTreeMetadata.getDataLength());
         root = new FixedLengthStarTreeNode(in, 0);
     }
 
     public StarTreeNode getRoot() {
         return root;
     }
 
-    /**
-     * Returns the number of nodes in star-tree
-     *
-     * @return number of nodes in te star-tree
-     */
-    public Integer getNumNodes() {
-        return numNodes;
-    }
-
 }

server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeFileFormatsTests.java

@@ -56,7 +56,7 @@ public void test_StarTreeNode() throws IOException {
         long starTreeDataLength = starTreeWriter.writeStarTree(dataOut, root, inMemoryTreeNodeMap.size(), "star-tree");
 
         // asserting on the actual length of the star tree data file
-        assertEquals(starTreeDataLength, (inMemoryTreeNodeMap.size() * 33L) + 16);
+        assertEquals(starTreeDataLength, (inMemoryTreeNodeMap.size() * 33L));
         dataOut.close();
 
         dataIn = directory.openInput("star-tree-data", IOContext.READONCE);
@@ -108,7 +108,7 @@ public void test_starTreeSearch() throws IOException {
         long starTreeDataLength = starTreeWriter.writeStarTree(dataOut, root, inMemoryTreeNodeMap.size(), "star-tree");
 
         // asserting on the actual length of the star tree data file
-        assertEquals(starTreeDataLength, (inMemoryTreeNodeMap.size() * 33L) + 16);
+        assertEquals(starTreeDataLength, (inMemoryTreeNodeMap.size() * 33L));
         dataOut.close();
 
         dataIn = directory.openInput("star-tree-data", IOContext.READONCE);

server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetaTests.java

@@ -147,10 +147,12 @@ public void test_starTreeMetadata() throws IOException {
         segmentDocumentCount = randomNonNegativeInt();
         metaOut = directory.createOutput("star-tree-metadata", IOContext.DEFAULT);
         StarTreeWriter starTreeWriter = new StarTreeWriter();
+        int numberOfNodes = randomNonNegativeInt();
         starTreeWriter.writeStarTreeMetadata(
             metaOut,
             starTreeField,
             metricAggregatorInfos,
+            numberOfNodes,
             segmentDocumentCount,
             dataFilePointer,
             dataFileLength
@@ -167,12 +169,13 @@ public void test_starTreeMetadata() throws IOException {
             metaIn.readString()
         );
 
-        StarTreeMetadata starTreeMetadata = new StarTreeMetadata(metaIn, compositeFieldName, compositeFieldType);
+        StarTreeMetadata starTreeMetadata = new StarTreeMetadata(metaIn, compositeFieldName, compositeFieldType, VERSION_CURRENT);
         assertEquals(starTreeField.getName(), starTreeMetadata.getStarTreeFieldName());
         assertEquals(starTreeField.getName(), starTreeMetadata.getCompositeFieldName());
         assertEquals(STAR_TREE, starTreeMetadata.getCompositeFieldType());
         assertEquals(STAR_TREE.getName(), starTreeMetadata.getStarTreeFieldType());
-
+        assertEquals(starTreeMetadata.getVersion(), VERSION_CURRENT);
+        assertEquals(starTreeMetadata.getNumberOfNodes(), numberOfNodes);
         assertNotNull(starTreeMetadata);
 
         for (int i = 0; i < dimensionsOrder.size(); i++) {

server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/node/FixedLengthStarTreeNodeTests.java

@@ -77,7 +77,7 @@ public void setup() throws IOException {
         long starTreeDataLength = starTreeWriter.writeStarTree(dataOut, node, 1 + node.children.size(), "star-tree");
 
         // asserting on the actual length of the star tree data file
-        assertEquals(starTreeDataLength, 33L * node.children.size() + 33 + 16);
+        assertEquals(starTreeDataLength, 33L * node.children.size() + 33);
         dataOut.close();
 
         dataIn = directory.openInput("star-tree-data", IOContext.READONCE);