[index] Add Batch method for inserting multiple documents at a time

doc/document.go

@@ -29,9 +29,8 @@ import (
 )
 
 var (
-	errMultipleIDs   = errors.New("document cannot contain multiple IDs")
-	errZeroLengthID  = errors.New("document ID cannot be of length zero")
-	errEmptyDocument = errors.New("document cannot be empty")
+	errReservedFieldName = fmt.Errorf("'%s' is a reserved field name", IDReservedFieldName)
+	errEmptyDocument     = errors.New("document cannot be empty")
 )
 
 // IDReservedFieldName is the field name reserved for IDs.
@@ -89,7 +88,7 @@ func (f Fields) shallowCopy() Fields {
 
 // Document represents a document to be indexed.
 type Document struct {
-	// Fields contains the list of fields by which to index the document.
+	ID     []byte
 	Fields []Field
 }
 
@@ -105,6 +104,10 @@ func (d Document) Get(fieldName []byte) ([]byte, bool) {
 
 // Equal returns whether this document is equal to another.
 func (d Document) Equal(other Document) bool {
+	if !bytes.Equal(d.ID, other.ID) {
+		return false
+	}
+
 	if len(d.Fields) != len(other.Fields) {
 		return false
 	}
@@ -133,35 +136,31 @@ func (d Document) Equal(other Document) bool {
 	return true
 }
 
-// Validate validates the given document and returns its ID if it has one.
-func (d Document) Validate() ([]byte, error) {
+// Validate returns a bool indicating whether the document is valid.
+func (d Document) Validate() error {
 	if len(d.Fields) == 0 {
-		return nil, errEmptyDocument
+		return errEmptyDocument
 	}
 
-	var id []byte
 	for _, f := range d.Fields {
 		// TODO: Should we enforce uniqueness of field names?
 		if !utf8.Valid(f.Name) {
-			return nil, fmt.Errorf("document contains invalid field name: %v", f.Name)
+			return fmt.Errorf("document contains invalid field name: %v", f.Name)
 		}
 
 		if bytes.Equal(f.Name, IDReservedFieldName) {
-			if id != nil {
-				return nil, errMultipleIDs
-			}
-
-			if len(f.Value) == 0 {
-				return nil, errZeroLengthID
-			}
-
-			id = f.Value
+			return errReservedFieldName
 		}
 
 		if !utf8.Valid(f.Value) {
-			return nil, fmt.Errorf("document contains invalid field value: %v", f.Value)
+			return fmt.Errorf("document contains invalid field value: %v", f.Value)
 		}
 	}
 
-	return id, nil
+	return nil
+}
+
+// HasID returns a bool indicating whether the document has an ID or not.
+func (d Document) HasID() bool {
+	return len(d.ID) > 0
 }

doc/document_test.go

@@ -159,6 +159,7 @@ func TestDocumentEquality(t *testing.T) {
 		{
 			name: "documents with the same fields in the same order are equal",
 			l: Document{
+				ID: []byte("831992"),
 				Fields: []Field{
 					Field{
 						Name:  []byte("apple"),
@@ -171,6 +172,7 @@ func TestDocumentEquality(t *testing.T) {
 				},
 			},
 			r: Document{
+				ID: []byte("831992"),
 				Fields: []Field{
 					Field{
 						Name:  []byte("apple"),
@@ -187,6 +189,7 @@ func TestDocumentEquality(t *testing.T) {
 		{
 			name: "documents with the same fields in different order are equal",
 			l: Document{
+				ID: []byte("831992"),
 				Fields: []Field{
 					Field{
 						Name:  []byte("banana"),
@@ -199,6 +202,7 @@ func TestDocumentEquality(t *testing.T) {
 				},
 			},
 			r: Document{
+				ID: []byte("831992"),
 				Fields: []Field{
 					Field{
 						Name:  []byte("apple"),
@@ -215,6 +219,7 @@ func TestDocumentEquality(t *testing.T) {
 		{
 			name: "documents with different fields are unequal",
 			l: Document{
+				ID: []byte("831992"),
 				Fields: []Field{
 					Field{
 						Name:  []byte("apple"),
@@ -227,6 +232,7 @@ func TestDocumentEquality(t *testing.T) {
 				},
 			},
 			r: Document{
+				ID: []byte("831992"),
 				Fields: []Field{
 					Field{
 						Name:  []byte("apple"),
@@ -240,6 +246,28 @@ func TestDocumentEquality(t *testing.T) {
 			},
 			expected: false,
 		},
+		{
+			name: "documents with different IDs are unequal",
+			l: Document{
+				ID: []byte("831992"),
+				Fields: []Field{
+					Field{
+						Name:  []byte("apple"),
+						Value: []byte("red"),
+					},
+				},
+			},
+			r: Document{
+				ID: []byte("080292"),
+				Fields: []Field{
+					Field{
+						Name:  []byte("apple"),
+						Value: []byte("red"),
+					},
+				},
+			},
+			expected: false,
+		},
 	}
 
 	for _, test := range tests {
@@ -254,7 +282,6 @@ func TestDocumentValidation(t *testing.T) {
 		name        string
 		input       Document
 		expectedErr bool
-		expectedID  []byte
 	}{
 		{
 			name:        "empty document",
@@ -286,46 +313,79 @@ func TestDocumentValidation(t *testing.T) {
 			expectedErr: true,
 		},
 		{
-			name: "valid document with ID",
+			name: "document contains field with reserved field name",
 			input: Document{
 				Fields: []Field{
 					Field{
 						Name:  []byte("apple"),
 						Value: []byte("red"),
 					},
+					Field{
+						Name:  IDReservedFieldName,
+						Value: []byte("123"),
+					},
 				},
 			},
-			expectedErr: false,
-			expectedID:  nil,
+			expectedErr: true,
 		},
 		{
-			name: "valid document with ID",
+			name: "valid document",
 			input: Document{
 				Fields: []Field{
 					Field{
 						Name:  []byte("apple"),
 						Value: []byte("red"),
 					},
-					Field{
-						Name:  IDReservedFieldName,
-						Value: []byte("123"),
-					},
 				},
 			},
 			expectedErr: false,
-			expectedID:  []byte("123"),
 		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			id, err := test.input.Validate()
+			err := test.input.Validate()
 			if test.expectedErr {
 				require.Error(t, err)
 				return
 			}
 			require.NoError(t, err)
-			require.Equal(t, test.expectedID, id)
+		})
+	}
+}
+
+func TestDocumentHasID(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    Document
+		expected bool
+	}{
+		{
+			name: "nil ID",
+			input: Document{
+				ID: nil,
+			},
+			expected: false,
+		},
+		{
+			name: "zero-length ID",
+			input: Document{
+				ID: make([]byte, 0, 16),
+			},
+			expected: false,
+		},
+		{
+			name: "valid ID",
+			input: Document{
+				ID: []byte("831992"),
+			},
+			expected: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			require.Equal(t, test.expected, test.input.HasID())
 		})
 	}
 }

generated/generics/generate.sh

@@ -1,22 +1,22 @@
 #!/bin/bash
 
 if [ -z $GOPATH ]; then
-  echo 'GOPATH is not set'
-  exit 1
+    echo 'GOPATH is not set'
+    exit 1
 fi
 
 GENERIC_MAP_PATH=${GOPATH}/src/github.com/m3db/m3ninx/vendor/github.com/m3db/m3x/generics/hashmap
 GENERIC_MAP_IMPL=${GENERIC_MAP_PATH}/map.go
 
 if [ ! -f "$GENERIC_MAP_IMPL" ]; then
-  echo "${GENERIC_MAP_IMPL} does not exist"
-  exit 1
+    echo "${GENERIC_MAP_IMPL} does not exist"
+    exit 1
 fi
 
 GENERATED_PATH=${GOPATH}/src/github.com/m3db/m3ninx/index/segment/mem
 if [ ! -d "$GENERATED_PATH" ]; then
-  echo "${GENERATED_PATH} does not exist"
-  exit 1
+    echo "${GENERATED_PATH} does not exist"
+    exit 1
 fi
 
 # NB: We use (genny)[1] to combat the lack of generics in Go. It allows us
@@ -32,11 +32,16 @@ fi
 # [1]: https://github.com/cheekybits/genny
 
 mkdir -p $GENERATED_PATH/postingsgen
-cat $GENERIC_MAP_IMPL                                                    \
-  | genny -out=${GENERATED_PATH}/postingsgen/generated_map.go            \
-    -pkg=postingsgen gen "KeyType=[]byte ValueType=postings.MutableList"
+cat $GENERIC_MAP_IMPL                                                  \
+| genny -out=${GENERATED_PATH}/postingsgen/generated_map.go            \
+-pkg=postingsgen gen "KeyType=[]byte ValueType=postings.MutableList"
 
 mkdir -p $GENERATED_PATH/fieldsgen
-cat $GENERIC_MAP_IMPL                                                        \
-  | genny -out=${GENERATED_PATH}/fieldsgen/generated_map.go                  \
-    -pkg=fieldsgen gen "KeyType=[]byte ValueType=*postingsgen.ConcurrentMap"
+cat $GENERIC_MAP_IMPL                                                      \
+| genny -out=${GENERATED_PATH}/fieldsgen/generated_map.go                  \
+-pkg=fieldsgen gen "KeyType=[]byte ValueType=*postingsgen.ConcurrentMap"
+
+mkdir -p $GENERATED_PATH/idsgen
+cat $GENERIC_MAP_IMPL                                   \
+| genny -out=${GENERATED_PATH}/idsgen/generated_map.go  \
+-pkg=idsgen gen "KeyType=[]byte ValueType=struct{}"

index/encoding/doc/README.md

@@ -67,12 +67,23 @@ The payload comes after the header and it contains the actual documents.
 
 #### Document
 
-Each document is composed of fields. The number of fields in the document is encoded
-first as a variable-sized unsigned integer and then the fields themselves are encoded.
+Each document is composed of an ID and fields. The ID is a sequence of valid UTF-8 bytes
+and it is encoded first by encoding the length of the ID, in bytes, as a variable-sized
+unsigned integer and then encoding the actual bytes which comprise the ID. Following
+the ID are the fields. The number of fields in the document is encoded first as a
+variable-sized unsigned integer and then the fields themselves are encoded.
 
 ```
 ┌───────────────────────────┐
 │ ┌───────────────────────┐ │
+│ │     Length of ID      │ │
+│ │       (uvarint)       │ │
+│ ├───────────────────────┤ │
+│ │                       │ │
+│ │          ID           │ │
+│ │        (bytes)        │ │
+│ │                       │ │
+│ ├───────────────────────┤ │
 │ │   Number of Fields    │ │
 │ │       (uvarint)       │ │
 │ ├───────────────────────┤ │
@@ -94,8 +105,9 @@ first as a variable-sized unsigned integer and then the fields themselves are en
 #### Field
 
 Each field is composed of a name and a value. The name and value are a sequence of valid
-UTF-8 bytes and they are encoded by prefixing the sequence of bytes with its length as
-a variable-sized unsigned integer. The name is encoded first and the value second.
+UTF-8 bytes and they are encoded by encoding the length of the name (value), in bytes, as
+a variable-sized unsigned integer and then encoding the actual bytes which comprise the
+name (value). The name is encoded first and the value second.
 
 ```
 ┌───────────────────────────┐

index/encoding/doc/doc.go

@@ -80,8 +80,9 @@ func (w *writer) Open() error {
 }
 
 func (w *writer) Write(d doc.Document) error {
-	w.enc.PutUvarint(uint64(len(d.Fields)))
+	w.enc.PutBytes(d.ID)
 
+	w.enc.PutUvarint(uint64(len(d.Fields)))
 	for _, f := range d.Fields {
 		w.enc.PutBytes(f.Name)
 		w.enc.PutBytes(f.Value)
@@ -228,13 +229,19 @@ func (r *reader) Read() (doc.Document, error) {
 		return doc.Document{}, io.EOF
 	}
 
+	id, err := r.dec.Bytes()
+	if err != nil {
+		return doc.Document{}, err
+	}
+
 	x, err := r.dec.Uvarint()
 	if err != nil {
 		return doc.Document{}, err
 	}
 	n := int(x)
 
 	d := doc.Document{
+		ID:     id,
 		Fields: make([]doc.Field, n),
 	}
 

index/encoding/doc/doc_test.go

@@ -48,6 +48,7 @@ func TestWriteReadDocuments(t *testing.T) {
 			name: "standard documents",
 			docs: []doc.Document{
 				doc.Document{
+					ID: []byte("831992"),
 					Fields: []doc.Field{
 						doc.Field{
 							Name:  []byte("fruit"),
@@ -60,6 +61,7 @@ func TestWriteReadDocuments(t *testing.T) {
 					},
 				},
 				doc.Document{
+					ID: []byte("080392"),
 					Fields: []doc.Field{
 						doc.Field{
 							Name:  []byte("fruit"),