Merge pull request #15 from coreweave/rwang.fixs3chan.10222023

fix minor file saving issue
coreweave · Oct 30, 2023 · e7f1dea · e7f1dea
2 parents eb30a00 + 263c230
commit e7f1dea
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/cmd/dataset_tokenizer/dataset_tokenizer.go b/cmd/dataset_tokenizer/dataset_tokenizer.go
@@ -1030,6 +1030,10 @@ func (tt TextsTokenizer) TokenizeTextsToContexts(
 func WriteContexts(outPath string, contexts chan gpt_bpe.Tokens,
 	encoder *gpt_bpe.GPTEncoder, sampling int, shuffle bool) (int, error) {
 	totalTokens := 0
+	// create file AND filepath if not exists
+	if err := os.MkdirAll(filepath.Dir(outPath), os.ModePerm); err != nil {
+		return 0, err
+	}
 	outFile, err := os.OpenFile(outPath, os.O_TRUNC|os.O_RDWR|os.O_CREATE,
 		0755)
 	if err != nil {