Skip to content

Commit

Permalink
db: cap size of multi-level compactions at expanded bytes size
Browse files Browse the repository at this point in the history
Previously, we had a missing check for the max "expanded" size
of a compaction being exceeded when we were trying to convert a
regular compaction to a multi-level compaction. This change adds
that check to prevent multi level compactions from getting too big.

Fixes #3120
  • Loading branch information
itsbilal committed Sep 17, 2024
1 parent 98fae22 commit 9e2ddbb
Show file tree
Hide file tree
Showing 3 changed files with 15 additions and 9 deletions.
7 changes: 7 additions & 0 deletions compaction_picker.go
Original file line number Diff line number Diff line change
Expand Up @@ -1683,6 +1683,13 @@ func (wa WriteAmpHeuristic) pick(
if !pcMulti.setupMultiLevelCandidate(opts, diskAvailBytes) {
return pcOrig
}
// We consider the addition of a level as an "expansion" of the compaction.
// If pcMulti is past the expanded compaction byte size limit already,
// we don't consider it.
if pcMulti.compactionSize() >= expandedCompactionByteSizeLimit(
opts, adjustedOutputLevel(pcMulti.outputLevel.level, pcMulti.baseLevel), diskAvailBytes) {
return pcOrig
}
picked := pcOrig
if pcMulti.predictedWriteAmp() <= pcOrig.predictedWriteAmp()+wa.AddPropensity {
picked = pcMulti
Expand Down
8 changes: 6 additions & 2 deletions event.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,14 +119,18 @@ func (i CompactionInfo) SafeFormat(w redact.SafePrinter, _ rune) {
w.Printf("[JOB %d] compacting(%s) ",
redact.Safe(i.JobID),
redact.SafeString(i.Reason))
w.Printf("%s", i.Annotations)
if len(i.Annotations) > 0 {
w.Printf("%s ", i.Annotations)
}
w.Printf("%s; ", levelInfos(i.Input))
w.Printf("OverlappingRatio: Single %.2f, Multi %.2f", i.SingleLevelOverlappingRatio, i.MultiLevelOverlappingRatio)
return
}
outputSize := tablesTotalSize(i.Output.Tables)
w.Printf("[JOB %d] compacted(%s) ", redact.Safe(i.JobID), redact.SafeString(i.Reason))
w.Printf("%s", i.Annotations)
if len(i.Annotations) > 0 {
w.Printf("%s ", i.Annotations)
}
w.Print(levelInfos(i.Input))
w.Printf(" -> L%d [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
redact.Safe(i.Output.Level),
Expand Down
9 changes: 2 additions & 7 deletions testdata/compaction_setup_inputs_multilevel_write_amp
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,8 @@ init-multi-level(1,2,3)
Original WriteAmp 2.25; ML WriteAmp 1.24
Original OverlappingRatio 1.25; ML OverlappingRatio 0.24

# If max size is exceeded, we no longer pick e-f in L2.
# If max expanded size is exceeded, we don't add an extra level.

setup-inputs avail-bytes=20 a a
L1
a.SET.1-c.SET.2 size=4
Expand All @@ -288,12 +289,6 @@ L1
000001:[a#1,SET-c#2,SET]
L2
000002:[a#5,SET-d#6,SET]
L3
000004:[a#3,SET-b#4,SET]
000005:[c#2,SET-g#2,SET]
init-multi-level(1,2,3)
Original WriteAmp 2.25; ML WriteAmp 1.44
Original OverlappingRatio 1.25; ML OverlappingRatio 0.44

# Don't init a multi-level compaction if the start level is L5.
setup-inputs a a
Expand Down

0 comments on commit 9e2ddbb

Please sign in to comment.