diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java index 4843147426..7b932d5587 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java @@ -402,7 +402,6 @@ private long tilesEmitted() { private void finishArchive() { archive.finish(tileArchiveMetadata.withLayerStats(layerAttrStats.getTileStats())); - archive.printStats(); } /** diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java index f64c70d614..d6dca49100 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/archive/WriteableTileArchive.java @@ -44,8 +44,6 @@ default void initialize() {} */ default void finish(TileArchiveMetadata tileArchiveMetadata) {} - default void printStats() {} - long bytesWritten(); interface TileWriter extends Closeable { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/files/FilesArchiveUtils.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/files/FilesArchiveUtils.java index d5f17fad40..2fd541bdb3 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/files/FilesArchiveUtils.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/files/FilesArchiveUtils.java @@ -7,6 +7,7 @@ import com.onthegomap.planetiler.config.Arguments; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Objects; import java.util.Optional; public final class FilesArchiveUtils { @@ -45,29 +46,48 @@ static TileSchemeEncoding tilesSchemeEncoding(Arguments options, Path basePath, } static BasePathWithTileSchemeEncoding basePathWithTileSchemeEncoding(Arguments options, Path basePath) { - final String basePathStr = basePath.toString(); - final int curlyIndex = basePathStr.indexOf('{'); - if (curlyIndex >= 0) { - final Path newBasePath = Paths.get(basePathStr.substring(0, curlyIndex)); - return new BasePathWithTileSchemeEncoding( - newBasePath, - tilesSchemeEncoding(options, newBasePath, basePathStr.substring(curlyIndex)) - ); - } else { - return new BasePathWithTileSchemeEncoding( - basePath, - tilesSchemeEncoding(options, basePath, Path.of(Z_TEMPLATE, X_TEMPLATE, Y_TEMPLATE + ".pbf").toString())); - } + + final SplitShortcutPath split = SplitShortcutPath.split(basePath); + + final String tileScheme = Objects + .requireNonNullElse(split.tileSchemePart(), Path.of(Z_TEMPLATE, X_TEMPLATE, Y_TEMPLATE + ".pbf")).toString(); + + return new BasePathWithTileSchemeEncoding( + split.basePart(), + tilesSchemeEncoding(options, split.basePart(), tileScheme) + ); } public static Path cleanBasePath(Path basePath) { - final String basePathStr = basePath.toString(); - final int curlyIndex = basePathStr.indexOf('{'); - if (curlyIndex >= 0) { - return Paths.get(basePathStr.substring(0, curlyIndex)); - } - return basePath; + return SplitShortcutPath.split(basePath).basePart(); } record BasePathWithTileSchemeEncoding(Path basePath, TileSchemeEncoding tileSchemeEncoding) {} + + private record SplitShortcutPath(Path basePart, Path tileSchemePart) { + public static SplitShortcutPath split(Path basePath) { + Path basePart = Objects.requireNonNullElse(basePath.getRoot(), Paths.get("")); + Path tileSchemePart = null; + + boolean remainingIsTileScheme = false; + for (int i = 0; i < basePath.getNameCount(); i++) { + final Path part = basePath.getName(i); + if (!remainingIsTileScheme && part.toString().contains("{")) { + remainingIsTileScheme = true; + } + if (remainingIsTileScheme) { + tileSchemePart = tileSchemePart == null ? part : tileSchemePart.resolve(part); + } else { + basePart = basePart.resolve(part); + } + } + + if (tileSchemePart == null) { + // just in case: use the "original" basePath in case no tile scheme is included, but basePart _should_ be identical + return new SplitShortcutPath(basePath, null); + } else { + return new SplitShortcutPath(basePart, tileSchemePart); + } + } + } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/files/TileSchemeEncoding.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/files/TileSchemeEncoding.java index 19632abc9b..1b9dfab6cd 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/files/TileSchemeEncoding.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/files/TileSchemeEncoding.java @@ -5,6 +5,7 @@ import java.io.File; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Objects; import java.util.Optional; import java.util.function.Function; import java.util.regex.Matcher; @@ -161,4 +162,22 @@ private static String validate(String tileScheme) { return tileScheme; } + @Override + public boolean equals(Object o) { + return this == o || (o instanceof TileSchemeEncoding that && Objects.equals(tileScheme, that.tileScheme) && + Objects.equals(basePath, that.basePath)); + } + + @Override + public int hashCode() { + return Objects.hash(tileScheme, basePath); + } + + @Override + public String toString() { + return "TileSchemeEncoding[" + + "tileScheme='" + tileScheme + '\'' + + ", basePath=" + basePath + + ']'; + } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/PrometheusStats.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/PrometheusStats.java index 71143a543f..ecbff23098 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/PrometheusStats.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/PrometheusStats.java @@ -1,6 +1,5 @@ package com.onthegomap.planetiler.stats; -import com.onthegomap.planetiler.util.FileUtils; import com.onthegomap.planetiler.util.MemoryEstimator; import io.prometheus.client.Collector; import io.prometheus.client.CollectorRegistry; @@ -49,8 +48,7 @@ class PrometheusStats implements Stats { private PushGateway pg; private ScheduledExecutorService executor; private final String job; - private final Map filesToMonitor = new ConcurrentSkipListMap<>(); - private final Map sizesOfFilesToMonitor = new ConcurrentSkipListMap<>(); + private final Map filesToMonitor = new ConcurrentSkipListMap<>(); private final Map dataErrorCounters = new ConcurrentHashMap<>(); private final Map heapObjectsToMonitor = new ConcurrentSkipListMap<>(); @@ -172,15 +170,10 @@ public Timers timers() { } @Override - public Map monitoredFiles() { + public Map monitoredFiles() { return filesToMonitor; } - @Override - public Map monitoredFileSizes() { - return sizesOfFilesToMonitor; - } - @Override public void monitorInMemoryObject(String name, MemoryEstimator.HasEstimate object) { heapObjectsToMonitor.put(name, object); @@ -254,11 +247,11 @@ private class FileSizeCollector extends Collector { @Override public List collect() { List results = new ArrayList<>(); - for (var file : filesToMonitor.entrySet()) { - String name = sanitizeMetricName(file.getKey()); - Path path = file.getValue(); - var sizeSupplier = monitoredFileSizes().getOrDefault(file.getKey(), () -> FileUtils.size(path)); - long size = sizeSupplier.getAsLong(); + for (var entry : filesToMonitor.entrySet()) { + String name = sanitizeMetricName(entry.getKey()); + MonitoredFile monitoredFile = entry.getValue(); + Path path = monitoredFile.path(); + long size = monitoredFile.sizeProvider().getAsLong(); results.add(new GaugeMetricFamily(BASE + "file_" + name + "_size_bytes", "Size of " + name + " in bytes", size)); if (Files.exists(path)) { diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java index d978620c02..9713a2681c 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java @@ -60,8 +60,7 @@ default void printSummary() { timers().printSummary(); logger.info("-".repeat(40)); for (var entry : monitoredFiles().entrySet()) { - var sizeSupplier = monitoredFileSizes().getOrDefault(entry.getKey(), () -> FileUtils.size(entry.getValue())); - long size = sizeSupplier.getAsLong(); + long size = entry.getValue().sizeProvider().getAsLong(); if (size > 0) { logger.info("\t{}\t{}B", entry.getKey(), format.storage(size, false)); } @@ -119,9 +118,7 @@ default Timers.Finishable startStage(String name, boolean log) { Timers timers(); /** Returns all the files being monitored. */ - Map monitoredFiles(); - - Map monitoredFileSizes(); + Map monitoredFiles(); /** Adds a stat that will track the size of a file or directory located at {@code path}. */ default void monitorFile(String name, Path path) { @@ -130,10 +127,7 @@ default void monitorFile(String name, Path path) { default void monitorFile(String name, Path path, LongSupplier sizeProvider) { if (path != null) { - monitoredFiles().put(name, path); - } - if (sizeProvider != null) { - monitoredFileSizes().put(name, sizeProvider); + monitoredFiles().put(name, new MonitoredFile(path, sizeProvider)); } } @@ -199,7 +193,7 @@ class InMemory implements Stats { private InMemory() {} private final Timers timers = new Timers(); - private final Map monitoredFiles = new ConcurrentSkipListMap<>(); + private final Map monitoredFiles = new ConcurrentSkipListMap<>(); private final Map monitoredFileSizes = new ConcurrentSkipListMap<>(); private final Map dataErrors = new ConcurrentHashMap<>(); @@ -212,15 +206,10 @@ public Timers timers() { } @Override - public Map monitoredFiles() { + public Map monitoredFiles() { return monitoredFiles; } - @Override - public Map monitoredFileSizes() { - return monitoredFileSizes; - } - @Override public void monitorInMemoryObject(String name, MemoryEstimator.HasEstimate object) {} @@ -259,4 +248,11 @@ public void close() { } } + + record MonitoredFile(Path path, LongSupplier sizeProvider) { + public MonitoredFile(Path path, LongSupplier sizeProvider) { + this.path = path; + this.sizeProvider = sizeProvider != null ? sizeProvider : () -> FileUtils.size(path); + } + } } diff --git a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableStreamArchive.java b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableStreamArchive.java index ab59111630..d902f3e0c0 100644 --- a/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableStreamArchive.java +++ b/planetiler-core/src/main/java/com/onthegomap/planetiler/stream/WriteableStreamArchive.java @@ -47,7 +47,7 @@ abstract class WriteableStreamArchive implements WriteableTileArchive { private WriteableStreamArchive(OutputStreamSupplier outputStreamFactory, StreamArchiveConfig config) { this.outputStreamFactory = - i -> new CountingOutputStream(outputStreamFactory.newOutputStream(i), bytesWritten::incBy); + i -> new CountingOutputStream(outputStreamFactory.newOutputStream(i), bytesWritten.counterForThread()::incBy); this.config = config; this.primaryOutputStream = this.outputStreamFactory.newOutputStream(0); diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/files/FilesArchiveUtilsTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/files/FilesArchiveUtilsTest.java new file mode 100644 index 0000000000..311b0f292e --- /dev/null +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/files/FilesArchiveUtilsTest.java @@ -0,0 +1,107 @@ +package com.onthegomap.planetiler.files; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.onthegomap.planetiler.config.Arguments; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +class FilesArchiveUtilsTest { + + @ParameterizedTest + @CsvSource(textBlock = """ + {z}/{x}/{y}.pbf , , {z}/{x}/{y}.pbf + , , {z}/{x}/{y}.pbf + {x}/{y}/{z}.pbf , , {x}/{y}/{z}.pbf + tiles/{z}/{x}/{y}.pbf , tiles, {z}/{x}/{y}.pbf + tiles/z{z}/{x}/{y}.pbf , tiles, z{z}/{x}/{y}.pbf + z{z}/x{x}/y{y}.pbf , , z{z}/x{x}/y{y}.pbf + tiles/tile-{z}-{x}-{y}.pbf, tiles, tile-{z}-{x}-{y}.pbf + /a , /a , {z}/{x}/{y}.pbf + / , / , {z}/{x}/{y}.pbf + """ + ) + void testBasePathWithTileSchemeEncoding(String shortcutBase, String actualBase, String tileScheme, + @TempDir Path tempDir) { + + final Path shortcutBasePath = makePath(shortcutBase, tempDir); + final Path actualBasePath = makePath(actualBase, tempDir); + + assertEquals( + new FilesArchiveUtils.BasePathWithTileSchemeEncoding( + actualBasePath, + new TileSchemeEncoding( + Paths.get(tileScheme).toString(), + actualBasePath + ) + ), + FilesArchiveUtils.basePathWithTileSchemeEncoding(Arguments.of(), shortcutBasePath) + ); + } + + @Test + void testBasePathWithTileSchemeEncodingPrefersArgOverShortcut() { + final Path basePath = Paths.get(""); + final Path schemeShortcutPath = Paths.get("{x}", "{y}", "{z}.pbf"); + final Path schemeArgumentPath = Paths.get("x{x}", "y{y}", "z{z}.pbf"); + final Path shortcutPath = basePath.resolve(schemeShortcutPath); + assertEquals( + new FilesArchiveUtils.BasePathWithTileSchemeEncoding( + basePath, + new TileSchemeEncoding( + schemeShortcutPath.toString(), + basePath + ) + ), + FilesArchiveUtils.basePathWithTileSchemeEncoding(Arguments.of(), shortcutPath) + ); + assertEquals( + new FilesArchiveUtils.BasePathWithTileSchemeEncoding( + basePath, + new TileSchemeEncoding( + schemeArgumentPath.toString(), + basePath + ) + ), + FilesArchiveUtils.basePathWithTileSchemeEncoding( + Arguments.of(Map.of(FilesArchiveUtils.OPTION_TILE_SCHEME, schemeArgumentPath.toString())), shortcutPath) + ); + } + + @ParameterizedTest + @CsvSource(textBlock = """ + {z}/{x}/{y}.pbf , + , + {x}/{y}/{z}.pbf , + tiles/{z}/{x}/{y}.pbf , tiles + tiles/z{z}/{x}/{y}.pbf , tiles + z{z}/x{x}/y{y}.pbf , + tiles/tile-{z}-{x}-{y}.pbf, tiles + /a , /a + / , / + """ + ) + void testCleanBasePath(String shortcutBase, String actualBase, @TempDir Path tempDir) { + + assertEquals( + makePath(actualBase, tempDir), + FilesArchiveUtils.cleanBasePath(makePath(shortcutBase, tempDir)) + ); + } + + + private static Path makePath(String in, @TempDir Path tempDir) { + if (in == null) { + return Paths.get(""); + } + if (in.startsWith("/")) { + return tempDir.resolve(in.substring(1)); + } + return Paths.get(in); + } +} diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/files/ReadableFilesArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/files/ReadableFilesArchiveTest.java index 86e0c3695d..ad9d003854 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/files/ReadableFilesArchiveTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/files/ReadableFilesArchiveTest.java @@ -16,6 +16,7 @@ import java.nio.file.Paths; import java.util.List; import java.util.Map; +import java.util.Objects; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -112,18 +113,29 @@ void testReadCustomScheme(String tileScheme, Path tileFile, @TempDir Path tempDi } } - @Test - void testTileSchemeFromBasePath(@TempDir Path tempDir) throws IOException { - final Path tilesDir = tempDir.resolve("tiles"); - final Path basePath = tilesDir.resolve(Paths.get("{x}", "{y}", "{z}.pbf")); - final Path tileFile = tilesDir.resolve(Paths.get("1", "2", "3.pbf")); + @ParameterizedTest + @CsvSource(textBlock = """ + {z}/{x}/{y}.pbf , , 3/1/2.pbf + tiles/{z}/{x}/{y}.pbf , tiles, tiles/3/1/2.pbf + tiles/z{z}/{x}/{y}.pbf , tiles, tiles/z3/1/2.pbf + z{z}/x{x}/y{y}.pbf , , z3/x1/y2.pbf + tiles/tile-{z}-{x}-{y}.pbf, tiles, tiles/tile-3-1-2.pbf + """ + ) + void testTileSchemeFromBasePath(Path shortcutBasePath, Path actualBasePath, Path tileFile, @TempDir Path tempDir) + throws IOException { + final Path testBase = tempDir.resolve("tiles"); + + shortcutBasePath = testBase.resolve(shortcutBasePath); + actualBasePath = testBase.resolve(Objects.requireNonNullElse(actualBasePath, Paths.get(""))); + tileFile = testBase.resolve(tileFile); Files.createDirectories(tileFile.getParent()); Files.write(tileFile, new byte[]{1}); - final Path metadataFile = tilesDir.resolve("metadata.json"); + final Path metadataFile = actualBasePath.resolve("metadata.json"); Files.writeString(metadataFile, TestUtils.MAX_METADATA_SERIALIZED); - try (var archive = ReadableFilesArchive.newReader(basePath, Arguments.of())) { + try (var archive = ReadableFilesArchive.newReader(shortcutBasePath, Arguments.of())) { assertEquals( List.of(TileCoord.ofXYZ(1, 2, 3)), archive.getAllTileCoords().stream().toList() diff --git a/planetiler-core/src/test/java/com/onthegomap/planetiler/files/WriteableFilesArchiveTest.java b/planetiler-core/src/test/java/com/onthegomap/planetiler/files/WriteableFilesArchiveTest.java index a804c9dbba..6f375107ee 100644 --- a/planetiler-core/src/test/java/com/onthegomap/planetiler/files/WriteableFilesArchiveTest.java +++ b/planetiler-core/src/test/java/com/onthegomap/planetiler/files/WriteableFilesArchiveTest.java @@ -14,6 +14,7 @@ import java.nio.file.Paths; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.OptionalLong; import java.util.stream.Stream; import org.junit.jupiter.api.Test; @@ -84,19 +85,32 @@ void testWriteCustomScheme(String tileScheme, Path expectedFile, @TempDir Path t assertTrue(Files.exists(expectedFile)); } - @Test - void testTileSchemeFromBasePath(@TempDir Path tempDir) throws IOException { - final Path tilesDir = tempDir.resolve("tiles"); - final Path basePath = tilesDir.resolve(Paths.get("{x}", "{y}", "{z}.pbf")); - try (var archive = WriteableFilesArchive.newWriter(basePath, Arguments.of(), false)) { + @ParameterizedTest + @CsvSource(textBlock = """ + {z}/{x}/{y}.pbf , , 3/1/2.pbf + tiles/{z}/{x}/{y}.pbf , tiles, tiles/3/1/2.pbf + tiles/z{z}/{x}/{y}.pbf , tiles, tiles/z3/1/2.pbf + z{z}/x{x}/y{y}.pbf , , z3/x1/y2.pbf + tiles/tile-{z}-{x}-{y}.pbf, tiles, tiles/tile-3-1-2.pbf + """ + ) + void testTileSchemeFromBasePath(Path shortcutBasePath, Path actualBasePath, Path tileFile, @TempDir Path tempDir) + throws IOException { + final Path testBase = tempDir.resolve("tiles"); + + shortcutBasePath = testBase.resolve(shortcutBasePath); + actualBasePath = testBase.resolve(Objects.requireNonNullElse(actualBasePath, Paths.get(""))); + tileFile = testBase.resolve(tileFile); + + try (var archive = WriteableFilesArchive.newWriter(shortcutBasePath, Arguments.of(), false)) { try (var tileWriter = archive.newTileWriter()) { tileWriter.write(new TileEncodingResult(TileCoord.ofXYZ(1, 2, 3), new byte[]{1}, OptionalLong.empty())); } archive.finish(TestUtils.MAX_METADATA_DESERIALIZED); } - assertTrue(Files.exists(tilesDir.resolve(Paths.get("1", "2", "3.pbf")))); - assertTrue(Files.exists(tilesDir.resolve("metadata.json"))); + assertTrue(Files.exists(tileFile)); + assertTrue(Files.exists(actualBasePath.resolve("metadata.json"))); } private void testMetadataWrite(Arguments options, Path archiveOutput, Path metadataTilesDir) throws IOException {