add tile-copy utility

to copy tiles from one archive into another e.g. mbtiles to files, ...

planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java

@@ -41,7 +41,6 @@
 import java.util.List;
 import java.util.Optional;
 import java.util.function.Function;
-import java.util.stream.IntStream;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -656,18 +655,8 @@ public void run() throws Exception {
       System.exit(0);
     } else if (onlyDownloadSources) {
       // don't check files if not generating map
-    } else if (config.append()) {
-      if (!output.format().supportsAppend()) {
-        throw new IllegalArgumentException("cannot append to " + output.format().id());
-      }
-      if (!output.exists()) {
-        throw new IllegalArgumentException(output.uri() + " must exist when appending");
-      }
-    } else if (overwrite || config.force()) {
-      output.delete();
-    } else if (output.exists()) {
-      throw new IllegalArgumentException(
-        output.uri() + " already exists, use the --force argument to overwrite or --append.");
+    } else {
+      output.setup(config.force() || overwrite, config.append(), config.tileWriteThreads());
     }
 
     Path layerStatsPath = arguments.file("layer_stats", "layer stats output path",
@@ -677,23 +666,6 @@ public void run() throws Exception {
     if (config.tileWriteThreads() < 1) {
       throw new IllegalArgumentException("require tile_write_threads >= 1");
     }
-    if (config.tileWriteThreads() > 1) {
-      if (!output.format().supportsConcurrentWrites()) {
-        throw new IllegalArgumentException(output.format() + " doesn't support concurrent writes");
-      }
-      IntStream.range(1, config.tileWriteThreads())
-        .mapToObj(output::getPathForMultiThreadedWriter)
-        .forEach(p -> {
-          if (!config.append() && (overwrite || config.force())) {
-            FileUtils.delete(p);
-          }
-          if (config.append() && !output.exists(p)) {
-            throw new IllegalArgumentException("indexed archive \"" + p + "\" must exist when appending");
-          } else if (!config.append() && output.exists(p)) {
-            throw new IllegalArgumentException("indexed archive \"" + p + "\" must not exist when not appending");
-          }
-        });
-    }
 
     LOGGER.info("Building {} profile into {} in these phases:", profile.getClass().getSimpleName(), output.uri());
 
@@ -718,7 +690,7 @@ public void run() throws Exception {
     // in case any temp files are left from a previous run...
     FileUtils.delete(tmpDir, nodeDbPath, featureDbPath, multipolygonPath);
     Files.createDirectories(tmpDir);
-    FileUtils.createParentDirectories(nodeDbPath, featureDbPath, multipolygonPath, output.getLocalBasePath());
+    FileUtils.createParentDirectories(nodeDbPath, featureDbPath, multipolygonPath);
 
     if (!toDownload.isEmpty()) {
       download();

planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveConfig.java

@@ -15,6 +15,7 @@
 import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
 /**
@@ -157,15 +158,14 @@ public Path getLocalPath() {
   /**
    * Returns the local <b>base</b> path for this archive, for which directories should be pre-created for.
    */
-  public Path getLocalBasePath() {
+  Path getLocalBasePath() {
     Path p = getLocalPath();
     if (format() == Format.FILES) {
       p = FilesArchiveUtils.cleanBasePath(p);
     }
     return p;
   }
 
-
   /**
    * Deletes the archive if possible.
    */
@@ -186,7 +186,7 @@ public boolean exists() {
    * @param p path to the archive
    * @return {@code true} if the archive already exists, {@code false} otherwise.
    */
-  public boolean exists(Path p) {
+  private boolean exists(Path p) {
     if (p == null) {
       return false;
     }
@@ -228,6 +228,41 @@ public Path getPathForMultiThreadedWriter(int index) {
     };
   }
 
+  public void setup(boolean force, boolean append, int tileWriteThreads) {
+    if (append) {
+      if (!format().supportsAppend()) {
+        throw new IllegalArgumentException("cannot append to " + format().id());
+      }
+      if (!exists()) {
+        throw new IllegalArgumentException(uri() + " must exist when appending");
+      }
+    } else if (force) {
+      delete();
+    } else if (exists()) {
+      throw new IllegalArgumentException(uri() + " already exists, use the --force argument to overwrite or --append.");
+    }
+
+    if (tileWriteThreads > 1) {
+      if (!format().supportsConcurrentWrites()) {
+        throw new IllegalArgumentException(format() + " doesn't support concurrent writes");
+      }
+      IntStream.range(1, tileWriteThreads)
+        .mapToObj(this::getPathForMultiThreadedWriter)
+        .forEach(p -> {
+          if (!append && force) {
+            FileUtils.delete(p);
+          }
+          if (append && !exists(p)) {
+            throw new IllegalArgumentException("indexed archive \"" + p + "\" must exist when appending");
+          } else if (!append && exists(p)) {
+            throw new IllegalArgumentException("indexed archive \"" + p + "\" must not exist when not appending");
+          }
+        });
+    }
+
+    FileUtils.createParentDirectories(getLocalBasePath());
+  }
+
   public enum Format {
     MBTILES("mbtiles",
       false /* TODO mbtiles could support append in the future by using insert statements with an "on conflict"-clause (i.e. upsert) and by creating tables only if they don't exist, yet */,

planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveMetadata.java

@@ -169,6 +169,11 @@ public TileArchiveMetadata withJson(TileArchiveMetadataJson json) {
       maxzoom, json, others, tileCompression);
   }
 
+  public TileArchiveMetadata withTileCompression(TileCompression tileCompression) {
+    return new TileArchiveMetadata(name, description, attribution, version, type, format, bounds, center, minzoom,
+      maxzoom, json, others, tileCompression);
+  }
+
   /*
    * few workarounds to make collect unknown fields to others work,
    * because @JsonAnySetter does not yet work on constructor/creator arguments

planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchiveWriter.java

@@ -260,7 +260,7 @@ private void tileEncoderSink(Iterable<TileBatch> prev) throws IOException {
      * To optimize emitting many identical consecutive tiles (like large ocean areas), memoize output to avoid
      * recomputing if the input hasn't changed.
      */
-    byte[] lastBytes = null, lastEncoded = null;
+    byte[] lastBytes = null;
     Long lastTileDataHash = null;
     boolean lastIsFill = false;
     List<TileSizeStats.LayerStats> lastLayerStats = null;
@@ -275,24 +275,22 @@ private void tileEncoderSink(Iterable<TileBatch> prev) throws IOException {
       for (int i = 0; i < batch.in.size(); i++) {
         FeatureGroup.TileFeatures tileFeatures = batch.in.get(i);
         featuresProcessed.incBy(tileFeatures.getNumFeaturesProcessed());
-        byte[] bytes, encoded;
+        byte[] bytes;
         List<TileSizeStats.LayerStats> layerStats;
         Long tileDataHash;
         if (tileFeatures.hasSameContents(last)) {
           bytes = lastBytes;
-          encoded = lastEncoded;
           tileDataHash = lastTileDataHash;
           layerStats = lastLayerStats;
           memoizedTiles.inc();
         } else {
           VectorTile tile = tileFeatures.getVectorTile(layerAttrStatsUpdater);
           if (skipFilled && (lastIsFill = tile.containsOnlyFills())) {
-            encoded = null;
             layerStats = null;
             bytes = null;
           } else {
             var proto = tile.toProto();
-            encoded = proto.toByteArray();
+            var encoded = proto.toByteArray();
             bytes = switch (config.tileCompression()) {
               case GZIP -> gzip(encoded);
               case NONE -> encoded;
@@ -306,7 +304,6 @@ private void tileEncoderSink(Iterable<TileBatch> prev) throws IOException {
             }
           }
           lastLayerStats = layerStats;
-          lastEncoded = encoded;
           lastBytes = bytes;
           last = tileFeatures;
           if (archive.deduplicates() && tile.likelyToBeDuplicated() && bytes != null) {
@@ -325,7 +322,6 @@ private void tileEncoderSink(Iterable<TileBatch> prev) throws IOException {
             new TileEncodingResult(
               tileFeatures.tileCoord(),
               bytes,
-              encoded.length,
               tileDataHash == null ? OptionalLong.empty() : OptionalLong.of(tileDataHash),
               layerStatsRows
             )

planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileArchives.java

@@ -1,17 +1,23 @@
 package com.onthegomap.planetiler.archive;
 
+import com.onthegomap.planetiler.config.Arguments;
+import com.onthegomap.planetiler.config.CommonConfigs;
 import com.onthegomap.planetiler.config.PlanetilerConfig;
 import com.onthegomap.planetiler.files.ReadableFilesArchive;
 import com.onthegomap.planetiler.files.WriteableFilesArchive;
 import com.onthegomap.planetiler.mbtiles.Mbtiles;
 import com.onthegomap.planetiler.pmtiles.ReadablePmtiles;
 import com.onthegomap.planetiler.pmtiles.WriteablePmtiles;
+import com.onthegomap.planetiler.stream.ReadableCsvArchive;
+import com.onthegomap.planetiler.stream.ReadableJsonStreamArchive;
+import com.onthegomap.planetiler.stream.ReadableProtoStreamArchive;
 import com.onthegomap.planetiler.stream.StreamArchiveConfig;
 import com.onthegomap.planetiler.stream.WriteableCsvArchive;
 import com.onthegomap.planetiler.stream.WriteableJsonStreamArchive;
 import com.onthegomap.planetiler.stream.WriteableProtoStreamArchive;
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.function.Supplier;
 
 /** Utilities for creating {@link ReadableTileArchive} and {@link WriteableTileArchive} instances. */
 public class TileArchives {
@@ -37,45 +43,58 @@ public static ReadableTileArchive newReader(String archive, PlanetilerConfig con
     return newReader(TileArchiveConfig.from(archive), config);
   }
 
+  public static WriteableTileArchive newWriter(TileArchiveConfig archive, PlanetilerConfig config)
+    throws IOException {
+    return newWriter(archive, config.arguments());
+  }
+
   /**
    * Returns a new {@link WriteableTileArchive} from the string definition in {@code archive}.
    *
    * @throws IOException if an error occurs creating the resource.
    */
-  public static WriteableTileArchive newWriter(TileArchiveConfig archive, PlanetilerConfig config)
+  public static WriteableTileArchive newWriter(TileArchiveConfig archive, Arguments baseArguments)
     throws IOException {
-    var options = archive.applyFallbacks(config.arguments());
+    var options = archive.applyFallbacks(baseArguments);
     var format = archive.format();
     return switch (format) {
       case MBTILES ->
         // pass-through legacy arguments for fallback
-        Mbtiles.newWriteToFileDatabase(archive.getLocalPath(), options.orElse(config.arguments()
+        Mbtiles.newWriteToFileDatabase(archive.getLocalPath(), options.orElse(baseArguments
           .subset(Mbtiles.LEGACY_VACUUM_ANALYZE, Mbtiles.LEGACY_COMPACT_DB, Mbtiles.LEGACY_SKIP_INDEX_CREATION)));
       case PMTILES -> WriteablePmtiles.newWriteToFile(archive.getLocalPath());
       case CSV, TSV -> WriteableCsvArchive.newWriteToFile(format, archive.getLocalPath(),
-        new StreamArchiveConfig(config, options));
+        new StreamArchiveConfig(baseArguments, options));
       case PROTO, PBF -> WriteableProtoStreamArchive.newWriteToFile(archive.getLocalPath(),
-        new StreamArchiveConfig(config, options));
+        new StreamArchiveConfig(baseArguments, options));
       case JSON -> WriteableJsonStreamArchive.newWriteToFile(archive.getLocalPath(),
-        new StreamArchiveConfig(config, options));
-      case FILES -> WriteableFilesArchive.newWriter(archive.getLocalPath(), options, config.force() || config.append());
+        new StreamArchiveConfig(baseArguments, options));
+      case FILES -> WriteableFilesArchive.newWriter(archive.getLocalPath(), options,
+        CommonConfigs.appendToArchive(baseArguments) || CommonConfigs.force(baseArguments));
     };
   }
 
+  public static ReadableTileArchive newReader(TileArchiveConfig archive, PlanetilerConfig config)
+    throws IOException {
+    return newReader(archive, config.arguments());
+  }
+
   /**
    * Returns a new {@link ReadableTileArchive} from the string definition in {@code archive}.
    *
    * @throws IOException if an error occurs opening the resource.
    */
-  public static ReadableTileArchive newReader(TileArchiveConfig archive, PlanetilerConfig config)
+  public static ReadableTileArchive newReader(TileArchiveConfig archive, Arguments baseArguments)
     throws IOException {
-    var options = archive.applyFallbacks(config.arguments());
+    var options = archive.applyFallbacks(baseArguments);
+    Supplier<StreamArchiveConfig> streamArchiveConfig = () -> new StreamArchiveConfig(baseArguments, options);
     return switch (archive.format()) {
       case MBTILES -> Mbtiles.newReadOnlyDatabase(archive.getLocalPath(), options);
       case PMTILES -> ReadablePmtiles.newReadFromFile(archive.getLocalPath());
-      case CSV, TSV -> throw new UnsupportedOperationException("reading CSV is not supported");
-      case PROTO, PBF -> throw new UnsupportedOperationException("reading PROTO is not supported");
-      case JSON -> throw new UnsupportedOperationException("reading JSON is not supported");
+      case CSV, TSV ->
+        ReadableCsvArchive.newReader(archive.format(), archive.getLocalPath(), streamArchiveConfig.get());
+      case PROTO, PBF -> ReadableProtoStreamArchive.newReader(archive.getLocalPath(), streamArchiveConfig.get());
+      case JSON -> ReadableJsonStreamArchive.newReader(archive.getLocalPath(), streamArchiveConfig.get());
       case FILES -> ReadableFilesArchive.newReader(archive.getLocalPath(), options);
     };
   }

planetiler-core/src/main/java/com/onthegomap/planetiler/archive/TileCompression.java

@@ -54,10 +54,5 @@ static class Deserializer extends JsonDeserializer<TileCompression> {
     public TileCompression deserialize(JsonParser p, DeserializationContext ctxt) throws IOException {
       return findById(p.getValueAsString()).orElse(TileCompression.UNKNWON);
     }
-
-    @Override
-    public TileCompression getNullValue(DeserializationContext ctxt) {
-      return TileCompression.GZIP;
-    }
   }
 }