TileServerCacheUI (#8)

* rewrite to vapor

* add todo to tests

* fix paths in docker file + fix wrong command for multistaticmap

* improve ImageMagick error reporting

* Update to Swift 5.2.3 to fix mem leak

* add missing dependencies to Dockerfile

* Add #format #pad and #round tags + add local marker support

* Add #index(array, index) tag because leaf doesn't support subscripting yet

* Add regeneratable support

* fix # in color

* fix encoding of collors

* Better Error Reporting
- Better error reporting
- Add name to /styles call

* Fix cache cleaner not cleaning

* undo debug to verbose logging change

* Some Cleanup

* Missing File

* some testing and fixes

* More Cleanup & Fix Cleaning

* Add POST support to for templates

* Update README with new routes

* Add support for 3rd party providers (e.g. MapBox, Google Maps) for tiles and static maps

* Fix templating problem with new leaf version

* add "fallback_url" to markers

* fix hit ratio stats for tiles

* skip markers outside of view

* Limit markers to content type "image/*"

* Clear template cache on changes, add circles, fixes

* initial TileServerCacheUI implementation

* Fix default config.json

* add MAX_BODY_SIZE option

* Add Maputnik link

* Show missing fonts/icons + auto unify font names on import

* Fix External Styles

* Fix Fonts and Icons Analysis

* feat: add regeneratable cache cleaner

* fix: fix cache cleaner paths

* fix: fix initial setup

* refactor: remove unused which command from Dockerfile

* fix: properly escape shell commands

* fix: fix mbtiles combining

* fix: fix local image check for fallback urls

* fix: clear affected images if imagemagick fails

* refactor: remove file headers

* doc: enable markers volume by default

* build: pin build dependencies to working versions

* fix: only copy config when ui is enabled

* feat: update to swift 5.7

* fix: correct templates view heading

* feat: add support to upload zipped styles

* feat: add support to upload `.mbtiles` files directly

.gitignore

@@ -9,4 +9,8 @@ Package.resolved
 /Templates
 /Markers
 /TileServer
+/Temp
 .swiftpm/
+.env
+.env.development
+docker-compose.development.yml

Dockerfile

@@ -1,7 +1,7 @@
 # ================================
 # Build image
 # ================================
-FROM swift:5.2 as build
+FROM swift:5.7 as build
 WORKDIR /build
 
 # Copy required folders into container
@@ -19,12 +19,27 @@ RUN swift build \
 # ================================
 # Run image
 # ================================
-FROM swift:5.2
+FROM swift:5.7
 WORKDIR /SwiftTileserverCache
 
 # Install imagemagick
 RUN apt-get -y update && apt-get install -y imagemagick
 
+# Install tippecanoe requirements
+RUN apt-get -y update && apt-get -y install build-essential libsqlite3-dev zlib1g-dev
+
+RUN git clone https://github.com/mapbox/tippecanoe.git -b 1.36.0 \
+ && cd tippecanoe \
+ && make -j \
+ && make install \
+ && rm -rf tippecanoe
+
+# Install fontnik requirements
+RUN apt-get -y update && apt-get -y install nodejs npm
+
+ # Install fontnik
+RUN npm install -g [email protected]
+
 # Copy build artifacts
 COPY --from=build /build/.build/release /SwiftTileserverCache
 # Copy Resources

Package.swift

@@ -1,27 +1,29 @@
-// swift-tools-version:5.2
+// swift-tools-version:5.7
 
 import PackageDescription
 
 let package = Package(
     name: "SwiftTileserverCache",
     platforms: [
-        .macOS(.v10_15) // linux does not yet have runntime availability checks so this doesn't apply to linux yet
+        .macOS(.v12) // linux does not yet have runtime availability checks so this doesn't apply to linux yet
     ],
     dependencies: [
-        .package(url: "https://github.com/vapor/vapor", from: "4.0.0"),
-        .package(url: "https://github.com/vapor/leaf", from: "4.0.0"),
-        .package(url: "https://github.com/JohnSundell/ShellOut", from: "2.3.0")
+        .package(url: "https://github.com/vapor/vapor", .upToNextMinor(from: "4.69.1")),
+        .package(url: "https://github.com/vapor/leaf", .upToNextMinor(from: "4.2.4")),
+        .package(url: "https://github.com/JohnSundell/ShellOut", .upToNextMinor(from: "2.3.0")),
+        .package(url: "https://github.com/weichsel/ZIPFoundation.git", .upToNextMajor(from: "0.9.16"))
     ],
     targets: [
         .target(
             name: "SwiftTileserverCache",
             dependencies: [
                 .product(name: "Vapor", package: "vapor"),
                 .product(name: "Leaf", package: "leaf"),
-                .product(name: "ShellOut", package: "ShellOut")
+                .product(name: "ShellOut", package: "ShellOut"),
+                .product(name: "ZIPFoundation", package: "ZIPFoundation")
             ]
         ),
-        .target(
+        .executableTarget(
             name: "SwiftTileserverCacheApp",
             dependencies: [
                 .target(name: "SwiftTileserverCache"),

Resources/TileServer/Empty.mbtiles

@@ -0,0 +1 @@
+

Resources/TileServer/config.json

@@ -0,0 +1,20 @@
+{
+  "options": {
+    "paths": {
+      "root": "/usr/src/app/node_modules/tileserver-gl-styles",
+      "fonts": "/data/Fonts",
+      "styles": "/data/Styles",
+      "sprites": "/data/Styles",
+      "mbtiles": "/data/Datasets"
+    },
+    "serveAllFonts": true,
+    "serveAllStyles": true,
+    "serveStaticMaps": true
+  },
+  "styles": {},
+  "data": {
+    "combined": {
+      "mbtiles": "Combined.mbtiles"
+    }
+  }
+}

Resources/Views/Base.leaf

@@ -0,0 +1,53 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <title>SwiftTileserverCache #if(pageName): - #(pageName)#endif</title>
+
+        <meta charset="utf-8"/>
+        <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">
+
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.13.0/css/all.min.css" integrity="sha256-h20CPZ0QyXlBuAw7A+KluUYx/3pK+c7lYEpqLTlxjYQ=" crossorigin="anonymous" />
+        <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css" integrity="sha384-JcKb8q3iqJ61gNV9KGb8thSsNjpSL0n8PARn9HuZOnIxN0hoP+VmmDGMN5t9UJ0Z" crossorigin="anonymous">
+        <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/v/bs4/dt-1.10.21/b-1.6.3/fh-3.1.7/r-2.2.5/rg-1.1.2/datatables.min.css"/>
+        #import("stylesheets")
+
+        <script src="https://code.jquery.com/jquery-3.5.1.min.js" integrity="sha256-9/aliU8dGd2tb6OSsuzixeV4y/faTqgFtohetphbbj0=" crossorigin="anonymous"></script>
+        <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/umd/popper.min.js" integrity="sha384-9/reFTGAW83EW2RDu2S0VKaIzap3H66lZH81PoYlFhbGU+6BZp6G7niu735Sk7lN" crossorigin="anonymous"></script>
+        <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/js/bootstrap.min.js" integrity="sha384-B4gt1jrGC7Jh4AgTPSdUtOBvfO8shuf57BaghqFfPlYxofvL8/KUEfYiJOMMV+rV" crossorigin="anonymous"></script>
+        <script type="text/javascript" src="https://cdn.datatables.net/v/bs4/dt-1.10.21/b-1.6.3/fh-3.1.7/r-2.2.5/rg-1.1.2/datatables.min.js"></script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/ace/1.4.12/ace.min.js" integrity="sha512-GoORoNnxst42zE3rYPj4bNBm0Q6ZRXKNH2D9nEmNvVF/z24ywVnijAWVi/09iBiVDQVf3UlZHpzhAJIdd9BXqw==" crossorigin="anonymous"></script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/ace/1.4.12/mode-json.min.js" integrity="sha512-VZZMcLUCIF2nfbd/WddhQvF0/K/hWtKLTMEEc3/ouKX3ceCgEEPJR/c1buV/XDm1lMJrP9ZZ4izN+3VUZxCGHA==" crossorigin="anonymous"></script>
+        #import("scripts")
+    </head>
+
+    <body>
+        <nav class="navbar navbar-expand-lg navbar-dark bg-dark">
+            <a class="navbar-brand " href="/">SwiftTileserverCache</a>
+            <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="\#navbarText" aria-controls="navbarText" aria-expanded="false" aria-label="Toggle navigation">
+                <span class="navbar-toggler-icon"></span>
+            </button>
+            <div class="collapse navbar-collapse" id="navbarText">
+                <ul class="navbar-nav mr-auto">
+                    <li class="nav-item #if(pageId=="stats"):active#endif">
+                        <a class="nav-link" href="/admin/stats">Stats</a>
+                    </li>
+                    <li class="nav-item #if(pageId=="datasets"):active#endif">
+                        <a class="nav-link" href="/admin/datasets">Datasets</a>
+                    </li>
+                    <li class="nav-item #if(pageId=="styles"):active#endif">
+                        <a class="nav-link" href="/admin/styles">Styles</a>
+                    </li>
+                    <li class="nav-item #if(pageId=="fonts"):active#endif">
+                        <a class="nav-link" href="/admin/fonts">Fonts</a>
+                    </li>
+                    <li class="nav-item #if(pageId=="templates"):active#endif">
+                        <a class="nav-link" href="/admin/templates">Templates</a>
+                    </li>
+                </ul>
+            </div>
+        </nav>
+        <br>
+        #import("content")
+        <br>
+    </body>
+</html>

Resources/Views/Datasets.leaf

@@ -0,0 +1,63 @@
+#extend("Resources/Views/Base"):
+    #export("stylesheets"):
+    #endexport
+    #export("scripts"):
+        <script>
+            $(document).ready(function() {
+                $('\#table').DataTable({
+                    ordering: true,
+                    paging: true,
+                    dom: 'Bfrtip',
+                    buttons: {
+                        buttons: [{
+                            text: 'Add Dataset',
+                            className: 'btn-success',
+                            action: function () {
+                                window.location = '/admin/datasets/add'
+                            }
+                        }],
+                        dom: {
+                            button: {
+                                className: 'btn'
+                            }
+                        }
+                    }
+                })
+            })
+
+            function deleteDataset(dataset) {
+                if (confirm('Are you sure, that you want to delete Dataset ' + dataset)) {
+                    window.location = '/admin/datasets/delete/' + dataset
+                }
+            }
+        </script>
+    #endexport
+    #export("content"):
+        <h1 align="center">Datasets</h1>
+        <br>
+        <div style="width:90%; margin-left: 5%; mergin-right: 5%">
+            <table style="width:100%;" id="table" class="table table-striped table-bordered dt-responsive nowrap">
+                <thead>
+                    <tr>
+                        <th>Dataset</th>
+                        <th style="width: 100px">Actions</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    #for(dataset in datasets):
+                    <tr>
+                        <td style="vertical-align:middle">
+                            #(dataset)
+                        </td>
+                        <td style="vertical-align:middle">
+                            <div style="display: flex" class="btn-group" role="group" aria-label="Basic example">
+                                <button style="flex: 1" role="button" class="btn btn-danger" onclick="deleteDataset('#(dataset)')">Delete</a>
+                            </div>
+                        </td>
+                    </tr>
+                    #endfor
+                </tbody>
+            </table>
+        </div>
+    #endexport
+#endextend

Resources/Views/DatasetsAdd.leaf

@@ -0,0 +1,116 @@
+#extend("Resources/Views/Base"):
+    #export("stylesheets"):
+    #endexport
+    #export("scripts"):
+        <script>
+            $(function() {
+                $('\#url').on('paste', function(event) {
+                    const data = event.clipboardData || event.originalEvent.clipboardData || window.clipboardData;
+                    const text = data.getData('text').replace('wget -c ', '')
+                    $('\#url').val(text)
+                    event.preventDefault()
+                });
+            });
+
+            function addDataset() {
+                startedLoading()
+                const url = $('\#url').val()
+                const file = $('\#file').prop('files')[0]
+                const name = $('\#name').val()
+                if (url) {
+                    addDatasetUrl(name, url)
+                } else if (file) {
+                    addDatasetFile(name, file)
+                } else {
+                    stoppedLoading('Please specify either a URL or a File!')
+                }
+                return false;
+            }
+
+            function addDatasetUrl(name, url) {
+                const webSocket = new WebSocket(window.location.protocol.replace('http','ws')+'//'+window.location.host+'/admin/api/datasets/add')
+                webSocket.onopen = function() {
+                    webSocket.send(name+';'+url)
+                }
+                webSocket.onerror = function(event) {
+                    stoppedLoading('Failed to connect to the server!')
+                }
+                webSocket.onmessage = function(event) {
+                    if (event.data == 'ok') {
+                        webSocket.close()
+                        window.location = '/admin/datasets'
+                    } else if (event.data == 'downloaded') {
+                        $('\#progress-text').html("Combining Datasets...")
+                    } else {
+                        webSocket.close()
+                        stoppedLoading(event.data ? 'Failed to download Dataset: ' + event.data : 'Failed to connect to the server!')
+                    }
+                }
+            }
+
+            function addDatasetFile(name, file) {
+                let data = new FormData();
+                data.append("name", name);
+                data.append("file", file);
+                $.ajax({
+                    type: "POST",
+                    data: data,
+                    url: "/admin/api/datasets/add",
+                    processData: false,
+                    contentType: false,
+                    success: function() {
+                        window.location = '/admin/datasets'
+                    },
+                    error: function(data, error, errorText) {
+                        stoppedLoading('Failed to upload Dataset: ' + (data.responseJSON ? data.responseJSON.reason : errorText))
+                    }
+                })
+            }
+
+            function setProgress(text, percentage) {
+                $('\#progress-text').html(text)
+            }
+
+            function startedLoading() {
+                $('\#progress-text').html($('\#file').prop('files').length ? "Uploading and Combining Dataset..." : "Downloading Dataset...")
+                $('\#submit').prop('disabled', true)
+                $('\#progress').show()
+            }
+
+            function stoppedLoading(error) {
+                $('\#submit').prop('disabled', false)
+                $('\#progress').hide()
+                alert(error)
+            }
+        </script>
+    #endexport
+    #export("content"):
+        <h1 align="center">Add Dataset</h1>
+        <br>
+        <div style="width:90%; margin-left: 5%; mergin-right: 5%">
+            <form onsubmit="return addDataset()">
+                <div class="form-group">
+                    Dataset Name
+                    <input id="name" maxlength="50" type="text" class="form-control" name="name" required>
+                </div>
+                <div>Specify either a URL to download or a File to upload. (<a target="_blank" href="https://openmaptiles.com/downloads/planet/">Download from OpenMapTiles</a>)</div>
+                <div class="row g-3">
+                    <div class="col form-group">
+                        Dataset URL
+                        <input id="url" type="url" class="form-control" name="url">
+                    </div>
+                    <div class="col form-group">
+                        Dataset File
+                        <input id="file" type="file" class="form-control" accept=".mbtiles">
+                    </div>
+                </div>
+                <button id="submit" type="submit" class="btn btn-primary float-right">Add Dataset</button>
+            </form>
+        </div>
+        <div id="progress" class="progress" style="width:90%; margin-left: 5%; mergin-right: 5%; margin-top: 70px; display: none;">
+          <div class="progress-bar progress-bar-striped progress-bar-animated" role="progressbar" aria-valuenow="100" aria-valuemin="0" aria-valuemax="100" style="width: 100%">
+            <span id="progress-text"></span>
+          </div>
+        </div>
+    #endexport
+#endextend