Merge pull request #478 from hotosm/feature/refactor-relations

Support for relations/multipolygon, integration tests for replication and raw data, code clean and improvements, updated some docs

Makefile.am

@@ -153,7 +153,6 @@ endif
 install-data-hook:
 	$(MKDIR_P) $(DESTDIR)/$(pkglibdir)
 	$(MKDIR_P) /etc/underpass
-	cp -rvp $(srcdir)/setup/service/underpass.service /etc/systemd/system/
 	cp -rvp $(srcdir)/config/priority.geojson /etc/underpass/
 	cp -rvp $(srcdir)/config/replicator /etc/underpass/
 	cp -rvp $(srcdir)/config/validate /etc/underpass/
@@ -162,6 +161,7 @@ install-data-hook:
 	$(MKDIR_P) $(DESTDIR)/$(docdir)
 	cp -rvp $(srcdir)/docs/*.md $(DESTDIR)/$(docdir)
 	cp -rvp $(srcdir)/setup/service $(DESTDIR)/$(pkglibdir)
+	cp -rvp $(srcdir)/setup/service/underpass.service /etc/systemd/system/
 
 dist-hook: apidoc
 	$(MKDIR_P) $(DESTDIR)/$(docdir)

docs/Dev/debugging.md

@@ -28,7 +28,7 @@ It's also possible to debug Underpass on MacOS. You should use `glibtool` instea
 
 `brew install gdb`
 
-Note that gdb requires special privileges to access Mach ports.
+Note that gdb requires special privileges to access Mac ports.
 You will need to codesign the binary. For instructions, see: https://sourceware.org/gdb/wiki/PermissionsDarwin
 
 Add the alias in your `~/.bashrc` file:
@@ -37,8 +37,11 @@ Add the alias in your `~/.bashrc` file:
 
 ### With LLDB
 
+When running Underpass on a MacOS system with an Arm64 architecture, `gdb` might be not available.
+You can use `lldb` instead.
+
 `lldb -- src/testsuite/libunderpass.all/yaml-test`
-`lldb -- .libs/underpass --bootstrap        `
+`lldb -- .libs/underpass --bootstrap`
 
 
 

setup/db/underpass.sql

@@ -99,13 +99,16 @@ CREATE TABLE IF NOT EXISTS public.relations (
     changeset int8,
     geom public.geometry(Geometry,4326),
     tags JSONB,
-    refs int8[],
+    refs JSONB,
     timestamp timestamp with time zone,
     version int,
     "user" text,
     uid int8
 );
 
+ALTER TABLE ONLY public.relations
+    ADD CONSTRAINT relations_pkey PRIMARY KEY (osm_id);
+
 CREATE TABLE IF NOT EXISTS public.way_refs (
     way_id int8,
     node_id int8
@@ -119,9 +122,13 @@ CREATE TABLE IF NOT EXISTS public.rel_refs (
 CREATE UNIQUE INDEX nodes_id_idx ON public.nodes (osm_id DESC);
 CREATE UNIQUE INDEX ways_poly_id_idx ON public.ways_poly (osm_id DESC);
 CREATE UNIQUE INDEX ways_line_id_idx ON public.ways_line(osm_id DESC);
+CREATE UNIQUE INDEX relations_id_idx ON public.relations(osm_id DESC);
 CREATE INDEX way_refs_node_id_idx ON public.way_refs (node_id);
 CREATE INDEX way_refs_way_id_idx ON public.way_refs (way_id);
 
+CREATE INDEX rel_refs_rel_id_idx ON public.rel_refs (rel_id);
+CREATE INDEX rel_refs_way_id_idx ON public.rel_refs (way_id);
+
 CREATE INDEX nodes_version_idx ON public.nodes (version);
 CREATE INDEX ways_poly_version_idx ON public.ways_poly (version);
 CREATE INDEX ways_line_version_idx ON public.ways_line (version);

setup/install-macos-arm64.sh

@@ -34,25 +34,15 @@ cd ..
 echo "Setting up build ..."
 ./autogen.sh
 mkdir build && cd build
-../configure CXXFLAGS="-arch arm64 \
-    -I/usr/local/include \
-    -L/opt/homebrew/lib \
-    -I/opt/homebrew/include \
-    -I/opt/homebrew/Cellar/ \
-    -L/usr/local/Cellar/ \
-    -g -O2" CXX="g++"
+../configure CXXFLAGS="-arch arm64 -g -O2" \
+    LDFLAGS="-L/opt/homebrew/lib -L/usr/local/lib" \
+    CPPFLAGS="-I/opt/homebrew/include -I/usr/local/include" \
+    CXX="g++" \
+    --enable-python=no \
+    --with-boost=/opt/local/libexec/boost/1.76/lib
+
 
 echo "Building ..."
 make -j$(nproc) && sudo make install
-echo "Setting up libs ..."
-ln -s /usr/local/lib/libboost_date_time.dylib .libs
-ln -s /usr/local/lib/libboost_system.dylib .libs
-ln -s /usr/local/lib/libboost_filesystem.dylib .libs
-ln -s /usr/local/lib/libboost_log.dylib .libs
-ln -s /usr/local/lib/libboost_program_options.dylib .libs
-ln -s /usr/local/lib/libboost_iostreams.dylib .libs
-ln -s /usr/local/lib/libboost_thread.dylib .libs
-ln -s /usr/local/lib/libboost_serialization.dylib .libs
-ln -s /usr/local/lib/libboost_regex.dylib .libs
 
 echo "Done! now you may want to initialize the database with the bootstrap script"

src/bootstrap/bootstrap.cc

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2023 Humanitarian OpenStreetMap Team
+// Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team
 //
 // This file is part of Underpass.
 //
@@ -58,9 +58,9 @@ Bootstrap::allTasksQueries(std::shared_ptr<std::vector<BootstrapTask>> tasks) {
     return queries;
 }
 
-void 
+void
 Bootstrap::start(const underpassconfig::UnderpassConfig &config) {
-    std::cout << "Connecting to the database ..." << std::endl;
+    std::cout << "Connecting to the database ... " << std::endl;
     db = std::make_shared<Pq>();
     if (!db->connect(config.underpass_db_url)) {
         std::cout << "Could not connect to Underpass DB, aborting bootstrapping thread!" << std::endl;
@@ -97,14 +97,14 @@ Bootstrap::start(const underpassconfig::UnderpassConfig &config) {
 
 }
 
-void 
+void
 Bootstrap::processWays() {
 
     std::vector<std::string> tables = {
         QueryRaw::polyTable,
         QueryRaw::lineTable
     };
-    
+
     for (auto table_it = tables.begin(); table_it != tables.end(); ++table_it) {
         std::cout << std::endl << "Processing ways ... ";
         long int total = queryraw->getCount(*table_it);
@@ -138,7 +138,7 @@ Bootstrap::processWays() {
                     std::ref(ways),
                 };
                 std::cout << "\r" << "Processing " << *table_it << ": " << count << "/" << total << " (" << percentage << "%)";
-                
+
                 boost::asio::post(pool, boost::bind(&Bootstrap::threadBootstrapWayTask, this, wayTask));
             }
 
@@ -157,9 +157,9 @@ Bootstrap::processWays() {
 
 }
 
-void 
+void
 Bootstrap::processNodes() {
-    
+
     std::cout << "Processing nodes ... ";
     long int total = queryraw->getCount("nodes");
     long int count = 0;
@@ -205,9 +205,9 @@ Bootstrap::processNodes() {
 
 }
 
-void 
+void
 Bootstrap::processRelations() {
-    
+
     std::cout << "Processing relations ... ";
     long int total = queryraw->getCount("relations");
     long int count = 0;
@@ -348,10 +348,10 @@ Bootstrap::threadBootstrapRelationTask(RelationTask relationTask)
         if (i < relations->size()) {
             auto relation = relations->at(i);
             // relationval->push_back(validator->checkRelation(way, "building"));
-            // Fill the rel_members table
-            // for (auto ref = relation.refs.begin(); ref != relation.refs.end(); ++ref) {
-            //     task.query += "INSERT INTO rel_refs (rel_id, way_id) VALUES (" + std::to_string(rel.id) + "," + std::to_string(*ref) + "); ";
-            // }
+            // Fill the rel_refs table
+            for (auto mit = relation.members.begin(); mit != relation.members.end(); ++mit) {
+                task.query += "INSERT INTO rel_refs (rel_id, way_id) VALUES (" + std::to_string(relation.id) + "," + std::to_string(mit->ref) + "); ";
+            }
             ++processed;
         }
     }

src/data/pq.cc

@@ -177,7 +177,7 @@ Pq::escapedString(const std::string &s)
     return sdb->esc(newstr);
 }
 
-std::string 
+std::string
 Pq::escapedJSON(const std::string &s) {
     std::ostringstream o;
     for (auto c = s.cbegin(); c != s.cend(); c++) {

src/osm/osmchange.cc

@@ -137,6 +137,9 @@ OsmChangeFile::buildGeometriesFromNodeCache() {
             for (auto lit = std::begin(way->refs); lit != std::end(way->refs); ++lit) {
                 boost::geometry::append(way->linestring, nodecache[*lit]);
             }
+            if (way->isClosed()) {
+                way->polygon = { {std::begin(way->linestring), std::end(way->linestring)} };
+            }
         }
     }
 }
@@ -422,7 +425,7 @@ OsmChange::dump(void)
         }
     }
     std::cerr << "Final timestamp: " << to_simple_string(final_entry) << std::endl;
-    
+
 }
 
 void

src/osm/osmchange.hh

@@ -258,8 +258,11 @@ class OsmChangeFile
 
     std::map<long, std::shared_ptr<ChangeStats>> userstats; ///< User statistics for this file
 
-    std::list<std::shared_ptr<OsmChange>> changes; ///< All the changes in this file
-    std::map<double, point_t> nodecache;           ///< Cache nodes across multiple changesets
+    std::list<std::shared_ptr<OsmChange>> changes;      ///< All the changes in this file
+
+    std::map<double, point_t> nodecache;                ///< Cache nodes across multiple changesets
+
+    std::map<long, std::shared_ptr<osmobjects::OsmWay>> waycache; ///< Cache ways across multiple changesets
 
     /// Collect statistics for each user
     std::shared_ptr<std::map<long, std::shared_ptr<ChangeStats>>>

src/osm/osmobjects.hh

@@ -170,7 +170,7 @@ class OsmWay : public OsmObject {
     /// is a linestring
     bool isClosed(void)
     {
-        return !linestring.empty() && boost::geometry::equals(linestring.back(), linestring.front());
+        return (refs.size() > 3 && refs.front() == refs.back());
     };
     /// Return the number of nodes in this way
     int numPoints(void) { return boost::geometry::num_points(linestring); };