diff --git a/Makefile.am b/Makefile.am index 3b2677fb..9ab61234 100644 --- a/Makefile.am +++ b/Makefile.am @@ -153,7 +153,6 @@ endif install-data-hook: $(MKDIR_P) $(DESTDIR)/$(pkglibdir) $(MKDIR_P) /etc/underpass - cp -rvp $(srcdir)/setup/service/underpass.service /etc/systemd/system/ cp -rvp $(srcdir)/config/priority.geojson /etc/underpass/ cp -rvp $(srcdir)/config/replicator /etc/underpass/ cp -rvp $(srcdir)/config/validate /etc/underpass/ @@ -162,6 +161,7 @@ install-data-hook: $(MKDIR_P) $(DESTDIR)/$(docdir) cp -rvp $(srcdir)/docs/*.md $(DESTDIR)/$(docdir) cp -rvp $(srcdir)/setup/service $(DESTDIR)/$(pkglibdir) + cp -rvp $(srcdir)/setup/service/underpass.service /etc/systemd/system/ dist-hook: apidoc $(MKDIR_P) $(DESTDIR)/$(docdir) diff --git a/docs/Dev/debugging.md b/docs/Dev/debugging.md index b109b5bd..e70f9189 100644 --- a/docs/Dev/debugging.md +++ b/docs/Dev/debugging.md @@ -28,7 +28,7 @@ It's also possible to debug Underpass on MacOS. You should use `glibtool` instea `brew install gdb` -Note that gdb requires special privileges to access Mach ports. +Note that gdb requires special privileges to access Mac ports. You will need to codesign the binary. For instructions, see: https://sourceware.org/gdb/wiki/PermissionsDarwin Add the alias in your `~/.bashrc` file: @@ -37,8 +37,11 @@ Add the alias in your `~/.bashrc` file: ### With LLDB +When running Underpass on a MacOS system with an Arm64 architecture, `gdb` might be not available. +You can use `lldb` instead. + `lldb -- src/testsuite/libunderpass.all/yaml-test` -`lldb -- .libs/underpass --bootstrap ` +`lldb -- .libs/underpass --bootstrap` diff --git a/setup/db/underpass.sql b/setup/db/underpass.sql index 0542e21d..3d30dfd4 100644 --- a/setup/db/underpass.sql +++ b/setup/db/underpass.sql @@ -99,13 +99,16 @@ CREATE TABLE IF NOT EXISTS public.relations ( changeset int8, geom public.geometry(Geometry,4326), tags JSONB, - refs int8[], + refs JSONB, timestamp timestamp with time zone, version int, "user" text, uid int8 ); +ALTER TABLE ONLY public.relations + ADD CONSTRAINT relations_pkey PRIMARY KEY (osm_id); + CREATE TABLE IF NOT EXISTS public.way_refs ( way_id int8, node_id int8 @@ -119,9 +122,13 @@ CREATE TABLE IF NOT EXISTS public.rel_refs ( CREATE UNIQUE INDEX nodes_id_idx ON public.nodes (osm_id DESC); CREATE UNIQUE INDEX ways_poly_id_idx ON public.ways_poly (osm_id DESC); CREATE UNIQUE INDEX ways_line_id_idx ON public.ways_line(osm_id DESC); +CREATE UNIQUE INDEX relations_id_idx ON public.relations(osm_id DESC); CREATE INDEX way_refs_node_id_idx ON public.way_refs (node_id); CREATE INDEX way_refs_way_id_idx ON public.way_refs (way_id); +CREATE INDEX rel_refs_rel_id_idx ON public.rel_refs (rel_id); +CREATE INDEX rel_refs_way_id_idx ON public.rel_refs (way_id); + CREATE INDEX nodes_version_idx ON public.nodes (version); CREATE INDEX ways_poly_version_idx ON public.ways_poly (version); CREATE INDEX ways_line_version_idx ON public.ways_line (version); diff --git a/setup/install-macos-arm64.sh b/setup/install-macos-arm64.sh index d67d942e..2404f9c3 100644 --- a/setup/install-macos-arm64.sh +++ b/setup/install-macos-arm64.sh @@ -34,25 +34,15 @@ cd .. echo "Setting up build ..." ./autogen.sh mkdir build && cd build -../configure CXXFLAGS="-arch arm64 \ - -I/usr/local/include \ - -L/opt/homebrew/lib \ - -I/opt/homebrew/include \ - -I/opt/homebrew/Cellar/ \ - -L/usr/local/Cellar/ \ - -g -O2" CXX="g++" +../configure CXXFLAGS="-arch arm64 -g -O2" \ + LDFLAGS="-L/opt/homebrew/lib -L/usr/local/lib" \ + CPPFLAGS="-I/opt/homebrew/include -I/usr/local/include" \ + CXX="g++" \ + --enable-python=no \ + --with-boost=/opt/local/libexec/boost/1.76/lib + echo "Building ..." make -j$(nproc) && sudo make install -echo "Setting up libs ..." -ln -s /usr/local/lib/libboost_date_time.dylib .libs -ln -s /usr/local/lib/libboost_system.dylib .libs -ln -s /usr/local/lib/libboost_filesystem.dylib .libs -ln -s /usr/local/lib/libboost_log.dylib .libs -ln -s /usr/local/lib/libboost_program_options.dylib .libs -ln -s /usr/local/lib/libboost_iostreams.dylib .libs -ln -s /usr/local/lib/libboost_thread.dylib .libs -ln -s /usr/local/lib/libboost_serialization.dylib .libs -ln -s /usr/local/lib/libboost_regex.dylib .libs echo "Done! now you may want to initialize the database with the bootstrap script" diff --git a/src/bootstrap/bootstrap.cc b/src/bootstrap/bootstrap.cc index 2f81f22d..1d84361f 100644 --- a/src/bootstrap/bootstrap.cc +++ b/src/bootstrap/bootstrap.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // @@ -58,9 +58,9 @@ Bootstrap::allTasksQueries(std::shared_ptr> tasks) { return queries; } -void +void Bootstrap::start(const underpassconfig::UnderpassConfig &config) { - std::cout << "Connecting to the database ..." << std::endl; + std::cout << "Connecting to the database ... " << std::endl; db = std::make_shared(); if (!db->connect(config.underpass_db_url)) { std::cout << "Could not connect to Underpass DB, aborting bootstrapping thread!" << std::endl; @@ -97,14 +97,14 @@ Bootstrap::start(const underpassconfig::UnderpassConfig &config) { } -void +void Bootstrap::processWays() { std::vector tables = { QueryRaw::polyTable, QueryRaw::lineTable }; - + for (auto table_it = tables.begin(); table_it != tables.end(); ++table_it) { std::cout << std::endl << "Processing ways ... "; long int total = queryraw->getCount(*table_it); @@ -138,7 +138,7 @@ Bootstrap::processWays() { std::ref(ways), }; std::cout << "\r" << "Processing " << *table_it << ": " << count << "/" << total << " (" << percentage << "%)"; - + boost::asio::post(pool, boost::bind(&Bootstrap::threadBootstrapWayTask, this, wayTask)); } @@ -157,9 +157,9 @@ Bootstrap::processWays() { } -void +void Bootstrap::processNodes() { - + std::cout << "Processing nodes ... "; long int total = queryraw->getCount("nodes"); long int count = 0; @@ -205,9 +205,9 @@ Bootstrap::processNodes() { } -void +void Bootstrap::processRelations() { - + std::cout << "Processing relations ... "; long int total = queryraw->getCount("relations"); long int count = 0; @@ -348,10 +348,10 @@ Bootstrap::threadBootstrapRelationTask(RelationTask relationTask) if (i < relations->size()) { auto relation = relations->at(i); // relationval->push_back(validator->checkRelation(way, "building")); - // Fill the rel_members table - // for (auto ref = relation.refs.begin(); ref != relation.refs.end(); ++ref) { - // task.query += "INSERT INTO rel_refs (rel_id, way_id) VALUES (" + std::to_string(rel.id) + "," + std::to_string(*ref) + "); "; - // } + // Fill the rel_refs table + for (auto mit = relation.members.begin(); mit != relation.members.end(); ++mit) { + task.query += "INSERT INTO rel_refs (rel_id, way_id) VALUES (" + std::to_string(relation.id) + "," + std::to_string(mit->ref) + "); "; + } ++processed; } } diff --git a/src/data/pq.cc b/src/data/pq.cc index 9dbe5435..ef7957f0 100644 --- a/src/data/pq.cc +++ b/src/data/pq.cc @@ -177,7 +177,7 @@ Pq::escapedString(const std::string &s) return sdb->esc(newstr); } -std::string +std::string Pq::escapedJSON(const std::string &s) { std::ostringstream o; for (auto c = s.cbegin(); c != s.cend(); c++) { diff --git a/src/osm/osmchange.cc b/src/osm/osmchange.cc index 76ec8164..f6b7df24 100644 --- a/src/osm/osmchange.cc +++ b/src/osm/osmchange.cc @@ -137,6 +137,9 @@ OsmChangeFile::buildGeometriesFromNodeCache() { for (auto lit = std::begin(way->refs); lit != std::end(way->refs); ++lit) { boost::geometry::append(way->linestring, nodecache[*lit]); } + if (way->isClosed()) { + way->polygon = { {std::begin(way->linestring), std::end(way->linestring)} }; + } } } } @@ -422,7 +425,7 @@ OsmChange::dump(void) } } std::cerr << "Final timestamp: " << to_simple_string(final_entry) << std::endl; - + } void diff --git a/src/osm/osmchange.hh b/src/osm/osmchange.hh index d04defc8..18f186df 100644 --- a/src/osm/osmchange.hh +++ b/src/osm/osmchange.hh @@ -258,8 +258,11 @@ class OsmChangeFile std::map> userstats; ///< User statistics for this file - std::list> changes; ///< All the changes in this file - std::map nodecache; ///< Cache nodes across multiple changesets + std::list> changes; ///< All the changes in this file + + std::map nodecache; ///< Cache nodes across multiple changesets + + std::map> waycache; ///< Cache ways across multiple changesets /// Collect statistics for each user std::shared_ptr>> diff --git a/src/osm/osmobjects.hh b/src/osm/osmobjects.hh index b090c3e6..2c3c6493 100644 --- a/src/osm/osmobjects.hh +++ b/src/osm/osmobjects.hh @@ -170,7 +170,7 @@ class OsmWay : public OsmObject { /// is a linestring bool isClosed(void) { - return !linestring.empty() && boost::geometry::equals(linestring.back(), linestring.front()); + return (refs.size() > 3 && refs.front() == refs.back()); }; /// Return the number of nodes in this way int numPoints(void) { return boost::geometry::num_points(linestring); }; diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index 9decd2ec..5cb53ba2 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -1,5 +1,5 @@ // -// Copyright (c) 2023 Humanitarian OpenStreetMap Team +// Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team // // This file is part of Underpass. // @@ -86,6 +86,72 @@ QueryRaw::buildTagsQuery(std::map tags) const { } } +std::string +buildMembersQuery(std::list members) { + if (members.size() > 0) { + std::string membersStr = "jsonb_build_array("; + int count = 0; + for (auto mit = std::begin(members); mit != std::end(members); ++mit) { + membersStr += "jsonb_build_object("; + std::string member_format = "'%s', '%s',"; + boost::format member_fmt(member_format); + member_fmt % "role"; + member_fmt % mit->role; + membersStr += member_fmt.str(); + member_fmt % "type"; + member_fmt % mit->type; + membersStr += member_fmt.str(); + member_fmt % "ref"; + member_fmt % mit->ref; + membersStr += member_fmt.str(); + membersStr.erase(membersStr.size() - 1); + membersStr += "),"; + } + membersStr.erase(membersStr.size() - 1); + return membersStr += ")"; + } else { + return "null"; + } +} + +std::map parseJSONObjectStr(std::string input) { + std::map obj; + boost::property_tree::ptree pt; + try { + std::istringstream jsonStream(input); + boost::property_tree::read_json(jsonStream, pt); + } catch (const boost::property_tree::json_parser::json_parser_error& e) { + std::cerr << "Error parsing JSON: " << e.what() << std::endl; + return obj; + } + for (const auto& pair : pt) { + obj[pair.first] = pair.second.get_value(); + } + return obj; +} + +std::vector> parseJSONArrayStr(std::string input) { + std::vector> arr; + boost::property_tree::ptree pt; + try { + std::istringstream jsonStream(input); + boost::property_tree::read_json(jsonStream, pt); + } catch (const boost::property_tree::json_parser::json_parser_error& e) { + std::cerr << "Error parsing JSON: " << e.what() << std::endl; + return arr; + } + + for (const auto& item : pt) { + std::map obj; + for (const auto& pair : item.second) { + obj[pair.first] = pair.second.get_value(); + } + arr.push_back(obj); + } + + return arr; +} + std::string QueryRaw::applyChange(const OsmNode &node) const { @@ -159,7 +225,7 @@ QueryRaw::applyChange(const OsmWay &way) const ss << std::setprecision(12) << boost::geometry::wkt(way.linestring); } std::string geostring = ss.str(); - + if (way.refs.size() > 2 && (way.action == osmobjects::create || way.action == osmobjects::modify)) { if ((way.refs.front() != way.refs.back() && way.refs.size() == boost::geometry::num_points(way.linestring)) || @@ -234,8 +300,70 @@ QueryRaw::applyChange(const OsmWay &way) const std::string QueryRaw::applyChange(const OsmRelation &relation) const { + std::string query = ""; + std::stringstream ss; + ss << std::setprecision(12) << boost::geometry::wkt(relation.multipolygon); + // TODO: support for both multipolygon & multilinestring + // ss << std::setprecision(12) << boost::geometry::wkt(relation.multilinestring); + std::string geostring = ss.str(); -} + if (relation.action == osmobjects::create || relation.action == osmobjects::modify) { + + query = "INSERT INTO relations as r (osm_id, tags, refs, geom, timestamp, version, \"user\", uid, changeset) VALUES("; + std::string format = "%d, %s, %s, %s, \'%s\', %d, \'%s\', %d, %d) \ + ON CONFLICT (osm_id) DO UPDATE SET tags = %s, refs = %s, geom = %s, timestamp = \'%s\', version = %d, \"user\" = \'%s\', uid = %d, changeset = %d WHERE r.version <= %d;"; + boost::format fmt(format); + + // osm_id + fmt % relation.id; + + // tags + auto tags = buildTagsQuery(relation.tags); + fmt % tags; + + // refs + auto refs = buildMembersQuery(relation.members); + fmt % refs; + + // geometry + std::string geometry; + geometry = "ST_GeomFromText(\'" + geostring + "\', 4326)"; + fmt % geometry; + + // timestamp + std::string timestamp = to_simple_string(boost::posix_time::microsec_clock::universal_time()); + fmt % timestamp; + // version + fmt % relation.version; + // user + fmt % dbconn->escapedString(relation.user); + // uid + fmt % relation.uid; + // changeset + fmt % relation.changeset; + + // ON CONFLICT + fmt % tags; + fmt % refs; + fmt % geometry; + fmt % timestamp; + fmt % relation.version; + fmt % dbconn->escapedString(relation.user); + fmt % relation.uid; + fmt % relation.changeset; + fmt % relation.version; + + query += fmt.str(); + + for (auto it = std::begin(relation.members); it != std::end(relation.members); ++it) { + query += "INSERT INTO rel_refs (rel_id, way_id) VALUES (" + std::to_string(relation.id) + "," + std::to_string(it->ref) + ");"; + } + + } else if (relation.action == osmobjects::remove) { + query += "DELETE FROM relations where osm_id = " + std::to_string(relation.id) + ";"; + } + + return query;} std::vector arrayStrToVector(std::string &refs_str) { refs_str.erase(0, 1); @@ -249,40 +377,118 @@ std::vector arrayStrToVector(std::string &refs_str) { return refs; } -void QueryRaw::getNodeCache(std::shared_ptr osmchanges, const multipolygon_t &poly) +std::list> +QueryRaw::getRelationsByWaysRefs(std::string &wayIds) const { #ifdef TIMING_DEBUG - boost::timer::auto_cpu_timer timer("getNodeCache(osmchanges): took %w seconds\n"); + boost::timer::auto_cpu_timer timer("getRelationsByWaysRefs(wayIds): took %w seconds\n"); +#endif + // Get all relations that have references to ways + std::list> rels; + + std::string relsQuery = "SELECT distinct(osm_id), refs, version, tags, uid, changeset from rel_refs join relations r on r.osm_id = rel_id where way_id = any(ARRAY[" + wayIds + "])"; + auto rels_result = dbconn->query(relsQuery); + + // Fill vector of OsmRelation objects + for (auto rel_it = rels_result.begin(); rel_it != rels_result.end(); ++rel_it) { + auto rel = std::make_shared(); + rel->id = (*rel_it)[0].as(); + std::string refs_str = (*rel_it)[1].as(); + auto members = parseJSONArrayStr(refs_str); + for (auto mit = members.begin(); mit != members.end(); ++mit) { + rel->addMember(std::stol(mit->at("ref")), osmobjects::osmtype_t::way, mit->at("role")); + } + rel->version = (*rel_it)[2].as(); + auto tags = (*rel_it)[3]; + if (!tags.is_null()) { + auto tags = parseJSONObjectStr((*rel_it)[3].as()); + for (auto const& [key, val] : tags) + { + rel->addTag(key, val); + } + } + auto uid = (*rel_it)[4]; + if (!uid.is_null()) { + rel->uid = (*rel_it)[4].as(); + } + auto changeset = (*rel_it)[5]; + if (!changeset.is_null()) { + rel->changeset = (*rel_it)[5].as(); + } + rels.push_back(rel); + } + return rels; +} + +void +QueryRaw::getWaysByIds(std::string &waysIds, std::map> &waycache) { +#ifdef TIMING_DEBUG + boost::timer::auto_cpu_timer timer("getWaysByIds(waysIds, waycache): took %w seconds\n"); +#endif + // Get all ways that have references to nodes + std::string waysQuery = "SELECT distinct(osm_id), ST_AsText(geom, 4326) from ways_poly wp where osm_id = any(ARRAY[" + waysIds + "])"; + auto ways_result = dbconn->query(waysQuery); + + // Fill vector of OsmWay objects + for (auto way_it = ways_result.begin(); way_it != ways_result.end(); ++way_it) { + auto way = std::make_shared(); + way->id = (*way_it)[0].as(); + boost::geometry::read_wkt((*way_it)[1].as(), way->polygon); + waycache.insert(std::pair(way->id, std::make_shared(*way))); + } +} + +// TODO: divide this function into multiple ones +void QueryRaw::buildGeometries(std::shared_ptr osmchanges, const multipolygon_t &poly) +{ +#ifdef TIMING_DEBUG + boost::timer::auto_cpu_timer timer("buildGeometries(osmchanges, poly): took %w seconds\n"); #endif std::string referencedNodeIds; std::string modifiedNodesIds; + std::string modifiedWaysIds; std::vector removedWays; + std::vector removedRelations; for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); it++) { OsmChange *change = it->get(); for (auto wit = std::begin(change->ways); wit != std::end(change->ways); ++wit) { OsmWay *way = wit->get(); if (way->action != osmobjects::remove) { - // Save referenced nodes for later use + // Save referenced nodes ids for later use for (auto rit = std::begin(way->refs); rit != std::end(way->refs); ++rit) { if (!osmchanges->nodecache.count(*rit)) { referencedNodeIds += std::to_string(*rit) + ","; } } + // Save ways for later use + if (way->isClosed()) { + // Save only ways with a geometry that are inside the priority area + // these are mostly created ways + if (poly.empty() || boost::geometry::within(way->linestring, poly)) { + osmchanges->waycache.insert(std::make_pair(way->id, std::make_shared(*way))); + } + } } else { removedWays.push_back(way->id); } } + // Save modified nodes for later use for (auto nit = std::begin(change->nodes); nit != std::end(change->nodes); ++nit) { OsmNode *node = nit->get(); if (node->action == osmobjects::modify) { - // Get only modified nodes inside priority area - if (boost::geometry::within(node->point, poly)) { + // Get only modified nodes ids inside the priority area + if (poly.empty() || boost::geometry::within(node->point, poly)) { modifiedNodesIds += std::to_string(node->id) + ","; } } } + + for (auto rel_it = std::begin(change->relations); rel_it != std::end(change->relations); ++rel_it) { + OsmRelation *relation = rel_it->get(); + removedRelations.push_back(relation->id); + } } // Add indirectly modified ways to osmchanges @@ -298,9 +504,28 @@ void QueryRaw::getNodeCache(std::shared_ptr osmchanges, const mul referencedNodeIds += std::to_string(*rit) + ","; } } + // If the way is not marked as removed, mark it as modified if (std::find(removedWays.begin(), removedWays.end(), way->id) == removedWays.end()) { way->action = osmobjects::modify; change->ways.push_back(way); + modifiedWaysIds += std::to_string(way->id) + ","; + } + } + osmchanges->changes.push_back(change); + } + + // Add indirectly modified relations to osmchanges + if (modifiedWaysIds.size() > 1) { + modifiedWaysIds.erase(modifiedWaysIds.size() - 1); + auto modifiedRelations = getRelationsByWaysRefs(modifiedWaysIds); + auto change = std::make_shared(none); + for (auto rel_it = modifiedRelations.begin(); rel_it != modifiedRelations.end(); ++rel_it) { + auto relation = std::make_shared(*rel_it->get()); + // If the relation is not marked as removed, mark it as modified + if (std::find(removedRelations.begin(), removedRelations.end(), relation->id) == removedRelations.end()) { + relation->action = osmobjects::modify; + change->relations.push_back(relation); + } } osmchanges->changes.push_back(change); @@ -336,13 +561,102 @@ void QueryRaw::getNodeCache(std::shared_ptr osmchanges, const mul if (way->isClosed()) { way->polygon = { {std::begin(way->linestring), std::end(way->linestring)} }; } + // Save way pointer for later use + if (poly.empty() || boost::geometry::within(way->linestring, poly)) { + if (osmchanges->waycache.count(way->id)) { + osmchanges->waycache.at(way->id)->polygon = way->polygon; + } else { + osmchanges->waycache.insert(std::make_pair(way->id, std::make_shared(*way))); + } + } } } - // Build relation multipolyon geometries - // TODO - + // Filter out all relations that doesn't have at least 1 way in cache + // and are not type=multipolygon + std::string relsForWayCacheIds; + for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); it++) { + OsmChange *change = it->get(); + for (auto rel_it = std::begin(change->relations); rel_it != std::end(change->relations); ++rel_it) { + OsmRelation *relation = rel_it->get(); + if (relation->tags.count("type") && relation->tags.at("type") == "multipolygon") { + bool getWaysForRelation = false; + for (auto mit = relation->members.begin(); mit != relation->members.end(); ++mit) { + if (osmchanges->waycache.count(mit->ref)) { + getWaysForRelation = true; + break; + } + } + if (getWaysForRelation) { + relation->priority = true; + for (auto mit = relation->members.begin(); mit != relation->members.end(); ++mit) { + if (!osmchanges->waycache.count(mit->ref)) { + relsForWayCacheIds += std::to_string(mit->ref) + ","; + } + } + } else { + relation->priority = false; + } + } + } + } + // Get all missing ways geometries for relations + if (relsForWayCacheIds != "") { + relsForWayCacheIds.erase(relsForWayCacheIds.size() - 1); + getWaysByIds(relsForWayCacheIds, osmchanges->waycache); + } + + // Build relation MultiPolyon geometries + for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); it++) { + OsmChange *change = it->get(); + for (auto rel_it = std::begin(change->relations); rel_it != std::end(change->relations); ++rel_it) { + OsmRelation *relation = rel_it->get(); + if (relation->priority) { + bool first = true; + std::string multipolygon_str = "MULTIPOLYGON(("; + std::string innerGeoStr; + bool noWay = false; + for (auto mit = relation->members.begin(); mit != relation->members.end(); ++mit) { + if (!osmchanges->waycache.count(mit->ref)) { + noWay = true; + break; + } + auto way = osmchanges->waycache.at(mit->ref); + if (boost::geometry::num_points(way->linestring) == 0 && + boost::geometry::num_points(way->polygon) == 0 + ) { + noWay = true; + break; + } + std::stringstream ss; + ss << std::setprecision(12) << boost::geometry::wkt(way->polygon); + std::string geometry = ss.str(); + geometry.erase(0,8); + geometry.erase(geometry.size() - 1); + if (first && mit->role == "outer") { + multipolygon_str += geometry + ","; + } else { + if (mit->role == "inner") { + innerGeoStr += geometry + ","; + } else { + multipolygon_str += geometry + ","; + multipolygon_str += innerGeoStr; + innerGeoStr = ""; + } + } + } + if (innerGeoStr.size() > 0) { + multipolygon_str += innerGeoStr; + } + multipolygon_str.erase(multipolygon_str.size() - 1); + multipolygon_str += "))"; + if (!noWay) { + boost::geometry::read_wkt(multipolygon_str, relation->multipolygon); + } + } + } + } } void @@ -377,44 +691,6 @@ QueryRaw::getNodeCacheFromWays(std::shared_ptr> ways, std::m } } -std::map parseJSONObjectStr(std::string input) { - std::map obj; - boost::property_tree::ptree pt; - try { - std::istringstream jsonStream(input); - boost::property_tree::read_json(jsonStream, pt); - } catch (const boost::property_tree::json_parser::json_parser_error& e) { - std::cerr << "Error parsing JSON: " << e.what() << std::endl; - return obj; - } - for (const auto& pair : pt) { - obj[pair.first] = pair.second.get_value(); - } - return obj; -} - -std::vector> parseJSONArrayStr(std::string input) { - std::vector> arr; - boost::property_tree::ptree pt; - try { - std::istringstream jsonStream(input); - boost::property_tree::read_json(jsonStream, pt); - } catch (const boost::property_tree::json_parser::json_parser_error& e) { - std::cerr << "Error parsing JSON: " << e.what() << std::endl; - return arr; - } - - for (const auto& item : pt) { - std::map obj; - for (const auto& pair : item.second) { - obj[pair.first] = pair.second.get_value(); - } - arr.push_back(obj); - } - - return arr; -} - std::list> QueryRaw::getWaysByNodesRefs(std::string &nodeIds) const { @@ -560,7 +836,7 @@ QueryRaw::getWaysFromDBWithoutRefs(long lastid, int pageSize, const std::string } else { waysQuery += ", tags FROM " + tableName + " order by osm_id desc limit " + std::to_string(pageSize) + ";"; } - + auto ways_result = dbconn->query(waysQuery); // Fill vector of OsmWay objects auto ways = std::make_shared>(); diff --git a/src/raw/queryraw.hh b/src/raw/queryraw.hh index 8cabf4cf..8bee5c6c 100644 --- a/src/raw/queryraw.hh +++ b/src/raw/queryraw.hh @@ -66,13 +66,16 @@ class QueryRaw { std::string applyChange(const OsmWay &way) const; /// Build query for processed Relation std::string applyChange(const OsmRelation &relation) const; - /// Get nodes for filling Node cache - void getNodeCache(std::shared_ptr osmchanges, const multipolygon_t &poly); + /// Build all geometries for osmchanges + void buildGeometries(std::shared_ptr osmchanges, const multipolygon_t &poly); /// Get nodes for filling Node cache from ways refs void getNodeCacheFromWays(std::shared_ptr> ways, std::map &nodecache) const; // Get ways by refs std::list> getWaysByNodesRefs(std::string &nodeIds) const; - + // Get ways by ids (used for getting relations geometries) + void getWaysByIds(std::string &relsForWayCacheIds, std::map> &waycache); + // Get relations by referenced ways + std::list> getRelationsByWaysRefs(std::string &wayIds) const; // DB connection std::shared_ptr dbconn; // Get ways count diff --git a/src/replicator/planetreplicator.cc b/src/replicator/planetreplicator.cc index 15ac3fe6..c8dc306b 100644 --- a/src/replicator/planetreplicator.cc +++ b/src/replicator/planetreplicator.cc @@ -182,6 +182,9 @@ std::shared_ptr PlanetReplicator::findRemotePath(const underpassconfi fullurl = "https://" + config.planet_server + "/" + cached; remote->parse(fullurl); remote->updatePath(major, minor, index); + if (!config.silent) { + remote->dump(); + } return remote; }; diff --git a/src/replicator/replication.cc b/src/replicator/replication.cc index 4655fcf4..0cdde75a 100644 --- a/src/replicator/replication.cc +++ b/src/replicator/replication.cc @@ -403,7 +403,6 @@ Planet::downloadFile(const std::string &url, const std::string &destdir_base) return file; } - RequestedFile Planet::readFile(std::string &filespec) { log_debug("Reading cached file: %1%", filespec); @@ -673,7 +672,6 @@ RemoteURL::updatePath(int _major, int _minor, int _index) filespec = parts[0] + "/" + parts[1] + "/" + newpath + suffix; destdir = parts[0] + "/" + parts[1] + "/" + majorfmt.str() + "/" + minorfmt.str(); subpath = newpath; - dump(); } void diff --git a/src/replicator/threads.cc b/src/replicator/threads.cc index 9187ab98..9815e0bf 100644 --- a/src/replicator/threads.cc +++ b/src/replicator/threads.cc @@ -181,6 +181,9 @@ startMonitorChangesets(std::shared_ptr &remote, if (last_task->status == reqfile_t::success || (last_task->status == reqfile_t::remoteNotFound && !caughtUpWithNow)) { remote->Increment(); + if (!config.silent) { + remote->dump(); + } } auto new_remote = std::make_shared(remote->getURL()); new_remote->destdir_base = remote->destdir_base; @@ -219,6 +222,9 @@ startMonitorChangesets(std::shared_ptr &remote, std::stoi(closest.url.substr(4, 3)), std::stoi(closest.url.substr(8, 3)) ); + if (!config.silent) { + remote->dump(); + } cores = 1; delay = std::chrono::seconds{45}; } @@ -309,6 +315,9 @@ startMonitorChanges(std::shared_ptr &remote, if (last_task->status == reqfile_t::success || (last_task->status == reqfile_t::remoteNotFound && !caughtUpWithNow)) { remote->Increment(); + if (!config.silent) { + remote->dump(); + } } auto new_remote = std::make_shared(remote->getURL()); new_remote->destdir_base = remote->destdir_base; @@ -353,6 +362,9 @@ startMonitorChanges(std::shared_ptr &remote, std::stoi(closest.url.substr(4, 3)), std::stoi(closest.url.substr(8, 3)) ); + if (!config.silent) { + remote->dump(); + } concurrentTasks = 1; delay = std::chrono::seconds{45}; } @@ -439,7 +451,7 @@ threadOsmChange(OsmChangeTask osmChangeTask) try { osmchanges->nodecache.clear(); - osmchanges->readXML(changes_xml); + osmchanges->readXML(changes_xml); if (osmchanges->changes.size() > 0) { task.timestamp = osmchanges->changes.back()->final_entry; log_debug("OsmChange final_entry: %1%", task.timestamp); @@ -457,10 +469,13 @@ threadOsmChange(OsmChangeTask osmChangeTask) } } - // Finish filling node cache with nodes referenced in modified - // or created ways and also ways affected by modified nodes + // - Fill node cache with nodes referenced in modified + // or created ways and also ways affected by modified nodes + // - Add indirectly modified ways to osmchanges + // - Build ways geometries using nodecache + // - Build relation multipolyon geometries if (!config->disable_raw) { - queryraw->getNodeCache(osmchanges, poly); + queryraw->buildGeometries(osmchanges, poly); } // Filter data by priority polygon @@ -481,7 +496,7 @@ threadOsmChange(OsmChangeTask osmChangeTask) auto removed_ways = std::make_shared>(); auto removed_relations = std::make_shared>(); auto validation_removals = std::make_shared>(); - + // Raw data and validation if (!config->disable_validation || !config->disable_raw) { for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); ++it) { @@ -526,23 +541,23 @@ threadOsmChange(OsmChangeTask osmChangeTask) } // Relations - // for (auto rit = std::begin(change->relations); rit != std::end(change->relations); ++rit) { - // osmobjects::OsmRelation *relation = rit->get(); + for (auto rit = std::begin(change->relations); rit != std::end(change->relations); ++rit) { + osmobjects::OsmRelation *relation = rit->get(); - // if (relation->action != osmobjects::remove && !relation->priority) { - // continue; - // } + if (relation->action != osmobjects::remove && !relation->priority) { + continue; + } - // // Remove deleted relations from validation table - // if (!config->disable_validation && relation->action == osmobjects::remove) { - // removed_relations->push_back(relation->id); - // } + // Remove deleted relations from validation table + if (!config->disable_validation && relation->action == osmobjects::remove) { + removed_relations->push_back(relation->id); + } - // // Update relations, ignore new ones outside priority area - // if (!config->disable_raw) { - // task.query += queryraw->applyChange(*relation); - // } - // } + // Update relations, ignore new ones outside priority area + if (!config->disable_raw) { + // task.query += queryraw->applyChange(*relation); + } + } } } @@ -565,7 +580,7 @@ threadOsmChange(OsmChangeTask osmChangeTask) task.query += queryvalidate->updateValidation(validation_removals); task.query += queryvalidate->updateValidation(removed_nodes); task.query += queryvalidate->updateValidation(removed_ways); - // task.query += queryvalidate->updateValidation(removed_relations); + task.query += queryvalidate->updateValidation(removed_relations); } diff --git a/src/stats/querystats.cc b/src/stats/querystats.cc index 30829923..437c53e0 100644 --- a/src/stats/querystats.cc +++ b/src/stats/querystats.cc @@ -271,7 +271,7 @@ QueryStats::applyChange(const changesets::ChangeSet &change) const } query += ", bbox=" + bbox.substr(2) + ")'));"; - + return query; } diff --git a/src/testsuite/libunderpass.all/Makefile.am b/src/testsuite/libunderpass.all/Makefile.am index f52ee79e..216e26dd 100644 --- a/src/testsuite/libunderpass.all/Makefile.am +++ b/src/testsuite/libunderpass.all/Makefile.am @@ -31,6 +31,7 @@ check_PROGRAMS = \ stats-test \ val-test \ val-unsquared-test \ + raw-test \ test-playground TOPSRC := $(shell cd $(top_srcdir) && pwd)/src @@ -82,6 +83,11 @@ val_unsquared_test_LDFLAGS = -L../.. val_unsquared_test_CPPFLAGS = -DDATADIR=\"$(TOPSRC)\" -I$(TOPSRC) val_unsquared_test_LDADD = -lpqxx -lunderpass $(BOOST_LIBS) ../../validate/defaultvalidation.lo ../../validate/geospatial.lo ../../validate/semantic.lo +raw_test_SOURCES = raw-test.cc +raw_test_LDFLAGS = -L../.. +raw_test_CPPFLAGS = -DDATADIR=\"$(TOPSRC)\" -I$(TOPSRC) +raw_test_LDADD = -lpqxx -lunderpass $(BOOST_LIBS) + # Test the replication classes #replication_test_SOURCES = replication-test.cc #replication_test_LDFLAGS = -L../.. diff --git a/src/testsuite/libunderpass.all/geo-test.cc b/src/testsuite/libunderpass.all/geo-test.cc index fc63ccc9..b90f1f21 100644 --- a/src/testsuite/libunderpass.all/geo-test.cc +++ b/src/testsuite/libunderpass.all/geo-test.cc @@ -49,7 +49,7 @@ main(int argc, char* argv[]) dbglogfile.setWriteDisk(true); dbglogfile.setLogFilename("geo-test.log"); dbglogfile.setVerbosity(3); - + TestGU tgu; std::string test_data_dir(DATADIR); @@ -59,15 +59,15 @@ main(int argc, char* argv[]) } else { runtest.fail("Read file with bad relative path"); return 1; - } + } if (tgu.readFile("/etc/underpass/priority.geojson")) { runtest.pass("Read file with absolute path"); } else { runtest.fail("Read file with absolute path"); return 1; - } - + } + }; // local Variables: diff --git a/src/testsuite/libunderpass.all/planetreplicator-test.cc b/src/testsuite/libunderpass.all/planetreplicator-test.cc index 7372c587..8997eb87 100644 --- a/src/testsuite/libunderpass.all/planetreplicator-test.cc +++ b/src/testsuite/libunderpass.all/planetreplicator-test.cc @@ -55,7 +55,7 @@ void testPath(underpassconfig::UnderpassConfig config) { TestCO change; if (boost::filesystem::exists(osmchange->filespec)) { change.readChanges(osmchange->filespec); - } else { + } else { TestPlanet planet; auto data = planet.downloadFile(osmchange->getURL()).data; auto xml = planet.processData(osmchange->filespec, *data); @@ -128,7 +128,7 @@ main(int argc, char *argv[]) { } } } - + } // local Variables: diff --git a/src/testsuite/libunderpass.all/raw-test.cc b/src/testsuite/libunderpass.all/raw-test.cc new file mode 100644 index 00000000..4260dca2 --- /dev/null +++ b/src/testsuite/libunderpass.all/raw-test.cc @@ -0,0 +1,212 @@ +// +// Copyright (c) 2024 Humanitarian OpenStreetMap Team +// +// This file is part of Underpass. +// +// Underpass is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Underpass is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Underpass. If not, see . +// + +#include +#include +#include "utils/log.hh" +#include "osm/osmobjects.hh" +#include "raw/queryraw.hh" +#include +#include "replicator/replication.hh" +#include + +using namespace replication; +using namespace logger; +using namespace queryraw; + +TestState runtest; +class TestOsmChange : public osmchange::OsmChangeFile {}; + +class TestPlanet : public Planet { + public: + TestPlanet() = default; + + //! Clear the test DB and fill it with with initial test data + bool init_test_case(const std::string &dbconn) + { + source_tree_root = getenv("UNDERPASS_SOURCE_TREE_ROOT") + ? getenv("UNDERPASS_SOURCE_TREE_ROOT") + : "../"; + + const std::string test_replication_db_name{"underpass_test"}; + { + pqxx::connection conn{dbconn + " dbname=template1"}; + pqxx::nontransaction worker{conn}; + worker.exec0("DROP DATABASE IF EXISTS " + test_replication_db_name); + worker.exec0("CREATE DATABASE " + test_replication_db_name); + worker.commit(); + } + + { + pqxx::connection conn{dbconn + " dbname=" + test_replication_db_name}; + pqxx::nontransaction worker{conn}; + worker.exec0("CREATE EXTENSION postgis"); + worker.exec0("CREATE EXTENSION hstore"); + + // Create schema + const auto schema_path{source_tree_root + "setup/db/underpass.sql"}; + std::ifstream schema_definition(schema_path); + std::string sql((std::istreambuf_iterator(schema_definition)), + std::istreambuf_iterator()); + + assert(!sql.empty()); + worker.exec0(sql); + } + + return true; + }; + + std::string source_tree_root; +}; + +bool processFile(const std::string &filename, std::shared_ptr &db) { + auto queryraw = std::make_shared(db); + auto osmchanges = std::make_shared(); + std::string destdir_base = DATADIR; + multipolygon_t poly; + osmchanges->readChanges(destdir_base + "/testsuite/testdata/raw/" + filename); + queryraw->buildGeometries(osmchanges, poly); + std::string rawquery; + + for (auto it = std::begin(osmchanges->changes); it != std::end(osmchanges->changes); ++it) { + osmchange::OsmChange *change = it->get(); + // Nodes + for (auto nit = std::begin(change->nodes); nit != std::end(change->nodes); ++nit) { + osmobjects::OsmNode *node = nit->get(); + rawquery += queryraw->applyChange(*node); + } + // Ways + for (auto wit = std::begin(change->ways); wit != std::end(change->ways); ++wit) { + osmobjects::OsmWay *way = wit->get(); + rawquery += queryraw->applyChange(*way); + } + // Relations + for (auto rit = std::begin(change->relations); rit != std::end(change->relations); ++rit) { + osmobjects::OsmRelation *relation = rit->get(); + rawquery += queryraw->applyChange(*relation); + } + } + db->query(rawquery); +} + +const std::map expectedGeometries = { + {101874, "POLYGON((21.726001473 4.62042952837,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.726001473 4.62042952837))"}, + {101875, "POLYGON((21.726001473 4.62042952837,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.726001473 4.62042952837))"}, + {101875-2, "POLYGON((21.72600148 4.62042953,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.72600148 4.62042953))"}, + {211766, "MULTIPOLYGON(((21.72600148 4.62042953,21.726086573 4.62042742837,21.726084973 4.62036492836,21.725999873 4.62036702836,21.72600148 4.62042953),(21.7260170728 4.62041343508,21.7260713875 4.62041326798,21.7260708846 4.62037684165,21.7260165699 4.62038035061,21.7260170728 4.62041343508)))"}, + {211766-2, "MULTIPOLYGON(((21.72600148 4.62042953,21.726086573 4.62042742837,21.7260807753 4.62037032501,21.725999873 4.62036702836,21.72600148 4.62042953),(21.7260170728 4.62041343508,21.7260713875 4.62041326798,21.7260708846 4.62037684165,21.7260165699 4.62038035061,21.7260170728 4.62041343508)))"} +}; + +std::string +getWKT(const polygon_t &polygon) { + std::stringstream ss; + ss << std::setprecision(12) << boost::geometry::wkt(polygon); + return ss.str(); +} + +std::string +getWKTFromDB(const std::string &table, const long id, std::shared_ptr &db) { + auto result = db->query("SELECT ST_AsText(geom, 4326) from relations where osm_id=" + std::to_string(id)); + for (auto r_it = result.begin(); r_it != result.end(); ++r_it) { + return (*r_it)[0].as(); + } +} + +int +main(int argc, char *argv[]) +{ + logger::LogFile &dbglogfile = logger::LogFile::getDefaultInstance(); + dbglogfile.setWriteDisk(true); + dbglogfile.setLogFilename("raw-test.log"); + dbglogfile.setVerbosity(3); + + const std::string dbconn{getenv("UNDERPASS_TEST_DB_CONN") + ? getenv("UNDERPASS_TEST_DB_CONN") + : "user=underpass_test host=localhost password=underpass_test"}; + + TestPlanet test_planet; + test_planet.init_test_case(dbconn); + auto db = std::make_shared(); + + if (db->connect(dbconn + " dbname=underpass_test")) { + auto queryraw = std::make_shared(db); + std::map> waycache; + std::string waysIds; + + processFile("raw-case-1.osc", db); + processFile("raw-case-2.osc", db); + + for (auto const& x : expectedGeometries) { + waysIds += std::to_string(x.first) + ","; + } + waysIds.erase(waysIds.size() - 1); + + queryraw->getWaysByIds(waysIds, waycache); + + // 4 created Nodes, 1 created Way (same changeset) + if ( getWKT(waycache.at(101874)->polygon).compare(expectedGeometries.at(101874)) == 0) { + runtest.pass("4 created Nodes, 1 created Ways (same changeset)"); + } else { + runtest.fail("4 created Nodes, 1 created Ways (same changeset)"); + return 1; + } + + // 1 created Way, 4 existing Nodes (different changeset) + if ( getWKT(waycache.at(101875)->polygon).compare(expectedGeometries.at(101875)) == 0) { + runtest.pass("1 created Way, 4 existing Nodes (different changesets)"); + } else { + runtest.fail("1 created Way, 4 existing Nodes (different changesets)"); + return 1; + } + + // 1 modified node, indirectly modify other existing ways + processFile("raw-case-3.osc", db); + waycache.erase(101875); + queryraw->getWaysByIds(waysIds, waycache); + + if ( getWKT(waycache.at(101875)->polygon).compare(expectedGeometries.at(101875-2)) == 0) { + runtest.pass("1 modified Node, indirectly modify other existing Ways (different changesets)"); + } else { + runtest.fail("1 modified Node, indirectly modify other existing Ways (different changesets)"); + return 1; + } + + // 1 created Relation referencing 1 created Way and 1 existing Way + processFile("raw-case-4.osc", db); + if ( getWKTFromDB("relations", 211766, db).compare(expectedGeometries.at(211766)) == 0) { + runtest.pass("1 created Relation referencing 1 created Way and 1 existing Way (different changesets)"); + } else { + runtest.fail("1 created Relation referencing 1 created Way and 1 existing Way (different changesets)"); + return 1; + } + + // 1 modified Node, indirectly modify other existing Ways and 1 Relation + processFile("raw-case-5.osc", db); + + if ( getWKTFromDB("relations", 211766, db).compare(expectedGeometries.at(211766-2)) == 0) { + runtest.pass("1 modified Node, indirectly modify other existing Ways and 1 Relation (different changesets)"); + } else { + runtest.fail("1 modified Node, indirectly modify other existing Ways and 1 Relation (different changesets)"); + return 1; + } + } + + + +} \ No newline at end of file diff --git a/src/testsuite/libunderpass.all/under-test.cc b/src/testsuite/libunderpass.all/under-test.cc index e063a17a..84ab26d2 100644 --- a/src/testsuite/libunderpass.all/under-test.cc +++ b/src/testsuite/libunderpass.all/under-test.cc @@ -46,9 +46,9 @@ int main(int argc, char* argv[]) { std::string basedir = DATADIR; - + Underpass under("underpass"); - + if (under.sdb->is_open()) { runtest.pass("Underpass::connect"); } else { diff --git a/src/testsuite/libunderpass.all/val-test.cc b/src/testsuite/libunderpass.all/val-test.cc index 301ea5ee..9c1c7207 100644 --- a/src/testsuite/libunderpass.all/val-test.cc +++ b/src/testsuite/libunderpass.all/val-test.cc @@ -144,9 +144,9 @@ test_semantic(std::shared_ptr &plugin) { node.addTag("building", "yes "); status = plugin->checkNode(node, "building"); if (status->hasStatus(badvalue)) { - runtest.pass("Validate::checkNode(single quotes) [semantic building]"); + runtest.pass("Validate::checkNode(extra space) [semantic building]"); } else { - runtest.fail("Validate::checkNode(single quotes) [semantic building]"); + runtest.fail("Validate::checkNode(extra space) [semantic building]"); return 1; } @@ -166,6 +166,7 @@ test_semantic(std::shared_ptr &plugin) { // Has valid tags, but it's incomplete node_place.addTag("place", "city"); status = plugin->checkNode(node_place, "place"); + if (!status->hasStatus(badvalue) && status->hasStatus(incomplete)) { runtest.pass("Validate::checkNode(incomplete but correct tagging) [semantic place]"); } else { diff --git a/src/testsuite/libunderpass.all/yaml-test.cc b/src/testsuite/libunderpass.all/yaml-test.cc index 4538daf4..8eab7fd2 100644 --- a/src/testsuite/libunderpass.all/yaml-test.cc +++ b/src/testsuite/libunderpass.all/yaml-test.cc @@ -46,7 +46,7 @@ main(int argc, char *argv[]) std::string filespec = DATADIR; filespec += "/testsuite/libunderpass.all/test.yaml"; yaml.read(filespec); - + if (yaml.get("tags").children.size() > 0) { runtest.pass("Yaml::get().children"); } else { diff --git a/src/testsuite/testdata/raw/raw-case-1.osc b/src/testsuite/testdata/raw/raw-case-1.osc new file mode 100644 index 00000000..d8adf181 --- /dev/null +++ b/src/testsuite/testdata/raw/raw-case-1.osc @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/testsuite/testdata/raw/raw-case-2.osc b/src/testsuite/testdata/raw/raw-case-2.osc new file mode 100644 index 00000000..893ab9d0 --- /dev/null +++ b/src/testsuite/testdata/raw/raw-case-2.osc @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/src/testsuite/testdata/raw/raw-case-3.osc b/src/testsuite/testdata/raw/raw-case-3.osc new file mode 100644 index 00000000..2d411bfb --- /dev/null +++ b/src/testsuite/testdata/raw/raw-case-3.osc @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/src/testsuite/testdata/raw/raw-case-4.osc b/src/testsuite/testdata/raw/raw-case-4.osc new file mode 100644 index 00000000..1c744f9e --- /dev/null +++ b/src/testsuite/testdata/raw/raw-case-4.osc @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/testsuite/testdata/raw/raw-case-5.osc b/src/testsuite/testdata/raw/raw-case-5.osc new file mode 100644 index 00000000..f6d041a6 --- /dev/null +++ b/src/testsuite/testdata/raw/raw-case-5.osc @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/src/testsuite/testdata/validation/config/place.yaml b/src/testsuite/testdata/validation/config/place.yaml index cd83ead7..1efc1f98 100644 --- a/src/testsuite/testdata/validation/config/place.yaml +++ b/src/testsuite/testdata/validation/config/place.yaml @@ -1,4 +1,10 @@ +config: + - badvalue: + - yes + - incomplete: + - yes + required_tags: - name diff --git a/src/underpass.cc b/src/underpass.cc index 723db754..ee74c60a 100644 --- a/src/underpass.cc +++ b/src/underpass.cc @@ -124,7 +124,8 @@ main(int argc, char *argv[]) ("disable-validation", "Disable validation") ("disable-raw", "Disable raw OSM data") ("norefs", "Disable refs (useful for non OSM data)") - ("bootstrap", "Bootstrap data tables"); + ("bootstrap", "Bootstrap data tables") + ("silent", "Silent"); // clang-format on opts::store(opts::command_line_parser(argc, argv).options(desc).positional(p).run(), vm); @@ -135,6 +136,7 @@ main(int argc, char *argv[]) return 1; } + // Data processing if (vm.count("norefs")) { config.norefs = true; } @@ -160,6 +162,10 @@ main(int argc, char *argv[]) config.destdir_base = vm["destdir_base"].as(); } + if (vm.count("silent")) { + config.silent = true; + } + // Concurrency if (vm.count("concurrency")) { const auto concurrency = vm["concurrency"].as(); @@ -176,7 +182,7 @@ main(int argc, char *argv[]) } else { config.concurrency = std::thread::hardware_concurrency(); } - + if (vm.count("timestamp") || vm.count("url")) { // Planet server if (vm.count("planet")) { @@ -307,6 +313,9 @@ main(int argc, char *argv[]) std::vector parts; boost::split(parts, vm["changeseturl"].as(), boost::is_any_of("/")); changeset->updatePath(stoi(parts[0]),stoi(parts[1]),stoi(parts[2])); + if (!config.silent) { + changeset->dump(); + } } if (!vm.count("osmchanges")) { multipolygon_t * oscboundary = &poly; @@ -326,7 +335,7 @@ main(int argc, char *argv[]) exit(0); } - + if (vm.count("bootstrap")){ std::thread bootstrapThread; std::cout << "Starting bootstrapping process ..." << std::endl; diff --git a/src/underpassconfig.hh b/src/underpassconfig.hh index 85f7abf7..5fd49fe8 100644 --- a/src/underpassconfig.hh +++ b/src/underpassconfig.hh @@ -168,6 +168,7 @@ struct UnderpassConfig { bool disable_stats = false; bool disable_raw = false; bool norefs = false; + bool silent = false; /// /// \brief getPlanetServer returns either the command line supplied planet server diff --git a/src/utils/geoutil.cc b/src/utils/geoutil.cc index 63c07587..a6b3d526 100644 --- a/src/utils/geoutil.cc +++ b/src/utils/geoutil.cc @@ -32,19 +32,19 @@ #include #include #include -#include +// #include #include #include #include #include -#include "boost/date_time/posix_time/posix_time.hpp" -#include -using namespace boost::posix_time; -using namespace boost::gregorian; +// #include "boost/date_time/posix_time/posix_time.hpp" +// #include +// using namespace boost::posix_time; +// using namespace boost::gregorian; #include "utils/geoutil.hh" -#include "osm/changeset.hh" +// #include "osm/changeset.hh" #include #include @@ -60,7 +60,7 @@ GeoUtil::readFile(const std::string &filespec) if (!std::filesystem::exists(boundary_file)) { log_error("File not found: %1%", boundary_file); return false; - } + } log_debug("Opening geo data file: %1%", boundary_file); std::string foo = boundary_file.string(); GDALDataset *poDS = (GDALDataset *)GDALOpenEx(foo.c_str(), GDAL_OF_VECTOR, NULL, NULL, NULL); diff --git a/src/utils/geoutil.hh b/src/utils/geoutil.hh index b94fbbd8..21a08157 100644 --- a/src/utils/geoutil.hh +++ b/src/utils/geoutil.hh @@ -37,12 +37,12 @@ #include #include #include -#include +// #include -#include -#include "boost/date_time/posix_time/posix_time.hpp" -using namespace boost::posix_time; -using namespace boost::gregorian; +// #include +// #include "boost/date_time/posix_time/posix_time.hpp" +// using namespace boost::posix_time; +// using namespace boost::gregorian; #include #include diff --git a/src/validate/geospatial.cc b/src/validate/geospatial.cc index 7a216679..81a2f992 100644 --- a/src/validate/geospatial.cc +++ b/src/validate/geospatial.cc @@ -56,7 +56,7 @@ Geospatial::checkWay(const osmobjects::OsmWay &way, const std::string &type, yam } auto config = tests.get("config"); - bool check_badgeom = config.get_value("badgeom") == "yes"; + bool check_badgeom = config.get_value("badgeom") == "yes"; // bool check_overlapping = config.get_value("overlapping") == "yes"; // bool check_duplicate = config.get_value("duplicate") == "yes"; @@ -96,7 +96,7 @@ Geospatial::checkWay(const osmobjects::OsmWay &way, const std::string &type, yam return status; } -bool +bool Geospatial::overlaps(const std::list> &allways, osmobjects::OsmWay &way) { #ifdef TIMING_DEBUG_X boost::timer::auto_cpu_timer timer("validate::overlaps: took %w seconds\n"); @@ -116,7 +116,7 @@ Geospatial::overlaps(const std::list> &allwa return false; } -bool +bool Geospatial::duplicate(const std::list> &allways, osmobjects::OsmWay &way) { #ifdef TIMING_DEBUG_X boost::timer::auto_cpu_timer timer("validate::duplicate: took %w seconds\n"); @@ -143,7 +143,7 @@ Geospatial::duplicate(const std::list> &allw return false; } -bool +bool Geospatial::unsquared( const linestring_t &way, double min_angle, diff --git a/src/validate/semantic.cc b/src/validate/semantic.cc index 91f8581c..12504249 100644 --- a/src/validate/semantic.cc +++ b/src/validate/semantic.cc @@ -183,7 +183,7 @@ Semantic::checkWay(const osmobjects::OsmWay &way, const std::string &type, yaml: yaml::Node required_tags; if (check_incomplete) { required_tags = tests.get("required_tags"); - } + } // List of valid tags to be validated yaml::Node tags; @@ -237,7 +237,7 @@ Semantic::checkRelation(const osmobjects::OsmRelation &relation, const std::stri yaml::Node required_tags; if (check_incomplete) { required_tags = tests.get("required_tags"); - } + } // List of valid tags to be validated yaml::Node tags; diff --git a/src/validate/validate.hh b/src/validate/validate.hh index d3ed80f2..1a216941 100644 --- a/src/validate/validate.hh +++ b/src/validate/validate.hh @@ -200,7 +200,6 @@ class BOOST_SYMBOL_VISIBLE Validate { } virtual ~Validate(void){}; - // Validate(std::vector> &changes) {}; virtual std::shared_ptr checkNode(const osmobjects::OsmNode &node, const std::string &type) = 0; virtual std::shared_ptr checkWay(const osmobjects::OsmWay &way, const std::string &type) = 0; diff --git a/src/wrappers/python.cc b/src/wrappers/python.cc index 7bc666d3..eff00f57 100644 --- a/src/wrappers/python.cc +++ b/src/wrappers/python.cc @@ -108,7 +108,7 @@ std::string dumpJSON(ValidateStatus& self) { BOOST_PYTHON_MODULE(underpass) { - // + // // using namespace defaultvalidation; // class_("Validate") // // .def("checkTag", &DefaultValidation::checkTag)