From c612ea7c63ebdc58767a11d5015311dbf7008469 Mon Sep 17 00:00:00 2001 From: Rob Savoye Date: Mon, 27 May 2024 14:22:26 -0600 Subject: [PATCH 1/6] fix: timer.hpp got moved, and in the latest Ubuntu, it's now an error --- m4/ax_boost_timer.m4 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/m4/ax_boost_timer.m4 b/m4/ax_boost_timer.m4 index 95c98335..35254128 100644 --- a/m4/ax_boost_timer.m4 +++ b/m4/ax_boost_timer.m4 @@ -67,8 +67,8 @@ AC_DEFUN([AX_BOOST_TIMER], [AC_LANG_PUSH([C++]) CXXFLAGS_SAVE=$CXXFLAGS - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include ]], - [[boost::timer timer;]])], + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include ]], + [[boost::timer::auto_cpu_timer t;]])], ax_cv_boost_timer=yes, ax_cv_boost_timer=no) CXXFLAGS=$CXXFLAGS_SAVE AC_LANG_POP([C++]) From 026ef9d5dd91d596260ae8830f4d76877eae6702 Mon Sep 17 00:00:00 2001 From: Emilio Mariscal Date: Thu, 30 May 2024 17:14:25 -0300 Subject: [PATCH 2/6] Fixes for Python DB API --- python/dbapi/api/db.py | 18 +++++++++--------- python/dbapi/api/raw.py | 6 +++--- python/dbapi/api/rawValidation.py | 9 +++++---- python/dbapi/api/stats.py | 2 ++ python/restapi/models.py | 4 ++-- 5 files changed, 21 insertions(+), 18 deletions(-) diff --git a/python/dbapi/api/db.py b/python/dbapi/api/db.py index 1738c41e..6c1687ae 100644 --- a/python/dbapi/api/db.py +++ b/python/dbapi/api/db.py @@ -58,21 +58,21 @@ async def run(self, query, singleObject = False, asJson=False): if not self.pool: await self.connect() if self.pool: + result = None try: conn = await self.pool.acquire() result = await conn.fetch(query) + data = None if asJson: - if singleObject: - return result[0]['result'] - return result[0]['result'] + data = result[0]['result'] + elif singleObject: + data = result[0] else: - if singleObject: - return result[0] - return result + data = result + await self.pool.release(conn) + return data except Exception as e: print("\n******* \n" + query + "\n******* \n") print(e) return None - finally: - await self.pool.release(conn) - return None + diff --git a/python/dbapi/api/raw.py b/python/dbapi/api/raw.py index e8500434..ca4c2f9c 100644 --- a/python/dbapi/api/raw.py +++ b/python/dbapi/api/raw.py @@ -238,7 +238,7 @@ async def getPolygons( params.table = Table.polygons result = await self.db.run(geoFeaturesQuery(params, asJson), asJson=asJson) if asJson: - return result + return result or {} return deserializeTags(result) # Get line features @@ -250,7 +250,7 @@ async def getLines( params.table = Table.lines result = await self.db.run(geoFeaturesQuery(params, asJson), asJson=asJson) if asJson: - return result + return result or {} return deserializeTags(result) @@ -263,7 +263,7 @@ async def getNodes( params.table = Table.nodes result = await self.db.run(geoFeaturesQuery(params, asJson), asJson=asJson) if asJson: - return result + return result or {} return deserializeTags(result) # Get all (polygon, line, node) features diff --git a/python/dbapi/api/rawValidation.py b/python/dbapi/api/rawValidation.py index eec66a6e..924f5a95 100644 --- a/python/dbapi/api/rawValidation.py +++ b/python/dbapi/api/rawValidation.py @@ -30,7 +30,7 @@ from .filters import tagsQueryFilter, hashtagQueryFilter from .serialization import queryToJSON from .config import RESULTS_PER_PAGE, RESULTS_PER_PAGE_LIST, DEBUG -from .raw import RawFeaturesParamsDTO, ListFeaturesParamsDTO, rawQueryToJSON, listQueryToJSON +from .raw import RawFeaturesParamsDTO, ListFeaturesParamsDTO, rawQueryToJSON, listQueryToJSON, OrderBy from .serialization import deserializeTags import json @@ -148,6 +148,7 @@ def listFeaturesQuery( geoType:GeoType = GeoType[params.table] osmType:OsmType = OsmType[params.table] table:Table = Table[params.table] + orderBy:OrderBy = OrderBy[params.orderBy] query = "( \ SELECT '{type}' as type, \n \ @@ -178,12 +179,12 @@ def listFeaturesQuery( ) if params.area else "", tags=" AND (" + tagsQueryFilter(params.tags, table.value) + ")" if params.tags else "", status=" AND status = '{status}'".format(status=params.status.value) if (params.status) else "", - order=" AND {order} IS NOT NULL ORDER BY {order} DESC LIMIT {limit} OFFSET {offset}" + order=" ORDER BY {order} DESC LIMIT {limit} OFFSET {offset}" .format( - order=params.orderBy.value, + order=orderBy.value, limit=RESULTS_PER_PAGE_LIST, offset=params.page * RESULTS_PER_PAGE_LIST - ) if params.page else "" + ) ).replace("WHERE AND", "WHERE") if asJson: return listQueryToJSON(query, params) diff --git a/python/dbapi/api/stats.py b/python/dbapi/api/stats.py index 9495a091..864fb759 100644 --- a/python/dbapi/api/stats.py +++ b/python/dbapi/api/stats.py @@ -65,6 +65,7 @@ async def getNodesCount( result = await self.db.run(featureCountQuery(params), singleObject=True) if asJson: return json.dumps(dict(result)) + return result async def getLinesCount( self, @@ -75,6 +76,7 @@ async def getLinesCount( result = await self.db.run(featureCountQuery(params), singleObject=True) if asJson: return json.dumps(dict(result)) + return result async def getPolygonsCount( self, diff --git a/python/restapi/models.py b/python/restapi/models.py index 32bc6aeb..e79def91 100644 --- a/python/restapi/models.py +++ b/python/restapi/models.py @@ -32,14 +32,14 @@ class BaseRequest(BaseModel): featureType: str = None class BaseListRequest(BaseRequest): - orderBy: str = None + orderBy: str = "id" page: int = None class BaseRawValidationRequest(BaseRequest): status: str = None class RawValidationListRequest(BaseRawValidationRequest): - orderBy: str = None + orderBy: str = "id" page: int = None class RawRequest(BaseRequest): From 1d6efc8c746fb2fccffbd9f32fff40040b873bdd Mon Sep 17 00:00:00 2001 From: Emilio Mariscal Date: Wed, 5 Jun 2024 18:51:24 +0000 Subject: [PATCH 3/6] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 881ec886..36318451 100644 --- a/README.md +++ b/README.md @@ -9,12 +9,12 @@ It **updates a local copy of the OSM database** in near real-time, and provides ## Demo -We've deployed a rudimentary demo that keeps a database up-to-date for (some country), +We've deployed a basic demo that keeps a database up-to-date for (some country), rendering buildings and highlighting the ones identified as "un-squared": [https://underpass.live](https://underpass.live) -Screenshot 2023-11-22 at 10 32 56 +Screenshot 2024-06-05 at 15 50 45 ## Getting started From 9044edcf1067b141618b33fd24882dcd5fb571b6 Mon Sep 17 00:00:00 2001 From: Emilio Mariscal Date: Wed, 5 Jun 2024 18:52:11 +0000 Subject: [PATCH 4/6] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 36318451..73644d20 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ rendering buildings and highlighting the ones identified as "un-squared": [https://underpass.live](https://underpass.live) -Screenshot 2024-06-05 at 15 50 45 +Screenshot 2024-06-05 at 15 51 57 ## Getting started From c8d22f22c21608236247d1a26ffc0155b0533fac Mon Sep 17 00:00:00 2001 From: Emilio Mariscal Date: Wed, 5 Jun 2024 16:07:34 -0300 Subject: [PATCH 5/6] Replace created_at for closed_at on Python DB API --- python/dbapi/api/raw.py | 18 +++++++++--------- python/dbapi/api/rawValidation.py | 14 +++++++------- python/dbapi/api/stats.py | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/python/dbapi/api/raw.py b/python/dbapi/api/raw.py index ca4c2f9c..86eea875 100644 --- a/python/dbapi/api/raw.py +++ b/python/dbapi/api/raw.py @@ -30,7 +30,7 @@ # Order by class OrderBy(Enum): - createdAt = "created_at" + closedAt = "closed_at" id = "id" timestamp = "timestamp" @@ -73,7 +73,7 @@ def geoFeaturesQuery(params: RawFeaturesParamsDTO, asJson: bool = False): tags, \n \ hashtags, \n \ editor, \n \ - created_at \n \ + closed_at \n \ FROM {table} \n \ LEFT JOIN changesets c ON c.id = {table}.changeset \n \ WHERE{area}{tags}{hashtag}{date} {limit}; \n \ @@ -84,7 +84,7 @@ def geoFeaturesQuery(params: RawFeaturesParamsDTO, asJson: bool = False): .format(area=params.area) if params.area else "", tags=" AND (" + tagsQueryFilter(params.tags, params.table.value) + ") \n" if params.tags else "", hashtag=" AND " + hashtagQueryFilter(params.hashtag, params.table.value) if params.hashtag else "", - date=" AND created_at >= {dateFrom} AND created_at <= {dateTo}\n" + date=" AND closed_at >= {dateFrom} AND closed_at <= {dateTo}\n" .format(dateFrom=params.dateFrom, dateTo=params.dateTo) if params.dateFrom and params.dateTo else "\n", limit=" LIMIT {limit}".format(limit=RESULTS_PER_PAGE) @@ -115,7 +115,7 @@ def listFeaturesQuery( {table}.timestamp, \n \ tags, \n \ {table}.changeset, \n \ - c.created_at \n \ + c.closed_at \n \ FROM {table} \n \ LEFT JOIN changesets c ON c.id = {table}.changeset \n \ WHERE{fromDate}{toDate}{hashtag}{area}{tags}{order} \ @@ -124,8 +124,8 @@ def listFeaturesQuery( type=osmType.value, geotype=geoType.value, table=table.value, - fromDate=" AND created_at >= '{dateFrom}'".format(dateFrom=params.dateFrom) if (params.dateFrom) else "", - toDate=" AND created_at <= '{dateTo}'".format(dateTo=params.dateTo) if (params.dateTo) else "", + fromDate=" AND closed_at >= '{dateFrom}'".format(dateFrom=params.dateFrom) if (params.dateFrom) else "", + toDate=" AND closed_at <= '{dateTo}'".format(dateTo=params.dateTo) if (params.dateTo) else "", hashtag=" AND " + hashtagQueryFilter(params.hashtag, table.value) if params.hashtag else "", area=" AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({area})))', 4326) )" .format( @@ -152,7 +152,7 @@ def listQueryToJSON(query: str, params: ListFeaturesParamsDTO): predata.timestamp, \n \ tags, \n \ predata.changeset, \n \ - predata.created_at as created_at, \n \ + predata.closed_at as closed_at, \n \ lat, \n \ lon \n \ from predata \n \ @@ -163,10 +163,10 @@ def listQueryToJSON(query: str, params: ListFeaturesParamsDTO): ) SELECT jsonb_agg(t_features.feature) as result FROM t_features;" \ .format( query=query, - date="created_at >= '{dateFrom}' AND created_at <= '{dateTo}'" + date="closed_at >= '{dateFrom}' AND closed_at <= '{dateTo}'" .format( dateFrom=params.dateFrom if (params.dateFrom) else "", - dateTo=" AND created_at <= '{dateTo}'".format(dateTo=params.dateTo) if (params.dateTo) else "" + dateTo=" AND closed_at <= '{dateTo}'".format(dateTo=params.dateTo) if (params.dateTo) else "" ) if params.dateFrom and params.dateTo else "", orderBy=" AND {orderBy} IS NOT NULL ORDER BY {orderBy} DESC" .format( diff --git a/python/dbapi/api/rawValidation.py b/python/dbapi/api/rawValidation.py index 924f5a95..abc9f47a 100644 --- a/python/dbapi/api/rawValidation.py +++ b/python/dbapi/api/rawValidation.py @@ -92,8 +92,8 @@ def countQuery( ) \ select count, total from count_validated_features, count_features".format( table=params.table.value, - dateFrom=" AND created_at >= '{dateFrom}'".format(dateFrom=params.dateFrom) if (params.dateFrom) else "", - dateTo=" AND created_at <= '{dateTo}'".format(dateTo=params.dateTo) if (params.dateTo) else "", + dateFrom=" AND closed_at >= '{dateFrom}'".format(dateFrom=params.dateFrom) if (params.dateFrom) else "", + dateTo=" AND closed_at <= '{dateTo}'".format(dateTo=params.dateTo) if (params.dateTo) else "", area=" AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({area})))', 4326) )".format(area=params.area) if params.area else "", tags=" AND (" + tagsQueryFilter(params.tags, params.table.value) + ") \n" if params.tags else "", hashtag=" AND " + hashtagQueryFilter(params.hashtag, params.table.value) if params.hashtag else "", @@ -115,7 +115,7 @@ def geoFeaturesQuery(params: RawValidationFeaturesParamsDTO, asJson: bool = Fals status, \n \ hashtags, \n \ editor, \n \ - created_at \n \ + closed_at \n \ FROM {table} \n \ LEFT JOIN changesets c ON c.id = {table}.changeset \n \ LEFT JOIN validation ON validation.osm_id = {table}.osm_id \ @@ -127,7 +127,7 @@ def geoFeaturesQuery(params: RawValidationFeaturesParamsDTO, asJson: bool = Fals .format(area=params.area) if params.area else "", tags=" AND (" + tagsQueryFilter(params.tags, params.table.value) + ") \n" if params.tags else "", hashtag=" AND " + hashtagQueryFilter(params.hashtag, params.table.value) if params.hashtag else "", - date=" AND created_at >= {dateFrom} AND created_at <= {dateTo}\n" + date=" AND closed_at >= {dateFrom} AND closed_at <= {dateTo}\n" .format(dateFrom=params.dateFrom, dateTo=params.dateTo) if params.dateFrom and params.dateTo else "\n", status=" AND status = '{status}'".format(status=params.status.value) if (params.status) else "", @@ -159,7 +159,7 @@ def listFeaturesQuery( {table}.timestamp, \n \ tags, \n \ {table}.changeset, \n \ - c.created_at, \n \ + c.closed_at, \n \ status \n \ FROM {table} \n \ LEFT JOIN changesets c ON c.id = {table}.changeset \n \ @@ -170,8 +170,8 @@ def listFeaturesQuery( type=osmType.value, geotype=geoType.value, table=table.value, - fromDate=" AND created_at >= '{dateFrom}'".format(dateFrom=params.dateFrom) if (params.dateFrom) else "", - toDate=" AND created_at <= '{dateTo}'".format(dateTo=params.dateTo) if (params.dateTo) else "", + fromDate=" AND closed_at >= '{dateFrom}'".format(dateFrom=params.dateFrom) if (params.dateFrom) else "", + toDate=" AND closed_at <= '{dateTo}'".format(dateTo=params.dateTo) if (params.dateTo) else "", hashtag=" AND " + hashtagQueryFilter(params.hashtag, table.value) if params.hashtag else "", area=" AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({area})))', 4326) )" .format( diff --git a/python/dbapi/api/stats.py b/python/dbapi/api/stats.py index 864fb759..698d361c 100644 --- a/python/dbapi/api/stats.py +++ b/python/dbapi/api/stats.py @@ -44,7 +44,7 @@ def featureCountQuery(params: StatsParamsDTO, asJson: bool = False): .format(area=params.area) if params.area else "", tags=" AND (" + tagsQueryFilter(params.tags, params.table.value) + ") \n" if params.tags else "", hashtag=" AND " + hashtagQueryFilter(params.hashtag, params.table.value) if params.hashtag else "", - date=" AND created_at >= {dateFrom} AND created_at <= {dateTo}\n" + date=" AND closed_at >= {dateFrom} AND closed_at <= {dateTo}\n" .format(dateFrom=params.dateFrom, dateTo=params.dateTo) if params.dateFrom and params.dateTo else "\n" ).replace("WHERE AND", "WHERE") From 224fc2ac04e492e35fe3282623359b39acd2b142 Mon Sep 17 00:00:00 2001 From: Emilio Mariscal Date: Wed, 5 Jun 2024 16:23:16 -0300 Subject: [PATCH 6/6] Fix for order/limit/pagination --- python/dbapi/api/raw.py | 2 +- python/dbapi/api/rawValidation.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/dbapi/api/raw.py b/python/dbapi/api/raw.py index 86eea875..a7e7ae7e 100644 --- a/python/dbapi/api/raw.py +++ b/python/dbapi/api/raw.py @@ -137,7 +137,7 @@ def listFeaturesQuery( order=params.orderBy.value, limit=RESULTS_PER_PAGE_LIST, offset=params.page * RESULTS_PER_PAGE_LIST - ) if params.page else "" + ) if params.page is not None else " LIMIT {limit} OFFSET {offset}" ).replace("WHERE AND", "WHERE") if asJson: return listQueryToJSON(query, params) diff --git a/python/dbapi/api/rawValidation.py b/python/dbapi/api/rawValidation.py index abc9f47a..0ac4d4fb 100644 --- a/python/dbapi/api/rawValidation.py +++ b/python/dbapi/api/rawValidation.py @@ -179,12 +179,12 @@ def listFeaturesQuery( ) if params.area else "", tags=" AND (" + tagsQueryFilter(params.tags, table.value) + ")" if params.tags else "", status=" AND status = '{status}'".format(status=params.status.value) if (params.status) else "", - order=" ORDER BY {order} DESC LIMIT {limit} OFFSET {offset}" + order=" AND {order} IS NOT NULL ORDER BY {order} DESC LIMIT {limit} OFFSET {offset}" .format( order=orderBy.value, limit=RESULTS_PER_PAGE_LIST, offset=params.page * RESULTS_PER_PAGE_LIST - ) + ) if params.page is not None else " LIMIT {limit} OFFSET {offset}" ).replace("WHERE AND", "WHERE") if asJson: return listQueryToJSON(query, params)