fix: retry_max_url_attempt

kshitij-raj-sharma · Dec 7, 2023 · 09d2e5e · 09d2e5e
1 parent fb970df
commit 09d2e5e
Show file tree

Hide file tree

Showing 17 changed files with 32 additions and 19 deletions.
diff --git a/.github/workflows/daily_Nepal.yml b/.github/workflows/daily_Nepal.yml
@@ -35,7 +35,7 @@ jobs:
           OSM_USERNAME: ${{ secrets.OSM_USERNAME }}
           OSM_PASSWORD: ${{ secrets.OSM_PASSWORD }}
         run: |
-          osmsg --url "http://download.geofabrik.de/asia/nepal-updates" --username '${{ secrets.OSM_USERNAME }}' --password '${{ secrets.OSM_PASSWORD }}' --format image csv --timezone Nepal --last_day --tags 'building' 'highway' 'waterway' 'amenity' --name stats --read_from_metadata stats/Nepal/Daily/stats_metadata.json --all_tags --changeset  --meta --force
+          osmsg --url "http://download.geofabrik.de/asia/nepal-updates" --username '${{ secrets.OSM_USERNAME }}' --password '${{ secrets.OSM_PASSWORD }}' --format image csv --timezone Nepal --last_day --tags 'building' 'highway' 'waterway' 'amenity' --name stats --read_from_metadata stats/Nepal/Daily/stats_metadata.json --all_tags --changeset  --meta --temp --force
         # run: |
         #   osmsg --url "https://planet.openstreetmap.org/replication/minute" --format image csv --timezone Nepal --tags building highway waterway amenity --name stats --all_tags --last_day --country Nepal --name stats --read_from_metadata stats/Nepal/Daily/stats_metadata.json
 

diff --git a/.github/workflows/daily_elinino2023.yml b/.github/workflows/daily_elinino2023.yml
@@ -24,7 +24,7 @@ jobs:
           pip install -r requirements.txt
       - name: Generate Daily Hashtag elinino2023 Stats
         run: |
-          osmsg --format csv image --tags 'building' 'highway' 'waterway' 'amenity' --name stats --all_tags --hashtags fao_swalim fao_rtea elinino2023 --charts --summary --length highway --update --name stats/elinino2023/Daily/stats --force
+          osmsg --format csv image --tags 'building' 'highway' 'waterway' 'amenity' --name stats --all_tags --hashtags fao_swalim fao_rtea elinino2023 --charts --summary --length highway --update --name stats/elinino2023/Daily/stats --force --temp
 
       - name: add stats to git
         continue-on-error: true

diff --git a/.github/workflows/daily_field_mappers.yml b/.github/workflows/daily_field_mappers.yml
@@ -30,7 +30,7 @@ jobs:
           OSM_USERNAME: ${{ secrets.OSM_USERNAME }}
           OSM_PASSWORD: ${{ secrets.OSM_PASSWORD }}
         run: |
-          osmsg --format csv image --name stats --tags amenity --all_tags --charts --summary --update --name stats/fieldmappers/Daily/stats --force --field_mappers
+          osmsg --format csv image --name stats --tags amenity --all_tags --charts --summary --update --name stats/fieldmappers/Daily/stats --force --field_mappers --temp
 
       - name: add stats to git
         continue-on-error: true

diff --git a/.github/workflows/daily_global.yml b/.github/workflows/daily_global.yml
@@ -30,7 +30,7 @@ jobs:
           OSM_USERNAME: ${{ secrets.OSM_USERNAME }}
           OSM_PASSWORD: ${{ secrets.OSM_PASSWORD }}
         run: |
-          osmsg  --url "https://planet.openstreetmap.org/replication/day" --format csv image --last_day --tags 'building' 'highway' 'waterway' 'amenity' --name stats/Global/Daily/stats --all_tags --read_from_metadata stats/Global/Daily/stats_metadata.json --changeset --summary --charts  --meta --force
+          osmsg  --url "https://planet.openstreetmap.org/replication/day" --format csv image --last_day --tags 'building' 'highway' 'waterway' 'amenity' --name stats/Global/Daily/stats --all_tags --read_from_metadata stats/Global/Daily/stats_metadata.json --changeset --summary --charts  --meta --force --temp
 
       - name: add stats to git
         continue-on-error: true

diff --git a/.github/workflows/daily_hotosm.yml b/.github/workflows/daily_hotosm.yml
@@ -27,7 +27,7 @@ jobs:
 
       - name: Generate Daily Hashtag Stats
         run: |
-          osmsg  --url "https://planet.openstreetmap.org/replication/day" --format csv image --last_day --tags 'building' 'highway' 'waterway' 'amenity' --name stats/hotosm/Daily/stats --all_tags --read_from_metadata stats/hotosm/Daily/stats_metadata.json --hashtags hotosm --summary --charts  --meta --force
+          osmsg  --url "https://planet.openstreetmap.org/replication/day" --format csv image --last_day --tags 'building' 'highway' 'waterway' 'amenity' --name stats/hotosm/Daily/stats --all_tags --read_from_metadata stats/hotosm/Daily/stats_metadata.json --hashtags hotosm --summary --charts  --meta --force --temp
 
       - name: add stats to git
         continue-on-error: true

diff --git a/.github/workflows/daily_ogasiapacific.yml b/.github/workflows/daily_ogasiapacific.yml
@@ -29,7 +29,7 @@ jobs:
           pip install -r requirements.txt
       - name: Generate Daily Hashtag OGAsiaPacific Stats
         run: |
-          osmsg --format csv image --tags 'building' 'highway' 'waterway' 'amenity' --name stats --all_tags --hashtags OGAsiaPacific --charts --summary --length highway --update --name stats/OGAsiaPacific/Daily/stats --force
+          osmsg --format csv image --tags 'building' 'highway' 'waterway' 'amenity' --name stats --all_tags --hashtags OGAsiaPacific --charts --summary --length highway --update --name stats/OGAsiaPacific/Daily/stats --temp --force
 
       - name: add stats to git
         continue-on-error: true

diff --git a/.github/workflows/daily_slsibhutan.yml b/.github/workflows/daily_slsibhutan.yml
@@ -26,7 +26,7 @@ jobs:
           pip install -r requirements.txt
       - name: Generate Daily Hashtag slsibhutan Stats
         run: |
-          osmsg --format csv image --tags 'building' 'highway' 'waterway' 'amenity' --name stats --all_tags --hashtags slsibhutan --charts --summary --length highway --update --name stats/slsibhutan/Daily/stats --force
+          osmsg --format csv image --tags 'building' 'highway' 'waterway' 'amenity' --name stats --all_tags --hashtags slsibhutan --charts --summary --length highway --update --name stats/slsibhutan/Daily/stats --temp --force
       - name: add stats to git
         continue-on-error: true
         run: |

diff --git a/.github/workflows/daily_smforst.yml b/.github/workflows/daily_smforst.yml
@@ -30,7 +30,7 @@ jobs:
           OSM_USERNAME: ${{ secrets.OSM_USERNAME }}
           OSM_PASSWORD: ${{ secrets.OSM_PASSWORD }}
         run: |
-          osmsg  --country turkey syria --format csv image --tags 'building' 'highway' 'waterway' 'amenity' --name stats --all_tags --hashtags smforst --charts --summary --length highway --update --name stats/smforst/Daily/stats --force --tm_stats
+          osmsg  --country turkey syria --format csv image --tags 'building' 'highway' 'waterway' 'amenity' --name stats --all_tags --hashtags smforst --charts --summary --length highway --update --name stats/smforst/Daily/stats --force --temp --tm_stats
 
       - name: add stats to git
         continue-on-error: true

diff --git a/.github/workflows/daily_turkey_eq.yml b/.github/workflows/daily_turkey_eq.yml
@@ -31,7 +31,7 @@ jobs:
           OSM_USERNAME: ${{ secrets.OSM_USERNAME }}
           OSM_PASSWORD: ${{ secrets.OSM_PASSWORD }}
         run: |
-          osmsg  --country turkey syria --format csv image --tags 'building' 'highway' 'waterway' 'amenity' --name stats/turkeyeq/Daily/stats --all_tags --hashtags TürkiyeEQ060223 --charts --summary --length highway --update --name stats/turkeyeq/Daily/stats --force --tm_stats
+          osmsg  --country turkey syria --format csv image --tags 'building' 'highway' 'waterway' 'amenity' --name stats/turkeyeq/Daily/stats --all_tags --hashtags TürkiyeEQ060223 --charts --summary --length highway --update --name stats/turkeyeq/Daily/stats --force --temp --tm_stats
 
       - name: add stats to git
         continue-on-error: true

diff --git a/.github/workflows/every_two_days_snrmpnepal.yml b/.github/workflows/every_two_days_snrmpnepal.yml
@@ -27,7 +27,7 @@ jobs:
       - name: Generate Every2days Hashtag SNRMPNepal Stats
         run: |
 
-          osmsg --format csv image --tags 'building' 'highway' 'waterway' 'amenity' --name stats --all_tags --hashtags SNRMPNepal --charts --summary --length highway --update --name stats/SNRMPNepal/Every2days/stats --force
+          osmsg --format csv image --tags 'building' 'highway' 'waterway' 'amenity' --name stats --all_tags --hashtags SNRMPNepal --charts --summary --length highway --update --name stats/SNRMPNepal/Every2days/stats --temp --force
 
       - name: add stats to git
         continue-on-error: true

diff --git a/.github/workflows/weekly_Nepal.yml b/.github/workflows/weekly_Nepal.yml
@@ -31,7 +31,7 @@ jobs:
           OSM_USERNAME: ${{ secrets.OSM_USERNAME }}
           OSM_PASSWORD: ${{ secrets.OSM_PASSWORD }}
         run: |
-          osmsg --url "http://download.geofabrik.de/asia/nepal-updates" --username '${{ secrets.OSM_USERNAME }}' --password '${{ secrets.OSM_PASSWORD }}' --format image csv --last_week --tags 'building' 'highway' 'waterway' 'amenity' --name stats/Nepal/Weekly/stats --read_from_metadata stats/Nepal/Weekly/stats_metadata.json --all_tags --changeset --charts --summary  --meta
+          osmsg --url "http://download.geofabrik.de/asia/nepal-updates" --username '${{ secrets.OSM_USERNAME }}' --password '${{ secrets.OSM_PASSWORD }}' --format image csv --last_week --tags 'building' 'highway' 'waterway' 'amenity' --name stats/Nepal/Weekly/stats --read_from_metadata stats/Nepal/Weekly/stats_metadata.json --all_tags --changeset --charts --summary  --meta --temp
         # run: |
         #   osmsg --url "https://planet.openstreetmap.org/replication/day" --format image csv --tags building highway waterway amenity --name stats --all_tags --last_week --country Nepal --name stats --read_from_metadata stats/Nepal/Weekly/stats_metadata.json --force
 

diff --git a/.github/workflows/weekly_field_mappers.yml b/.github/workflows/weekly_field_mappers.yml
@@ -30,7 +30,7 @@ jobs:
           OSM_USERNAME: ${{ secrets.OSM_USERNAME }}
           OSM_PASSWORD: ${{ secrets.OSM_PASSWORD }}
         run: |
-          osmsg --format csv image --name stats --tags amenity --all_tags --charts --summary --update --name stats/fieldmappers/Weekly/stats --force --field_mappers
+          osmsg --format csv image --name stats --tags amenity --all_tags --charts --summary --update --name stats/fieldmappers/Weekly/stats --force --field_mappers --temp
 
       - name: add stats to git
         continue-on-error: true

diff --git a/.github/workflows/weekly_global.yml b/.github/workflows/weekly_global.yml
@@ -30,7 +30,7 @@ jobs:
           OSM_USERNAME: ${{ secrets.OSM_USERNAME }}
           OSM_PASSWORD: ${{ secrets.OSM_PASSWORD }}
         run: |
-          osmsg --url "https://planet.openstreetmap.org/replication/day" --format csv image --last_week --tags 'building' 'highway' 'waterway' 'amenity' --name stats/Global/Weekly/stats --read_from_metadata stats/Global/Weekly/stats_metadata.json --all_tags --changeset --force --charts --summary  --meta --workers 8
+          osmsg --url "https://planet.openstreetmap.org/replication/day" --format csv image --last_week --tags 'building' 'highway' 'waterway' 'amenity' --name stats/Global/Weekly/stats --read_from_metadata stats/Global/Weekly/stats_metadata.json --all_tags --changeset --force --charts --summary  --meta --workers 8 --temp
 
       - name: Upload stats to Github
         run: |

diff --git a/.github/workflows/weekly_hotosm.yml b/.github/workflows/weekly_hotosm.yml
@@ -27,7 +27,7 @@ jobs:
 
       - name: Generate Weekly Hashtag Stats
         run: |
-          osmsg  --url "https://planet.openstreetmap.org/replication/day" --format csv image --last_week --tags 'building' 'highway' 'waterway' 'amenity' --name stats/hotosm/Weekly/stats --all_tags --read_from_metadata stats/hotosm/Weekly/stats_metadata.json --hashtags hotosm --force --charts --summary  --meta
+          osmsg  --url "https://planet.openstreetmap.org/replication/day" --format csv image --last_week --tags 'building' 'highway' 'waterway' 'amenity' --name stats/hotosm/Weekly/stats --all_tags --read_from_metadata stats/hotosm/Weekly/stats_metadata.json --hashtags hotosm --force --charts --summary  --meta --temp
 
       - name: add stats to git
         continue-on-error: true

diff --git a/osmsg/changefiles.py b/osmsg/changefiles.py
@@ -29,6 +29,13 @@
 import pytz
 import requests
 from osmium.replication.server import ReplicationServer
+from requests.adapters import HTTPAdapter
+
+retry_count = 5
+session = requests.Session()
+retries = HTTPAdapter(max_retries=retry_count)
+session.mount("https://", retries)
+session.mount("http://", retries)
 
 
 def in_local_timezone(date, timezone):
@@ -124,7 +131,7 @@ def previous_week(timezone):
 
 
 def seq_to_timestamp(url, timezone):
-    response = requests.get(url)
+    response = session.get(url)
     rtxt = response.text
     # find the index of the "timestamp=" substring
     timestamp_start = rtxt.find("timestamp=") + len("timestamp=")

diff --git a/osmsg/changesets.py b/osmsg/changesets.py
@@ -26,6 +26,13 @@
 import sys
 
 import requests
+from requests.adapters import HTTPAdapter
+
+retry_count = 5
+session = requests.Session()
+retries = HTTPAdapter(max_retries=retry_count)
+session.mount("https://", retries)
+session.mount("http://", retries)
 
 
 class ChangesetToolKit:
@@ -35,7 +42,7 @@ def __init__(
         self.replication_url = replication_url
 
     def get_current_state(self):
-        state_yml = requests.get(self.replication_url + "state.yaml").text
+        state_yml = session.get(self.replication_url + "state.yaml").text
         current_sequence = int(state_yml.split("sequence: ")[1])
         last_run = datetime.datetime.strptime(
             state_yml.split("last_run: ")[1][:19], "%Y-%m-%d %H:%M:%S"
@@ -89,7 +96,7 @@ def get_state_url(self, sequence):
     def sequence_to_timestamp(self, sequence):
         state_url = self.get_state_url(sequence)
 
-        state_yml = requests.get(state_url).text
+        state_yml = session.get(state_url).text
         last_run = datetime.datetime.strptime(
             state_yml.split("last_run: ")[1][:19], "%Y-%m-%d %H:%M:%S"
         ).replace(tzinfo=dt.timezone.utc)

diff --git a/osmsg/utils.py b/osmsg/utils.py
@@ -43,8 +43,7 @@
 from shapely.geometry import MultiPolygon, Polygon, box
 
 # number of times to retry
-retry_count = 3
-# create a session with retry configuration
+retry_count = 5
 session = requests.Session()
 retries = HTTPAdapter(max_retries=retry_count)
 session.mount("https://", retries)