Initial commit

consolidate_tiles.py

@@ -0,0 +1,87 @@
+from argparse import ArgumentParser
+from pathlib import Path
+import time
+import urllib.error
+import urllib.request
+
+
+def download_tile(zoom: int, x: int, y: int, tile_path: Path):
+    osm_timeout = 3
+    http_code_ok = 200
+
+    request = urllib.request.Request(f'https://tile.openstreetmap.org/{zoom}/{x}/{y}.png')
+    request.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:138.0) Gecko/20100101 Firefox/138.0')
+    request.add_header('Accept', 'image/avif,image/webp,image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5')
+    request.add_header('Accept-Language', 'en-US,en;q=0.5')
+    request.add_header('Accept-Encoding', 'gzip, deflate, br, zstd')
+    try:
+        with urllib.request.urlopen(request, timeout=osm_timeout) as response:
+            if response.status != http_code_ok:
+                raise RuntimeError(
+                    f'tile.openstreetmap.org answer is not OK (status: {response.status}) : {response.read()}')
+            tile_path.write_bytes(response.read())
+    except urllib.error.URLError as error:
+        raise RuntimeError(f'Cannot retrieve tile: {error}') from error
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument('data', type=Path, help='Path to tile data to consolidate')
+    arguments = parser.parse_args()
+
+    data_path: Path = arguments.data
+    del arguments
+
+    zoom_levels = sorted([int(p.name) for p in data_path.iterdir()])
+    download_time = 0.2
+    try:
+        for zoom in zoom_levels:
+            x_min = 10_000_000
+            x_max = -10_000_000
+            y_min = 10_000_000
+            y_max = -10_000_000
+            tile_count = 0
+            for tile_path in (data_path / str(zoom)).rglob('*.png'):
+                x_value = int(tile_path.parent.name)
+                x_min = min(x_min, x_value)
+                x_max = max(x_max, x_value)
+                y_value = int(tile_path.stem)
+                y_min = min(y_min, y_value)
+                y_max = max(y_max, y_value)
+                tile_count += 1
+            total_tile_count = (x_max - x_min + 1) * (y_max - y_min + 1)
+            print(f'{zoom=} x in [{x_min}, {x_max}], y in [{y_min}, {y_max}]:'
+                f' {tile_count} files out of {total_tile_count} -> {total_tile_count - tile_count} to download')
+            start_download = time.monotonic()
+            for x in range(x_min, x_max + 1):
+                for y in range(y_min, y_max + 1):
+                    tile_path = Path(data_path / str(zoom) / str(x) / f'{y}.png')
+                    if tile_path.exists():
+                        continue
+                    try:
+                        if not tile_path.parent.exists():
+                            tile_path.parent.mkdir(parents=True, exist_ok=True)
+                        download_tile(zoom=zoom, x=x, y=y, tile_path=tile_path)
+                        time.sleep(0.1)
+                    except RuntimeError as error:
+                        print(error)
+                        continue
+                    tile_count += 1
+                    if tile_count % 5 == 0 or tile_count == total_tile_count:
+                        download_time = 0.99 * download_time + (0.01 * (time.monotonic() - start_download) / 5)
+                        remaining_time = int((total_tile_count - tile_count) * download_time)
+                        remaining_hours = remaining_time // 3600
+                        remaining_time -= 3600 * remaining_hours
+                        remaining_minutes = remaining_time // 60
+                        remaining_time -= 60 * remaining_minutes
+                        print(f'Downloading tiles {tile_count} / {total_tile_count}'
+                              f' ({remaining_hours:02d}:{remaining_minutes:02d}:{remaining_time:02d} remaining)',
+                              end='\r')
+                        start_download = time.monotonic()
+        print()
+    except KeyboardInterrupt:
+        print('\rDo')
+
+
+if __name__ == '__main__':
+    main()