87 lines
3.9 KiB
Python
87 lines
3.9 KiB
Python
from argparse import ArgumentParser
|
|
from pathlib import Path
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
|
|
def download_tile(zoom: int, x: int, y: int, tile_path: Path):
|
|
osm_timeout = 3
|
|
http_code_ok = 200
|
|
|
|
request = urllib.request.Request(f'https://tile.openstreetmap.org/{zoom}/{x}/{y}.png')
|
|
request.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:138.0) Gecko/20100101 Firefox/138.0')
|
|
request.add_header('Accept', 'image/avif,image/webp,image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5')
|
|
request.add_header('Accept-Language', 'en-US,en;q=0.5')
|
|
request.add_header('Accept-Encoding', 'gzip, deflate, br, zstd')
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=osm_timeout) as response:
|
|
if response.status != http_code_ok:
|
|
raise RuntimeError(
|
|
f'tile.openstreetmap.org answer is not OK (status: {response.status}) : {response.read()}')
|
|
tile_path.write_bytes(response.read())
|
|
except urllib.error.URLError as error:
|
|
raise RuntimeError(f'Cannot retrieve tile: {error}') from error
|
|
|
|
|
|
def main():
|
|
parser = ArgumentParser()
|
|
parser.add_argument('data', type=Path, help='Path to tile data to consolidate')
|
|
arguments = parser.parse_args()
|
|
|
|
data_path: Path = arguments.data
|
|
del arguments
|
|
|
|
zoom_levels = sorted([int(p.name) for p in data_path.iterdir()])
|
|
download_time = 0.2
|
|
try:
|
|
for zoom in zoom_levels:
|
|
x_min = 10_000_000
|
|
x_max = -10_000_000
|
|
y_min = 10_000_000
|
|
y_max = -10_000_000
|
|
tile_count = 0
|
|
for tile_path in (data_path / str(zoom)).rglob('*.png'):
|
|
x_value = int(tile_path.parent.name)
|
|
x_min = min(x_min, x_value)
|
|
x_max = max(x_max, x_value)
|
|
y_value = int(tile_path.stem)
|
|
y_min = min(y_min, y_value)
|
|
y_max = max(y_max, y_value)
|
|
tile_count += 1
|
|
total_tile_count = (x_max - x_min + 1) * (y_max - y_min + 1)
|
|
print(f'{zoom=} x in [{x_min}, {x_max}], y in [{y_min}, {y_max}]:'
|
|
f' {tile_count} files out of {total_tile_count} -> {total_tile_count - tile_count} to download')
|
|
start_download = time.monotonic()
|
|
for x in range(x_min, x_max + 1):
|
|
for y in range(y_min, y_max + 1):
|
|
tile_path = Path(data_path / str(zoom) / str(x) / f'{y}.png')
|
|
if tile_path.exists():
|
|
continue
|
|
try:
|
|
if not tile_path.parent.exists():
|
|
tile_path.parent.mkdir(parents=True, exist_ok=True)
|
|
download_tile(zoom=zoom, x=x, y=y, tile_path=tile_path)
|
|
time.sleep(0.1)
|
|
except RuntimeError as error:
|
|
print(error)
|
|
continue
|
|
tile_count += 1
|
|
if tile_count % 5 == 0 or tile_count == total_tile_count:
|
|
download_time = 0.99 * download_time + (0.01 * (time.monotonic() - start_download) / 5)
|
|
remaining_time = int((total_tile_count - tile_count) * download_time)
|
|
remaining_hours = remaining_time // 3600
|
|
remaining_time -= 3600 * remaining_hours
|
|
remaining_minutes = remaining_time // 60
|
|
remaining_time -= 60 * remaining_minutes
|
|
print(f'Downloading tiles {tile_count} / {total_tile_count}'
|
|
f' ({remaining_hours:02d}:{remaining_minutes:02d}:{remaining_time:02d} remaining)',
|
|
end='\r')
|
|
start_download = time.monotonic()
|
|
print()
|
|
except KeyboardInterrupt:
|
|
print('\rDo')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|