65 lines
2.6 KiB
Python
65 lines
2.6 KiB
Python
import shutil
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
from PIL import Image
|
|
from tqdm import tqdm
|
|
|
|
from pkg.proxies import PROXIES
|
|
from .image import convert, is_png
|
|
|
|
|
|
def del_path(p: Path):
|
|
if p.is_file() or p.is_symlink():
|
|
p.unlink(missing_ok=True)
|
|
else:
|
|
shutil.rmtree(p)
|
|
|
|
|
|
def download_tile(task):
|
|
row, col, base_url, r_headers, output, use_proxy, convert_to_png = task
|
|
corrupted_image = False
|
|
try:
|
|
output_path: Path = output / f"{row}_{col}.png"
|
|
if output_path.exists():
|
|
valid_png_file, image_type = is_png(output_path)
|
|
if convert_to_png and image_type != 'PNG':
|
|
# The image was read sucessfully by PIL but it's in the wrong format.
|
|
convert(output_path)
|
|
if not is_png(output_path):
|
|
tqdm.write(f'PNG conversion for {output_path} failed. Deleting and retrying...')
|
|
corrupted_image = True
|
|
elif not valid_png_file:
|
|
# We will re-download the image. Don't need to delete it, just overwrite it.
|
|
# del_path(output_path)
|
|
corrupted_image = True
|
|
tqdm.write(f'Bad image file: "{output_path}" (is format: {image_type}), deleting and retrying...')
|
|
else:
|
|
return row, col, 'exist'
|
|
tile_url = f"{base_url}/{row}/{col}".replace('//', '/').replace(':/', '://')
|
|
response = requests.get(tile_url, headers=r_headers, proxies=PROXIES if use_proxy else None, timeout=60)
|
|
if response.status_code == 200:
|
|
if not convert_to_png and not response.headers.get('Content-Type') == 'image/png':
|
|
# If we will convert the image to a PNG, ignore this header.
|
|
raise Exception(f'Response gave Content-Type: {response.headers.get("Content-Type")}')
|
|
with open(output_path, "wb") as f:
|
|
f.write(response.content)
|
|
|
|
if convert_to_png:
|
|
convert(output_path)
|
|
corrupted_image = True # force a re-check
|
|
|
|
# Recheck the PNG if it was corrupted.
|
|
if corrupted_image and not is_png(output_path):
|
|
print(f"Retry for {row}_{col} failed a second time: cannot identify image file")
|
|
return row, col, 'failure'
|
|
else:
|
|
return row, col, 'success'
|
|
else:
|
|
print(f"Failed to download tile {row}_{col}")
|
|
return row, col, 'failure'
|
|
except Exception as e:
|
|
# traceback.print_exc()
|
|
tqdm.write(f'Exception on {(row, col)} - {e.__class__.__name__}: {e}')
|
|
return row, col, 'failure'
|