adjust downloader and conversion
This commit is contained in:
parent
1c257f3102
commit
28ad261c83
|
@ -107,16 +107,12 @@ if __name__ == '__main__':
|
||||||
tiles.add((result_row, result_col))
|
tiles.add((result_row, result_col))
|
||||||
elif new_image == 'failure':
|
elif new_image == 'failure':
|
||||||
retry_files.add((result_row, result_col))
|
retry_files.add((result_row, result_col))
|
||||||
update_bar_postfix()
|
|
||||||
elif new_image == 'fixed':
|
elif new_image == 'fixed':
|
||||||
tiles.add((result_row, result_col))
|
tiles.add((result_row, result_col))
|
||||||
fixed_files += 1
|
fixed_files += 1
|
||||||
update_bar_postfix()
|
|
||||||
elif new_image == 'converted':
|
elif new_image == 'converted':
|
||||||
tiles.add((result_row, result_col))
|
tiles.add((result_row, result_col))
|
||||||
converted_files += 1
|
converted_files += 1
|
||||||
# Don't update bar on every loop since we could be converting many files.
|
|
||||||
# update_bar_postfix()
|
|
||||||
col_bar.update()
|
col_bar.update()
|
||||||
row_bar.refresh()
|
row_bar.refresh()
|
||||||
col_bar.close()
|
col_bar.close()
|
||||||
|
|
17
pkg/image.py
17
pkg/image.py
|
@ -29,9 +29,14 @@ def is_png(file_path):
|
||||||
|
|
||||||
|
|
||||||
def convert_to_png(file_path):
|
def convert_to_png(file_path):
|
||||||
img = Image.open(file_path)
|
try:
|
||||||
if img.format != 'PNG':
|
img = Image.open(file_path)
|
||||||
img.save(file_path, format='PNG')
|
if img.format != 'PNG':
|
||||||
# tqdm.write(f'Converted {file_path} from {img.format} to PNG')
|
img.save(file_path, format='PNG')
|
||||||
return True
|
return True
|
||||||
return False
|
else:
|
||||||
|
return False
|
||||||
|
except Exception:
|
||||||
|
# import traceback
|
||||||
|
# traceback.print_exc()
|
||||||
|
return False
|
||||||
|
|
|
@ -40,22 +40,26 @@ def download_tile(task):
|
||||||
tile_url = f"{base_url}/{row}/{col}".replace('//', '/').replace(':/', '://')
|
tile_url = f"{base_url}/{row}/{col}".replace('//', '/').replace(':/', '://')
|
||||||
response = requests.get(tile_url, headers=r_headers, proxies=PROXIES if use_proxy else None, timeout=60)
|
response = requests.get(tile_url, headers=r_headers, proxies=PROXIES if use_proxy else None, timeout=60)
|
||||||
if response.status_code == 200:
|
if response.status_code == 200:
|
||||||
if not do_convert_to_png and not response.headers.get('Content-Type') == 'image/png':
|
# if not do_convert_to_png and not response.headers.get('Content-Type') == 'image/png':
|
||||||
# If we will convert the image to a PNG, ignore this header.
|
# # If we will convert the image to a PNG, ignore this header.
|
||||||
raise Exception(f'Response gave Content-Type: {response.headers.get("Content-Type")}')
|
# raise Exception(f'Response gave Content-Type: {response.headers.get("Content-Type")}')
|
||||||
with open(output_path, "wb") as f:
|
with open(output_path, "wb") as f:
|
||||||
f.write(response.content)
|
f.write(response.content)
|
||||||
|
|
||||||
if do_convert_to_png:
|
if do_convert_to_png:
|
||||||
convert_to_png(output_path)
|
convert_to_png(output_path)
|
||||||
corrupted_image = True # force a re-check
|
if not is_png(output_path)[0]:
|
||||||
|
tqdm.write(f'PNG conversion for {output_path} failed')
|
||||||
# Recheck the PNG if it was corrupted.
|
else:
|
||||||
if corrupted_image and not is_png(output_path):
|
return row, col, 'success' if not corrupted_image else 'fixed'
|
||||||
print(f"Retry for {row}_{col} failed a second time: cannot identify image file")
|
|
||||||
return row, col, 'failure'
|
|
||||||
else:
|
else:
|
||||||
return row, col, 'success' if not corrupted_image else 'fixed'
|
# Recheck the PNG if it was corrupted.
|
||||||
|
valid_png_file, image_type = is_png(output_path)
|
||||||
|
if not valid_png_file:
|
||||||
|
tqdm.write(f'Bad image file: "{output_path}" (is format: {image_type}).')
|
||||||
|
return row, col, 'failure'
|
||||||
|
else:
|
||||||
|
return row, col, 'success' if not corrupted_image else 'fixed'
|
||||||
else:
|
else:
|
||||||
print(f"Failed to download tile {row}_{col}")
|
print(f"Failed to download tile {row}_{col}")
|
||||||
return row, col, 'failure'
|
return row, col, 'failure'
|
||||||
|
|
Loading…
Reference in New Issue