test
This commit is contained in:
parent
65953c9bde
commit
1d8b45fae4
|
@ -62,28 +62,28 @@ if __name__ == '__main__':
|
|||
for row in row_iter:
|
||||
row_i = row
|
||||
col_iter = range(min_col, max_col + 1)
|
||||
col_bar = tqdm(total=len(col_iter), leave=False)
|
||||
|
||||
if args.no_download:
|
||||
for col in col_iter:
|
||||
tiles.append((row, col))
|
||||
else:
|
||||
with (ThreadPoolExecutor(args.threads) as executor):
|
||||
futures = [executor.submit(download_tile, (row, col, args.base_url, r_headers, tiles_output, args.proxy)) for col in col_iter]
|
||||
for future in as_completed(futures):
|
||||
result = future.result()
|
||||
if result:
|
||||
result_row, result_col, new_image = result
|
||||
if new_image == 'success':
|
||||
total_downloaded += 1
|
||||
tiles.append((result_row, result_col))
|
||||
elif new_image == 'exist':
|
||||
tiles.append((result_row, result_col))
|
||||
elif new_image == 'failure':
|
||||
retries.append((result_row, result_col))
|
||||
row_bar.set_postfix({'new_files': total_downloaded, 'failures': len(retries)})
|
||||
col_bar.update()
|
||||
row_bar.refresh()
|
||||
# if args.no_download:
|
||||
# for col in col_iter:
|
||||
# tiles.append((row, col))
|
||||
# else:
|
||||
with (ThreadPoolExecutor(args.threads) as executor):
|
||||
col_bar = tqdm(total=len(col_iter), leave=False)
|
||||
futures = [executor.submit(download_tile, (row, col, args.base_url, r_headers, tiles_output, args.proxy, args.no_download)) for col in col_iter]
|
||||
for future in as_completed(futures):
|
||||
result = future.result()
|
||||
if result:
|
||||
result_row, result_col, new_image = result
|
||||
if new_image == 'success':
|
||||
total_downloaded += 1
|
||||
tiles.append((result_row, result_col))
|
||||
elif new_image == 'exist':
|
||||
tiles.append((result_row, result_col))
|
||||
elif new_image == 'failure':
|
||||
retries.append((result_row, result_col))
|
||||
row_bar.set_postfix({'new_files': total_downloaded, 'failures': len(retries)})
|
||||
col_bar.update()
|
||||
row_bar.refresh()
|
||||
col_bar.close()
|
||||
row_bar.set_postfix({'new_files': total_downloaded, 'failures': len(retries)})
|
||||
row_bar.update()
|
||||
|
|
|
@ -16,11 +16,11 @@ def del_path(p: Path):
|
|||
|
||||
|
||||
def download_tile(task):
|
||||
row, col, base_url, r_headers, output, use_proxy = task
|
||||
row, col, base_url, r_headers, output, use_proxy, no_download = task
|
||||
try:
|
||||
output_path: Path = output / f"{row}_{col}.png"
|
||||
if output_path.exists():
|
||||
if not is_png(output_path):
|
||||
if no_download and not is_png(output_path):
|
||||
# Delete the file and try again.
|
||||
del_path(output_path)
|
||||
tqdm.write(f'cannot identify image file: "{output_path}", deleting and retrying...')
|
||||
|
|
Loading…
Reference in New Issue