exit download loop early if all already downloaded
This commit is contained in:
parent
37ed3dc93a
commit
a70e078c3f
|
@ -62,14 +62,15 @@ if __name__ == '__main__':
|
|||
|
||||
retry_files = set()
|
||||
tiles = set()
|
||||
total_downloaded = 0
|
||||
total_new_files = 0
|
||||
total_fixed_files = 0
|
||||
|
||||
row_bar = tqdm(total=0, desc='Row 000 | Loop 0/0', postfix={'new_files': 0, 'failures': 0, 'fixed': 0})
|
||||
for i in range(1, args.download_loops + 1):
|
||||
row_bar.reset()
|
||||
converted_files = 0
|
||||
fixed_files = 0
|
||||
total_downloaded = 0
|
||||
new_files = 0
|
||||
row_i = min_row
|
||||
row_iter = range(min_row, max_row + 1)
|
||||
row_bar.total = len(row_iter)
|
||||
|
@ -77,7 +78,7 @@ if __name__ == '__main__':
|
|||
|
||||
|
||||
def update_bar_postfix():
|
||||
row_bar.set_postfix({'new': total_downloaded, 'failures': len(retry_files), 'fixed': fixed_files, 'converted': converted_files})
|
||||
row_bar.set_postfix({'new': new_files, 'failures': len(retry_files), 'fixed': fixed_files, 'converted': converted_files})
|
||||
|
||||
|
||||
update_bar_postfix()
|
||||
|
@ -101,7 +102,8 @@ if __name__ == '__main__':
|
|||
if result:
|
||||
result_row, result_col, new_image = result
|
||||
if new_image == 'success':
|
||||
total_downloaded += 1
|
||||
new_files += 1
|
||||
total_new_files += 1
|
||||
tiles.add((result_row, result_col))
|
||||
elif new_image == 'exist':
|
||||
tiles.add((result_row, result_col))
|
||||
|
@ -110,6 +112,7 @@ if __name__ == '__main__':
|
|||
elif new_image == 'fixed':
|
||||
tiles.add((result_row, result_col))
|
||||
fixed_files += 1
|
||||
total_fixed_files += 1
|
||||
elif new_image == 'converted':
|
||||
tiles.add((result_row, result_col))
|
||||
converted_files += 1
|
||||
|
@ -118,9 +121,13 @@ if __name__ == '__main__':
|
|||
col_bar.close()
|
||||
update_bar_postfix()
|
||||
row_bar.update()
|
||||
|
||||
if total_new_files == 0 and total_fixed_files == 0:
|
||||
break
|
||||
row_bar.close()
|
||||
|
||||
if total_new_files == 0 and total_fixed_files == 0:
|
||||
print('All files downloaded, exiting download loop.')
|
||||
|
||||
col_bar = tqdm(total=len(retry_files), desc=f'Tile Retries')
|
||||
with ThreadPoolExecutor(args.dl_threads) as executor:
|
||||
futures = [executor.submit(download_tile, (row, col, args.base_url, r_headers, tiles_output, args.proxy)) for row, col in retry_files]
|
||||
|
@ -130,13 +137,13 @@ if __name__ == '__main__':
|
|||
result_row, result_col, new_image = result
|
||||
tiles.add((result_row, result_col))
|
||||
if new_image == 'success':
|
||||
total_downloaded += 1
|
||||
new_files += 1
|
||||
elif new_image == 'failure':
|
||||
col_bar.write(f'{(result_row, result_col)} failed!')
|
||||
col_bar.update()
|
||||
col_bar.close()
|
||||
|
||||
print(f'Downloaded {total_downloaded} images.')
|
||||
print(f'Downloaded {new_files} images.')
|
||||
|
||||
print('Determining tile width...', end='')
|
||||
tile_size = random_file_width(tiles_output)
|
||||
|
|
Loading…
Reference in New Issue