adjust loop logic
This commit is contained in:
parent
0fb26ccf1d
commit
c45da1cf0c
|
@ -60,23 +60,26 @@ if __name__ == '__main__':
|
|||
if args.referer:
|
||||
r_headers['Referer'] = args.referer
|
||||
|
||||
row_bar = tqdm(total=0, desc='Row 000 | Loop 0/0', postfix={'new_files': 0, 'failures': 0})
|
||||
retry_files = set()
|
||||
tiles = set()
|
||||
total_downloaded = 0
|
||||
|
||||
row_bar = tqdm(total=0, desc='Row 000 | Loop 0/0', postfix={'new_files': 0, 'failures': 0, 'fixed': 0})
|
||||
for i in range(1, args.download_loops + 1):
|
||||
row_bar.reset()
|
||||
tiles = []
|
||||
retries = []
|
||||
fixed_files = 0
|
||||
total_downloaded = 0
|
||||
row_i = min_row
|
||||
row_iter = range(min_row, max_row + 1)
|
||||
row_bar.total = len(row_iter)
|
||||
row_bar.desc = f'Row {row_i} | Loop {i}/{args.download_loops}'
|
||||
row_bar.set_postfix({'new_files': total_downloaded, 'failures': len(retries)})
|
||||
row_bar.set_postfix({'new_files': total_downloaded, 'failures': len(retry_files), 'fixed': fixed_files})
|
||||
for row in row_iter:
|
||||
row_i = row
|
||||
col_iter = range(min_col, max_col + 1)
|
||||
if args.no_download:
|
||||
for col in col_iter:
|
||||
tiles.append((row, col))
|
||||
tiles.add((row, col))
|
||||
else:
|
||||
col_bar = tqdm(total=len(col_iter), leave=False)
|
||||
|
||||
|
@ -91,28 +94,32 @@ if __name__ == '__main__':
|
|||
result_row, result_col, new_image = result
|
||||
if new_image == 'success':
|
||||
total_downloaded += 1
|
||||
tiles.append((result_row, result_col))
|
||||
tiles.add((result_row, result_col))
|
||||
elif new_image == 'exist':
|
||||
tiles.append((result_row, result_col))
|
||||
tiles.add((result_row, result_col))
|
||||
elif new_image == 'failure':
|
||||
retries.append((result_row, result_col))
|
||||
row_bar.set_postfix({'new_files': total_downloaded, 'failures': len(retries)})
|
||||
retry_files.add((result_row, result_col))
|
||||
row_bar.set_postfix({'new_files': total_downloaded, 'failures': len(retry_files), 'fixed': fixed_files})
|
||||
elif new_image == 'fixed':
|
||||
tiles.add((result_row, result_col))
|
||||
fixed_files += 1
|
||||
row_bar.set_postfix({'new_files': total_downloaded, 'failures': len(retry_files), 'fixed': fixed_files})
|
||||
col_bar.update()
|
||||
row_bar.refresh()
|
||||
col_bar.close()
|
||||
row_bar.set_postfix({'new_files': total_downloaded, 'failures': len(retries)})
|
||||
row_bar.set_postfix({'new_files': total_downloaded, 'failures': len(retry_files), 'fixed': fixed_files})
|
||||
row_bar.update()
|
||||
|
||||
row_bar.close()
|
||||
|
||||
col_bar = tqdm(total=len(retries), desc=f'Tile Retries')
|
||||
col_bar = tqdm(total=len(retry_files), desc=f'Tile Retries')
|
||||
with ThreadPoolExecutor(args.dl_threads) as executor:
|
||||
futures = [executor.submit(download_tile, (row, col, args.base_url, r_headers, tiles_output, args.proxy)) for row, col in retries]
|
||||
futures = [executor.submit(download_tile, (row, col, args.base_url, r_headers, tiles_output, args.proxy)) for row, col in retry_files]
|
||||
for future in as_completed(futures):
|
||||
result = future.result()
|
||||
if result:
|
||||
result_row, result_col, new_image = result
|
||||
tiles.append((result_row, result_col))
|
||||
tiles.add((result_row, result_col))
|
||||
if new_image == 'success':
|
||||
total_downloaded += 1
|
||||
elif new_image == 'failure':
|
||||
|
|
|
@ -55,7 +55,7 @@ def download_tile(task):
|
|||
print(f"Retry for {row}_{col} failed a second time: cannot identify image file")
|
||||
return row, col, 'failure'
|
||||
else:
|
||||
return row, col, 'success'
|
||||
return row, col, 'success' if not corrupted_image else 'fixed'
|
||||
else:
|
||||
print(f"Failed to download tile {row}_{col}")
|
||||
return row, col, 'failure'
|
||||
|
|
Loading…
Reference in New Issue