diff --git a/README.md b/README.md index 1f909da..4d7145f 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,8 @@ python3 exfiltrate.py \ --threads 30 ``` -Building the GeoTIFF will take dozens of gigs of memory for any significant extent! For example, a 336 square mile -extent required about 400GB of memory. You can use swap for this, but don't expect it to be very quick if you go this +Building the GeoTIFF will take dozens of gigs of memory for any significant extent! For example, a ??? square mile +extent required about 280GB of memory. You can use swap for this, but don't expect it to be very quick if you go this route. Be careful not to go overboard with your spatial extent. Use only what you need to avoid unnecessary processing time or @@ -48,4 +48,4 @@ TODO ### Inspiration -https://jimmyutterstrom.com/blog/2019/06/05/map-tiles-to-geotiff/ \ No newline at end of file +https://jimmyutterstrom.com/blog/2019/06/05/map-tiles-to-geotiff/ diff --git a/exfiltrate.py b/exfiltrate.py index e94a3c9..62a4ced 100644 --- a/exfiltrate.py +++ b/exfiltrate.py @@ -11,11 +11,13 @@ from PIL import Image from rasterio import Affine from tqdm import tqdm +from pkg.helpers import convert_seconds from pkg.image import random_file_width from pkg.spatial import deg2num from pkg.thread import download_tile if __name__ == '__main__': + main_start_time = time.time() parser = argparse.ArgumentParser(description='Exfiltrate data from WMS servers.') parser.add_argument('base_url', help='The base URL for the WMS server. Example: https://wmts.nlsc.gov.tw/wmts/nURBAN/default/EPSG:3857/') parser.add_argument('--zoom', type=int, required=True, help='The zoom level to use.') @@ -168,8 +170,9 @@ if __name__ == '__main__': * Affine.scale((bottom_right_lon - top_left_lon) / image_data.shape[2], (bottom_right_lat - top_left_lat) / image_data.shape[1])) # Write the image data to a GeoTIFF file - print('Saving to:', output_tiff) - start = time.time() + print('Writing TIFF to:', output_tiff) + start_write_tiff = time.time() with rasterio.open(output_tiff, "w", driver="GTiff", height=num_rows * tile_size, width=num_cols * tile_size, count=3, dtype=str(image_data.dtype), crs='EPSG:4326', transform=transform, compress="DEFLATE", nodata=0) as dst: dst.write(image_data, indexes=[1, 2, 3]) - print(f'Saved in {int(time.time() - start)} seconds.') + now = time.time() + print(f'Time to write TIFF: {convert_seconds(int(now - start_write_tiff))} seconds. Total run time: {convert_seconds(int(now - main_start_time))}') diff --git a/pkg/helpers.py b/pkg/helpers.py new file mode 100644 index 0000000..c990585 --- /dev/null +++ b/pkg/helpers.py @@ -0,0 +1,6 @@ +def convert_seconds(seconds): + hours = seconds // 3600 + seconds %= 3600 + minutes = seconds // 60 + seconds %= 60 + return "%d:%02d:%02d" % (hours, minutes, seconds)