Download additional large test data from public sources to '../data'.
Data is used for performance testing and some of the notebooks.
# make data dir (already in .gitignore)
import os
data_dir = "../data"
if not os.path.exists(data_dir):
os.makedirs(data_dir)
File download from url to filename
from tqdm.notebook import tqdm
import requests
def file_download(url,filename):
print(f"Download {url} \nto {filename}")
r = requests.get(url, stream=True)
size = int(r.headers.get('content-length', 0))
if os.path.exists(filename):
if size == os.path.getsize(filename):
print("Local file already exists, skip download.")
return
pbar = tqdm(total=size, unit='iB', unit_scale=True)
with open(filename, 'wb') as file:
for chunk in r.iter_content(1024*1024*2): # To-Do: best choice for chunk size?
pbar.update(len(chunk))
file.write(chunk)
pbar.close()
filenames = ["altimetry_3a_2018_filter1.dfs0",
"SW_gwm_3a_extracted_2018.dfs0",
"SW_gwm_domain_Hm0_201801.dfsu"]
for fn in filenames:
url = f"https://automodelstorage.blob.core.windows.net/globalaltimetry/{fn}"
fn_target = f"{data_dir}/{fn}"
file_download(url,fn_target)
Download https://automodelstorage.blob.core.windows.net/globalaltimetry/altimetry_3a_2018_filter1.dfs0 to ../data/altimetry_3a_2018_filter1.dfs0 Local file already exists, skip download. Download https://automodelstorage.blob.core.windows.net/globalaltimetry/SW_gwm_3a_extracted_2018.dfs0 to ../data/SW_gwm_3a_extracted_2018.dfs0 Local file already exists, skip download. Download https://automodelstorage.blob.core.windows.net/globalaltimetry/SW_gwm_domain_Hm0_201801.dfsu to ../data/SW_gwm_domain_Hm0_201801.dfsu
HBox(children=(FloatProgress(value=0.0, max=1580035484.0), HTML(value='')))