###################################################################################
# set the parameters and press Ctrl-ENTER
spacecraft = 'A'
start = "2016-01-01" # included
end = "2016-02-01" # not included
#sampling = "PT60S" # set to None to use the original sampling
sampling = None
models = [
"CHAOS-6-Core",
"CHAOS-6-Static",
"CHAOS-6-MMA-Primary",
"CHAOS-6-MMA-Secondary"
]
variables=[
"B_NEC",
"B_NEC_CHAOS-6-Core",
"B_NEC_CHAOS-6-Static",
"B_NEC_CHAOS-6-MMA-Primary",
"B_NEC_CHAOS-6-MMA-Secondary",
'Flags_B', 'Flags_q', 'Flags_Platform',
'Kp', 'Dst', 'F10_INDEX', 'OrbitNumber', 'QDLat', 'QDLon', 'QDBasis', 'MLT',
]
output_filename_template = "VIRES_MAG{spacecraft}_LR_1B_{start}_{end}.cdf"
import sys
from os import remove, rename
from os.path import exists
from viresclient._wps.wps_vires import ViresWPS10Service
from viresclient._wps.environment import JINJA2_ENVIRONMENT
from viresclient._wps.time_util import parse_datetime, parse_duration, encode_duration, timedelta, datetime
from viresclient._client import ProgressBarProcessing, ProgressBarDownloading, ClientRequest
wps_proxy = ViresWPS10Service(
url="https://staging.viresdisc.vires.services/openows"
)
request_template = JINJA2_ENVIRONMENT.get_template(
"vires_fetch_filtered_data_async.xml"
)
spcectraft_to_collections_mapping = {
"A": "SW_OPER_MAGA_LR_1B",
"B": "SW_OPER_MAGB_LR_1B",
"C": "SW_OPER_MAGC_LR_1B",
}
def chop_time_range_by_days(start_time, end_time):
year, month, day = start_time.year, start_time.month, start_time.day
chunk_offset= datetime(year, month, day)
chunk_duration = timedelta(days=1, microseconds=-1)
chunk_step = timedelta(days=1)
while chunk_offset < end_time:
yield (chunk_offset, chunk_offset + chunk_duration)
chunk_offset += chunk_step
Can't download new leap second table Can't find leap second table. Using one built into code. Last leap second in built in table is on Jan 01 2017.
start_time = parse_datetime(start)
end_time = parse_datetime(end)
# split time into daily chunks
time_windows = list(chop_time_range_by_days(start_time, end_time))
output_filenames = [
output_filename_template.format(
spacecraft=spacecraft,
start=chunk_start_time.strftime("%Y%m%d-%H%M%S"),
end=chunk_end_time.strftime("%Y%m%d-%H%M%S"),
) for chunk_start_time, chunk_end_time in time_windows
]
# validate sampling step
if sampling:
parse_duration(sampling)
# prepare collection identifiers
collections = {
spacecraft: [spcectraft_to_collections_mapping[spacecraft]]
}
# print request summary
print("request summary:")
print("\toutput_filenames:")
for filename in output_filenames:
print("\t\t%s" % filename)
print("\tstart time: %sZ" % start_time.isoformat())
print("\tend time: %sZ" % end_time.isoformat())
print("\tsampling: %s" % (sampling or "original data sampling"))
print("\tcollections:")
for spacecraft, colls in collections.items():
print("\t%s: %s" % (spacecraft, ", ".join(colls)))
print("\tmodels:")
for model in models:
print("\t\t%s" % model)
print("\tvariables:")
for variable in variables:
print("\t\t%s" % variable)
print()
# download data by chunks
for (chunk_start_time, chunk_end_time), output_filename in zip(time_windows, output_filenames):
print("processing %s" % output_filename)
sys.stdout.flush()
# XML WPS request
request = request_template.render(
begin_time=chunk_start_time,
end_time=chunk_end_time,
sampling_step=sampling,
variables=variables,
collection_ids=collections,
model_ids=models,
response_type="application/x-cdf",
)
# download the data
if not exists(output_filename):
tmp_filename = output_filename + ".tmp"
try:
with open(tmp_filename, "wb") as fout:
response_handler=ClientRequest._response_handler(fout)
with ProgressBarProcessing() as progress_bar:
wps_proxy.retrieve_async(
request.encode('UTF-8'),
handler=response_handler,
status_handler=progress_bar.update
)
except Exception as error:
if exists(tmp_filename):
remove(tmp_filename)
print("ERROR: %s faled: %s" % (output_filename, error), file=sys.stderr)
else:
if exists(tmp_filename):
rename(tmp_filename, output_filename)
else:
print("%s exists, skipped ... " % output_filename)
2016-01-01 00:00:00 2016-02-01 00:00:00 2016-01-01 00:00:00 2016-01-01 23:59:59.999999 2016-01-02 00:00:00 2016-01-02 23:59:59.999999 2016-01-03 00:00:00 2016-01-03 23:59:59.999999 2016-01-04 00:00:00 2016-01-04 23:59:59.999999 2016-01-05 00:00:00 2016-01-05 23:59:59.999999 2016-01-06 00:00:00 2016-01-06 23:59:59.999999 2016-01-07 00:00:00 2016-01-07 23:59:59.999999 2016-01-08 00:00:00 2016-01-08 23:59:59.999999 2016-01-09 00:00:00 2016-01-09 23:59:59.999999 2016-01-10 00:00:00 2016-01-10 23:59:59.999999 2016-01-11 00:00:00 2016-01-11 23:59:59.999999 2016-01-12 00:00:00 2016-01-12 23:59:59.999999 2016-01-13 00:00:00 2016-01-13 23:59:59.999999 2016-01-14 00:00:00 2016-01-14 23:59:59.999999 2016-01-15 00:00:00 2016-01-15 23:59:59.999999 2016-01-16 00:00:00 2016-01-16 23:59:59.999999 2016-01-17 00:00:00 2016-01-17 23:59:59.999999 2016-01-18 00:00:00 2016-01-18 23:59:59.999999 2016-01-19 00:00:00 2016-01-19 23:59:59.999999 2016-01-20 00:00:00 2016-01-20 23:59:59.999999 2016-01-21 00:00:00 2016-01-21 23:59:59.999999 2016-01-22 00:00:00 2016-01-22 23:59:59.999999 2016-01-23 00:00:00 2016-01-23 23:59:59.999999 2016-01-24 00:00:00 2016-01-24 23:59:59.999999 2016-01-25 00:00:00 2016-01-25 23:59:59.999999 2016-01-26 00:00:00 2016-01-26 23:59:59.999999 2016-01-27 00:00:00 2016-01-27 23:59:59.999999 2016-01-28 00:00:00 2016-01-28 23:59:59.999999 2016-01-29 00:00:00 2016-01-29 23:59:59.999999 2016-01-30 00:00:00 2016-01-30 23:59:59.999999 2016-01-31 00:00:00 2016-01-31 23:59:59.999999 request summary: output_filenames: VIRES_MAGA_LR_1B_20160101-000000_20160101-235959.cdf VIRES_MAGA_LR_1B_20160102-000000_20160102-235959.cdf VIRES_MAGA_LR_1B_20160103-000000_20160103-235959.cdf VIRES_MAGA_LR_1B_20160104-000000_20160104-235959.cdf VIRES_MAGA_LR_1B_20160105-000000_20160105-235959.cdf VIRES_MAGA_LR_1B_20160106-000000_20160106-235959.cdf VIRES_MAGA_LR_1B_20160107-000000_20160107-235959.cdf VIRES_MAGA_LR_1B_20160108-000000_20160108-235959.cdf VIRES_MAGA_LR_1B_20160109-000000_20160109-235959.cdf VIRES_MAGA_LR_1B_20160110-000000_20160110-235959.cdf VIRES_MAGA_LR_1B_20160111-000000_20160111-235959.cdf VIRES_MAGA_LR_1B_20160112-000000_20160112-235959.cdf VIRES_MAGA_LR_1B_20160113-000000_20160113-235959.cdf VIRES_MAGA_LR_1B_20160114-000000_20160114-235959.cdf VIRES_MAGA_LR_1B_20160115-000000_20160115-235959.cdf VIRES_MAGA_LR_1B_20160116-000000_20160116-235959.cdf VIRES_MAGA_LR_1B_20160117-000000_20160117-235959.cdf VIRES_MAGA_LR_1B_20160118-000000_20160118-235959.cdf VIRES_MAGA_LR_1B_20160119-000000_20160119-235959.cdf VIRES_MAGA_LR_1B_20160120-000000_20160120-235959.cdf VIRES_MAGA_LR_1B_20160121-000000_20160121-235959.cdf VIRES_MAGA_LR_1B_20160122-000000_20160122-235959.cdf VIRES_MAGA_LR_1B_20160123-000000_20160123-235959.cdf VIRES_MAGA_LR_1B_20160124-000000_20160124-235959.cdf VIRES_MAGA_LR_1B_20160125-000000_20160125-235959.cdf VIRES_MAGA_LR_1B_20160126-000000_20160126-235959.cdf VIRES_MAGA_LR_1B_20160127-000000_20160127-235959.cdf VIRES_MAGA_LR_1B_20160128-000000_20160128-235959.cdf VIRES_MAGA_LR_1B_20160129-000000_20160129-235959.cdf VIRES_MAGA_LR_1B_20160130-000000_20160130-235959.cdf VIRES_MAGA_LR_1B_20160131-000000_20160131-235959.cdf start time: 2016-01-01T00:00:00Z end time: 2016-02-01T00:00:00Z sampling: original data sampling collections: A: SW_OPER_MAGA_LR_1B models: CHAOS-6-Core CHAOS-6-Static CHAOS-6-MMA-Primary CHAOS-6-MMA-Secondary variables: B_NEC B_NEC_CHAOS-6-Core B_NEC_CHAOS-6-Static B_NEC_CHAOS-6-MMA-Primary B_NEC_CHAOS-6-MMA-Secondary Flags_B Flags_q Flags_Platform Kp Dst F10_INDEX OrbitNumber QDLat QDLon QDBasis MLT processing VIRES_MAGA_LR_1B_20160101-000000_20160101-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160102-000000_20160102-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160103-000000_20160103-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160104-000000_20160104-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160105-000000_20160105-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160106-000000_20160106-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160107-000000_20160107-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160108-000000_20160108-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:54, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160109-000000_20160109-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:54, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160110-000000_20160110-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160111-000000_20160111-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160112-000000_20160112-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160113-000000_20160113-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160114-000000_20160114-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160115-000000_20160115-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160116-000000_20160116-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160117-000000_20160117-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160118-000000_20160118-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160119-000000_20160119-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160120-000000_20160120-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:54, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160121-000000_20160121-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:54, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160122-000000_20160122-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160123-000000_20160123-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160124-000000_20160124-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160125-000000_20160125-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160126-000000_20160126-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160127-000000_20160127-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160128-000000_20160128-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160129-000000_20160129-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160130-000000_20160130-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)
processing VIRES_MAGA_LR_1B_20160131-000000_20160131-235959.cdf
Processing: 100%|██████████| [ Elapsed: 00:53, Remaining: 00:00 ]
Downloading: 100%|██████████| [ Elapsed: 00:00, Remaining: 00:00 ] (20.835MB)