Assess perfomance of the VirES magnetic model evaluation and other parameters wrt. the plain data rerival. Get per sample times.
See also VirES Python Client
from benchmark import get_random_collection, run_benchmark_set, write_record
SERVER_URL = "https://testing.vires.services"
FILENAME = "results/2022-11-14_testing_benchmark.log"
COLLECTIONS = ["SW_OPER_MAGA_LR_1B", "SW_OPER_MAGB_LR_1B", "SW_OPER_MAGC_LR_1B"]
MODELS = [
"CHAOS-Core",
"CHAOS-Static80 = 'CHAOS-Static'(max_degree=80)",
"CHAOS-Static",
"CHAOS-MMA-Primary",
"CHAOS-MMA-Secondary",
"CHAOS-MMA",
"CHAOS80 = 'CHAOS-Core' + 'CHAOS-Static'(max_degree=80) + 'CHAOS-MMA-Primary' + 'CHAOS-MMA-Secondary'",
"CHAOS",
"MIO_SHA_2C-Primary",
"MIO_SHA_2C-Secondary",
"MIO_SHA_2C",
]
TEST_CASES = [
dict(description="plain request"),
dict(description="plain request (cached)"),
dict(description="filter: Flags_B != 255", filters=["Flags_B != 255"]),
dict(description="aux.var.: MLT, QDLat", auxiliaries=["MLT", "QDLat"]),
*(
dict(description=f"model: { model.partition('=')[0].strip() }", models=[model])
for model in MODELS
),
]
N_RUNS = 10
SELECTION_TIMES = [
"P1D",
"PT6H",
"PT1H",
"PT5M",
]
with open(FILENAME, "a", encoding="utf8") as log_file:
for selection_time in SELECTION_TIMES:
for idx in range(N_RUNS):
print(f"{selection_time} pass #{idx+1}")
collection = get_random_collection(COLLECTIONS)
for record in run_benchmark_set(SERVER_URL, collection, selection_time, TEST_CASES):
write_record(record, log_file)
P1D pass #1 86400 6.54s plain request 86400 1.5s plain request (cached) 86400 1.53s filter: Flags_B != 255 86400 7.1s aux.var.: MLT, QDLat 86400 1.78s model: CHAOS-Core 86400 3.93s model: CHAOS-Static80 86400 13.9s model: CHAOS-Static 86400 7.35s model: CHAOS-MMA-Primary 86400 5.13s model: CHAOS-MMA-Secondary 86400 2.51s model: CHAOS-MMA 86400 4.05s model: CHAOS80 86400 14.2s model: CHAOS 86400 18.9s model: MIO_SHA_2C-Primary 86400 18.7s model: MIO_SHA_2C-Secondary 86400 35s model: MIO_SHA_2C P1D pass #2 86400 1.27s plain request 86400 1.32s plain request (cached) 86387 0.627s filter: Flags_B != 255 86400 1.11s aux.var.: MLT, QDLat 86400 1.91s model: CHAOS-Core 86400 3.01s model: CHAOS-Static80 86400 13.6s model: CHAOS-Static 86400 0.728s model: CHAOS-MMA-Primary 86400 3.18s model: CHAOS-MMA-Secondary 86400 0.748s model: CHAOS-MMA 86400 4.67s model: CHAOS80 86400 13.8s model: CHAOS 86400 17.7s model: MIO_SHA_2C-Primary 86400 18.5s model: MIO_SHA_2C-Secondary 86400 34.5s model: MIO_SHA_2C P1D pass #3 86400 1.72s plain request 86400 1.38s plain request (cached) 86400 1.21s filter: Flags_B != 255 86400 3.12s aux.var.: MLT, QDLat 86400 17s model: CHAOS-Core 86400 5.55s model: CHAOS-Static80 86400 14.4s model: CHAOS-Static 86400 1.5s model: CHAOS-MMA-Primary 86400 4.57s model: CHAOS-MMA-Secondary 86400 1.48s model: CHAOS-MMA 86400 4.77s model: CHAOS80 86400 14.5s model: CHAOS 86400 18s model: MIO_SHA_2C-Primary 86400 17.5s model: MIO_SHA_2C-Secondary 86400 33.6s model: MIO_SHA_2C P1D pass #4 86400 0.965s plain request 86400 0.524s plain request (cached) 86400 0.77s filter: Flags_B != 255 86400 0.968s aux.var.: MLT, QDLat 86400 2.1s model: CHAOS-Core 86400 3.01s model: CHAOS-Static80 86400 15.6s model: CHAOS-Static 86400 4.89s model: CHAOS-MMA-Primary 86400 0.755s model: CHAOS-MMA-Secondary 86400 0.821s model: CHAOS-MMA 86400 3.78s model: CHAOS80 86400 13.7s model: CHAOS 86400 17.5s model: MIO_SHA_2C-Primary 86400 17.9s model: MIO_SHA_2C-Secondary 86400 33.8s model: MIO_SHA_2C P1D pass #5 86400 1.29s plain request 86400 0.817s plain request (cached) 86400 0.597s filter: Flags_B != 255 86400 0.732s aux.var.: MLT, QDLat 86400 0.877s model: CHAOS-Core 86400 3.66s model: CHAOS-Static80 86400 13.3s model: CHAOS-Static 86400 0.899s model: CHAOS-MMA-Primary 86400 0.704s model: CHAOS-MMA-Secondary 86400 0.703s model: CHAOS-MMA 86400 3.68s model: CHAOS80 86400 13.6s model: CHAOS 86400 29.5s model: MIO_SHA_2C-Primary 86400 17.3s model: MIO_SHA_2C-Secondary 86400 34.1s model: MIO_SHA_2C P1D pass #6 86400 0.985s plain request 86400 0.571s plain request (cached) 86400 1.99s filter: Flags_B != 255 86400 1.23s aux.var.: MLT, QDLat 86400 2.68s model: CHAOS-Core 86400 3.04s model: CHAOS-Static80 86400 14.5s model: CHAOS-Static 86400 1.35s model: CHAOS-MMA-Primary 86400 4.67s model: CHAOS-MMA-Secondary 86400 0.764s model: CHAOS-MMA 86400 3.76s model: CHAOS80 86400 13.7s model: CHAOS 86400 17.5s model: MIO_SHA_2C-Primary 86400 18.4s model: MIO_SHA_2C-Secondary 86400 35.3s model: MIO_SHA_2C P1D pass #7 86400 1.26s plain request 86400 1.14s plain request (cached) 86400 0.541s filter: Flags_B != 255 86400 1.97s aux.var.: MLT, QDLat 86400 1.48s model: CHAOS-Core 86400 3.08s model: CHAOS-Static80 86400 13.4s model: CHAOS-Static 86400 1.57s model: CHAOS-MMA-Primary 86400 1.09s model: CHAOS-MMA-Secondary 86400 0.85s model: CHAOS-MMA 86400 3.34s model: CHAOS80 86400 13.9s model: CHAOS 86400 17.5s model: MIO_SHA_2C-Primary 86400 17.4s model: MIO_SHA_2C-Secondary 86400 34.2s model: MIO_SHA_2C P1D pass #8 86400 1.02s plain request 86400 1.88s plain request (cached) 86400 0.999s filter: Flags_B != 255 86400 0.876s aux.var.: MLT, QDLat 86400 1.02s model: CHAOS-Core 86400 2.93s model: CHAOS-Static80 86400 13.8s model: CHAOS-Static 86400 1.44s model: CHAOS-MMA-Primary 86400 1.02s model: CHAOS-MMA-Secondary 86400 0.764s model: CHAOS-MMA 86400 3.97s model: CHAOS80 86400 13.7s model: CHAOS 86400 17.4s model: MIO_SHA_2C-Primary 86400 17.7s model: MIO_SHA_2C-Secondary 86400 34.9s model: MIO_SHA_2C P1D pass #9 86400 0.907s plain request 86400 1.49s plain request (cached) 86400 1.17s filter: Flags_B != 255 86400 1.66s aux.var.: MLT, QDLat 86400 0.855s model: CHAOS-Core 86400 3.35s model: CHAOS-Static80 86400 14s model: CHAOS-Static 86400 0.927s model: CHAOS-MMA-Primary 86400 1.66s model: CHAOS-MMA-Secondary 86400 1.51s model: CHAOS-MMA 86400 3.36s model: CHAOS80 86400 14s model: CHAOS 86400 17.2s model: MIO_SHA_2C-Primary 86400 17.5s model: MIO_SHA_2C-Secondary 86400 34s model: MIO_SHA_2C P1D pass #10 86400 1.26s plain request 86400 0.553s plain request (cached) 86400 1.51s filter: Flags_B != 255 86400 1.12s aux.var.: MLT, QDLat 86400 1.57s model: CHAOS-Core 86400 3.73s model: CHAOS-Static80 86400 13.4s model: CHAOS-Static 86400 1.63s model: CHAOS-MMA-Primary 86400 0.642s model: CHAOS-MMA-Secondary 86400 1.4s model: CHAOS-MMA 86400 4.03s model: CHAOS80 86400 13.7s model: CHAOS 86400 17.4s model: MIO_SHA_2C-Primary 86400 19.5s model: MIO_SHA_2C-Secondary 86400 33.9s model: MIO_SHA_2C PT6H pass #1 21600 1.36s plain request 21600 1.09s plain request (cached) 21600 0.76s filter: Flags_B != 255 21600 0.707s aux.var.: MLT, QDLat 21600 0.688s model: CHAOS-Core 21600 1.1s model: CHAOS-Static80 21600 4.11s model: CHAOS-Static 21600 1.57s model: CHAOS-MMA-Primary 21600 0.542s model: CHAOS-MMA-Secondary 21600 0.878s model: CHAOS-MMA 21600 1.23s model: CHAOS80 21600 3.79s model: CHAOS 21600 4.6s model: MIO_SHA_2C-Primary 21600 5.4s model: MIO_SHA_2C-Secondary 21600 9.78s model: MIO_SHA_2C PT6H pass #2 21600 0.763s plain request 21600 0.563s plain request (cached) 21600 1.01s filter: Flags_B != 255 21600 1.3s aux.var.: MLT, QDLat 21600 0.525s model: CHAOS-Core 21600 1.03s model: CHAOS-Static80 21600 3.6s model: CHAOS-Static 21600 1.25s model: CHAOS-MMA-Primary 21600 1.18s model: CHAOS-MMA-Secondary 21600 0.83s model: CHAOS-MMA 21600 2.22s model: CHAOS80 21600 3.73s model: CHAOS 21600 4.61s model: MIO_SHA_2C-Primary 21600 5.29s model: MIO_SHA_2C-Secondary 21600 9.02s model: MIO_SHA_2C PT6H pass #3 21600 1.75s plain request 21600 0.984s plain request (cached) 21600 0.837s filter: Flags_B != 255 21600 0.545s aux.var.: MLT, QDLat 21600 0.621s model: CHAOS-Core 21600 1.68s model: CHAOS-Static80 21600 4.03s model: CHAOS-Static 21600 0.731s model: CHAOS-MMA-Primary 21600 0.718s model: CHAOS-MMA-Secondary 21600 1.04s model: CHAOS-MMA 21600 1.27s model: CHAOS80 21600 4.86s model: CHAOS 21600 5.65s model: MIO_SHA_2C-Primary 21600 5.38s model: MIO_SHA_2C-Secondary 21600 8.84s model: MIO_SHA_2C PT6H pass #4 21600 0.973s plain request 21600 0.888s plain request (cached) 21600 0.889s filter: Flags_B != 255 21600 1.12s aux.var.: MLT, QDLat 21600 0.521s model: CHAOS-Core 21600 1.75s model: CHAOS-Static80 21600 3.64s model: CHAOS-Static 21600 0.578s model: CHAOS-MMA-Primary 21600 1.3s model: CHAOS-MMA-Secondary 21600 1.53s model: CHAOS-MMA 21600 2.31s model: CHAOS80 21600 3.92s model: CHAOS 21600 5.34s model: MIO_SHA_2C-Primary 21600 4.62s model: MIO_SHA_2C-Secondary 21600 9.82s model: MIO_SHA_2C PT6H pass #5 21600 0.758s plain request 21600 0.583s plain request (cached) 21600 0.393s filter: Flags_B != 255 21600 0.889s aux.var.: MLT, QDLat 21600 1.09s model: CHAOS-Core 21600 1.03s model: CHAOS-Static80 21600 4.03s model: CHAOS-Static 21600 0.507s model: CHAOS-MMA-Primary 21600 1.12s model: CHAOS-MMA-Secondary 21600 0.711s model: CHAOS-MMA 21600 1.9s model: CHAOS80 21600 4.27s model: CHAOS 21600 5.28s model: MIO_SHA_2C-Primary 21600 4.64s model: MIO_SHA_2C-Secondary 21600 9.95s model: MIO_SHA_2C PT6H pass #6 21600 1.3s plain request 21600 1.1s plain request (cached) 21600 1.12s filter: Flags_B != 255 21600 0.463s aux.var.: MLT, QDLat 21600 0.535s model: CHAOS-Core 21600 1.44s model: CHAOS-Static80 21600 4.54s model: CHAOS-Static 21600 0.661s model: CHAOS-MMA-Primary 21600 0.542s model: CHAOS-MMA-Secondary 21600 0.583s model: CHAOS-MMA 21600 1.97s model: CHAOS80 21600 3.71s model: CHAOS 21600 4.69s model: MIO_SHA_2C-Primary 21600 5.59s model: MIO_SHA_2C-Secondary 21600 8.86s model: MIO_SHA_2C PT6H pass #7 21600 0.718s plain request 21600 0.981s plain request (cached) 21600 1.08s filter: Flags_B != 255 21600 0.798s aux.var.: MLT, QDLat 21600 0.594s model: CHAOS-Core 21600 2.15s model: CHAOS-Static80 21600 4.25s model: CHAOS-Static 21600 0.986s model: CHAOS-MMA-Primary 21600 0.506s model: CHAOS-MMA-Secondary 21600 0.615s model: CHAOS-MMA 21600 2.33s model: CHAOS80 21600 4.21s model: CHAOS 21600 4.8s model: MIO_SHA_2C-Primary 21600 5.04s model: MIO_SHA_2C-Secondary 21600 9.25s model: MIO_SHA_2C PT6H pass #8 21600 1.13s plain request 21600 0.872s plain request (cached) 21600 1.17s filter: Flags_B != 255 21600 1.18s aux.var.: MLT, QDLat 21600 0.504s model: CHAOS-Core 21600 2.14s model: CHAOS-Static80 21600 3.77s model: CHAOS-Static 21600 1.09s model: CHAOS-MMA-Primary 21600 0.613s model: CHAOS-MMA-Secondary 21600 1.68s model: CHAOS-MMA 21600 1.25s model: CHAOS80 21600 4.12s model: CHAOS 21600 5.32s model: MIO_SHA_2C-Primary 21600 4.69s model: MIO_SHA_2C-Secondary 21600 9.58s model: MIO_SHA_2C PT6H pass #9 21600 1.14s plain request 21600 0.784s plain request (cached) 21600 0.473s filter: Flags_B != 255 21600 1.16s aux.var.: MLT, QDLat 21600 0.975s model: CHAOS-Core 21600 1.47s model: CHAOS-Static80 21600 3.66s model: CHAOS-Static 21600 1.2s model: CHAOS-MMA-Primary 21600 0.531s model: CHAOS-MMA-Secondary 21600 1.38s model: CHAOS-MMA 21600 2.09s model: CHAOS80 21600 4.04s model: CHAOS 21600 5.5s model: MIO_SHA_2C-Primary 21600 5.63s model: MIO_SHA_2C-Secondary 21600 9.91s model: MIO_SHA_2C PT6H pass #10 21600 1.27s plain request 21600 0.424s plain request (cached) 21600 1.05s filter: Flags_B != 255 21600 1.18s aux.var.: MLT, QDLat 21600 0.449s model: CHAOS-Core 21600 1.87s model: CHAOS-Static80 21600 3.71s model: CHAOS-Static 21600 1.14s model: CHAOS-MMA-Primary 21600 0.413s model: CHAOS-MMA-Secondary 21600 0.744s model: CHAOS-MMA 21600 2.11s model: CHAOS80 21600 3.71s model: CHAOS 21600 4.67s model: MIO_SHA_2C-Primary 21600 4.69s model: MIO_SHA_2C-Secondary 21600 9.9s model: MIO_SHA_2C PT1H pass #1 3600 0.551s plain request 3600 0.424s plain request (cached) 3600 0.403s filter: Flags_B != 255 3600 0.393s aux.var.: MLT, QDLat 3600 0.389s model: CHAOS-Core 3600 0.59s model: CHAOS-Static80 3600 1.02s model: CHAOS-Static 3600 0.461s model: CHAOS-MMA-Primary 3600 0.545s model: CHAOS-MMA-Secondary 3600 0.438s model: CHAOS-MMA 3600 0.622s model: CHAOS80 3600 0.974s model: CHAOS 3600 1.16s model: MIO_SHA_2C-Primary 3600 1.17s model: MIO_SHA_2C-Secondary 3600 1.89s model: MIO_SHA_2C PT1H pass #2 3600 0.629s plain request 3600 0.43s plain request (cached) 3600 0.408s filter: Flags_B != 255 3600 0.446s aux.var.: MLT, QDLat 3600 0.449s model: CHAOS-Core 3600 0.528s model: CHAOS-Static80 3600 1.05s model: CHAOS-Static 3600 0.384s model: CHAOS-MMA-Primary 3600 0.42s model: CHAOS-MMA-Secondary 3600 0.443s model: CHAOS-MMA 3600 0.629s model: CHAOS80 3600 1.03s model: CHAOS
--------------------------------------------------------------------------- HTTPError Traceback (most recent call last) File /opt/conda/lib/python3.9/site-packages/viresclient/_wps/wps.py:317, in WPS10Service._retrieve(self, request, response_handler, error_handler) 316 try: --> 317 with closing(urlopen(request)) as file_in: 318 output = (response_handler or self._default_handler)(file_in) File /opt/conda/lib/python3.9/urllib/request.py:214, in urlopen(url, data, timeout, cafile, capath, cadefault, context) 213 opener = _opener --> 214 return opener.open(url, data, timeout) File /opt/conda/lib/python3.9/urllib/request.py:523, in OpenerDirector.open(self, fullurl, data, timeout) 522 meth = getattr(processor, meth_name) --> 523 response = meth(req, response) 525 return response File /opt/conda/lib/python3.9/urllib/request.py:632, in HTTPErrorProcessor.http_response(self, request, response) 631 if not (200 <= code < 300): --> 632 response = self.parent.error( 633 'http', request, response, code, msg, hdrs) 635 return response File /opt/conda/lib/python3.9/urllib/request.py:561, in OpenerDirector.error(self, proto, *args) 560 args = (dict, 'default', 'http_error_default') + orig_args --> 561 return self._call_chain(*args) File /opt/conda/lib/python3.9/urllib/request.py:494, in OpenerDirector._call_chain(self, chain, kind, meth_name, *args) 493 func = getattr(handler, meth_name) --> 494 result = func(*args) 495 if result is not None: File /opt/conda/lib/python3.9/urllib/request.py:641, in HTTPDefaultErrorHandler.http_error_default(self, req, fp, code, msg, hdrs) 640 def http_error_default(self, req, fp, code, msg, hdrs): --> 641 raise HTTPError(req.full_url, code, msg, hdrs, fp) HTTPError: HTTP Error 400: Bad Request During handling of the above exception, another exception occurred: WPSError Traceback (most recent call last) File /opt/conda/lib/python3.9/site-packages/viresclient/_client.py:491, in ClientRequest._get(self, request, asynchronous, response_handler, message, show_progress, leave_progress_bar) 490 else: --> 491 return self._wps_service.retrieve(request, handler=response_handler) 492 except WPSError: File /opt/conda/lib/python3.9/site-packages/viresclient/_wps/wps.py:127, in WPS10Service.retrieve(self, request, handler) 124 """Send a synchronous POST WPS request to a server and retrieve 125 the output. 126 """ --> 127 return self._retrieve( 128 Request(self.url, request, self.headers), handler, self.error_handler 129 ) File /opt/conda/lib/python3.9/site-packages/viresclient/_wps/wps.py:326, in WPS10Service._retrieve(self, request, response_handler, error_handler) 325 if error_handler: --> 326 return error_handler(error) 327 raise File /opt/conda/lib/python3.9/site-packages/viresclient/_wps/wps.py:289, in WPS10Service.error_handler(cls, http_error) 288 raise http_error --> 289 raise cls.parse_ows_exception(ows_exception, namespace) WPSError: WPS Request Failed! Reason: NoApplicableCode: Incompatible dimensions of the time and reshaped variable arrays. During handling of the above exception, another exception occurred: RuntimeError Traceback (most recent call last) Input In [1], in <module> 46 print(f"{selection_time} pass #{idx+1}") 47 collection = get_random_collection(COLLECTIONS) ---> 48 for record in run_benchmark_set(SERVER_URL, collection, selection_time, TEST_CASES): 49 write_record(record, log_file) File ~/jupyter_notebooks/MAG/MAG_LR_model_benchmark/benchmark.py:112, in run_benchmark_set(url, collection, selection_time, test_cases) 105 common_options = { 106 "url": url, 107 "collection": collection, 108 "start_time": start_time, 109 "end_time": end_time, 110 } 111 for options in test_cases: --> 112 yield run_benchmark(**common_options, **options) File ~/jupyter_notebooks/MAG/MAG_LR_model_benchmark/benchmark.py:77, in run_benchmark(url, description, collection, start_time, end_time, **options) 76 def run_benchmark(url, description, collection, start_time, end_time, **options): ---> 77 elapsed_time, size = measure_request_duration( 78 url=url, 79 collection=collection, 80 start_time=start_time, 81 end_time=end_time, 82 **options 83 ) 84 print(f"{size} {elapsed_time:.3g}s {description}") 85 return { 86 "timestamp": datetime.datetime.now(TZ_UTC), 87 "serverURL": url, (...) 93 "description": description, 94 } File ~/jupyter_notebooks/MAG/MAG_LR_model_benchmark/benchmark.py:50, in measure_request_duration(url, collection, start_time, end_time, auxiliaries, models, filters) 48 request.add_filter(filter_) 49 request_start = time.perf_counter_ns() ---> 50 data = request.get_between( 51 start_time=start_time, 52 end_time=end_time, 53 asynchronous=False, 54 show_progress=False, 55 ) 56 request_stop = time.perf_counter_ns() 57 request_duration = (request_stop - request_start) * 1e-9 File /opt/conda/lib/python3.9/site-packages/viresclient/_client.py:639, in ClientRequest.get_between(self, start_time, end_time, filetype, asynchronous, show_progress, show_progress_chunks, leave_intermediate_progress_bars, nrecords_limit, tmpdir) 637 _get_chunk(i, start_time_i, end_time_i) 638 else: --> 639 _get_chunk(0, start_time, end_time, leave_progress_bar=True) 641 return retdatagroup File /opt/conda/lib/python3.9/site-packages/viresclient/_client.py:609, in ClientRequest.get_between.<locals>._get_chunk(i, start_time_i, end_time_i, leave_progress_bar) 602 # Make the request, as either asynchronous or synchronous 603 # The response handler streams the data to the ReturnedData object 604 response_handler = self._response_handler( 605 retdatafile, 606 show_progress=show_progress, 607 leave_progress_bar=leave_progress_bar, 608 ) --> 609 self._get( 610 request=self._request, 611 asynchronous=asynchronous, 612 response_handler=response_handler, 613 message=message, 614 show_progress=show_progress, 615 leave_progress_bar=leave_progress_bar, 616 ) File /opt/conda/lib/python3.9/site-packages/viresclient/_client.py:493, in ClientRequest._get(self, request, asynchronous, response_handler, message, show_progress, leave_progress_bar) 491 return self._wps_service.retrieve(request, handler=response_handler) 492 except WPSError: --> 493 raise RuntimeError( 494 "Server error. Or perhaps the request is invalid? " 495 "Check the output of: print(request) and " 496 "print(request._request.decode())" 497 ) 498 except AuthenticationError: 499 raise AuthenticationError(AUTH_ERROR_TEXT) RuntimeError: Server error. Or perhaps the request is invalid? Check the output of: print(request) and print(request._request.decode())