Performance of concat operator on cmip6 decadal: DWD¶

Rooki calls climate data operations on the rook processing service.

In [1]:

import os
import time
os.environ['ROOK_URL'] = 'http://rook.dkrz.de/wps'

from rooki import operators as ops

Load intake catalog for cmip6 decadal¶

In [2]:

import intake

In [3]:

cat_url = "https://raw.githubusercontent.com/cp4cds/c3s_34g_manifests/master/intake/catalogs/c3s.yaml"

cat = intake.open_catalog(cat_url)
df_cmip6_decadal = cat['c3s-cmip6-decadal'].read()
df_cmip6_decadal.head()

/usr/local/Caskroom/mambaforge/base/envs/rooki/lib/python3.11/site-packages/dask/dataframe/io/csv.py:542: UserWarning: Warning gzip compression does not support breaking apart files
Please ensure that each individual file can fit in memory and
use the keyword ``blocksize=None to remove this message``
Setting ``blocksize=None``
  warn(

Out[3]:

	ds_id	path	size	mip_era	activity_id	institution_id	source_id	experiment_id	member_id	table_id	variable_id	grid_label	version	start_time	end_time	bbox	level	member_year	member_ensemble	prefix
0	c3s-cmip6-decadal.DCPP.MPI-M.MPI-ESM1-2-HR.dcp...	DCPP/MPI-M/MPI-ESM1-2-HR/dcppA-hindcast/s2016-...	492479692	c3s-cmip6-decadal	DCPP	MPI-M	MPI-ESM1-2-HR	dcppA-hindcast	s2016-r8i1p1f1	day	tasmax	gn	v20210111	2016-11-01T12:00:00	2026-12-31T12:00:00	0.00, -89.28, 359.06, 89.28	2.00	s2016	r8i1p1f1	c3s-cmip6-decadal
1	c3s-cmip6-decadal.DCPP.MPI-M.MPI-ESM1-2-HR.dcp...	DCPP/MPI-M/MPI-ESM1-2-HR/dcppA-hindcast/s2009-...	737204109	c3s-cmip6-decadal	DCPP	MPI-M	MPI-ESM1-2-HR	dcppA-hindcast	s2009-r8i1p1f1	day	pr	gn	v20210107	2009-11-01T12:00:00	2019-12-31T12:00:00	0.00, -89.28, 359.06, 89.28	NaN	s2009	r8i1p1f1	c3s-cmip6-decadal
2	c3s-cmip6-decadal.DCPP.MOHC.HadGEM3-GC31-MM.dc...	DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/s1987...	953384	c3s-cmip6-decadal	DCPP	MOHC	HadGEM3-GC31-MM	dcppA-hindcast	s1987-r3i1p1f2	Amon	pr	gn	v20200417	1987-11-16T00:00:00	1987-12-16T00:00:00	0.42, -89.72, 359.58, 89.72	NaN	s1987	r3i1p1f2	c3s-cmip6-decadal
3	c3s-cmip6-decadal.DCPP.MOHC.HadGEM3-GC31-MM.dc...	DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/s1995...	603757	c3s-cmip6-decadal	DCPP	MOHC	HadGEM3-GC31-MM	dcppA-hindcast	s1995-r6i1p1f2	Amon	tas	gn	v20200417	1995-11-16T00:00:00	1995-12-16T00:00:00	0.42, -89.72, 359.58, 89.72	1.50	s1995	r6i1p1f2	c3s-cmip6-decadal
4	c3s-cmip6-decadal.DCPP.MOHC.HadGEM3-GC31-MM.dc...	DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/s2004...	14824913	c3s-cmip6-decadal	DCPP	MOHC	HadGEM3-GC31-MM	dcppA-hindcast	s2004-r10i1p1f2	AERday	zg500	gn	v20200417	2004-11-01T12:00:00	2004-12-30T12:00:00	0.42, -89.72, 359.58, 89.72	50000.00	s2004	r10i1p1f2	c3s-cmip6-decadal

In [4]:

df = df_cmip6_decadal.loc[
    (df_cmip6_decadal.variable_id=="pr") 
    & (df_cmip6_decadal.experiment_id=="dcppA-hindcast")
    & (df_cmip6_decadal.table_id=="Amon")
    & (df_cmip6_decadal.source_id=="MPI-ESM1-2-LR")
    & (df_cmip6_decadal.member_year=="s2018")
]
dsets = list(df.ds_id.unique())
dsets

Out[4]:

['c3s-cmip6-decadal.DCPP.DWD.MPI-ESM1-2-LR.dcppA-hindcast.s2018-r5i1p1f1.Amon.pr.gn.v20220126',
 'c3s-cmip6-decadal.DCPP.DWD.MPI-ESM1-2-LR.dcppA-hindcast.s2018-r2i1p1f1.Amon.pr.gn.v20220126',
 'c3s-cmip6-decadal.DCPP.DWD.MPI-ESM1-2-LR.dcppA-hindcast.s2018-r6i1p1f1.Amon.pr.gn.v20220126',
 'c3s-cmip6-decadal.DCPP.DWD.MPI-ESM1-2-LR.dcppA-hindcast.s2018-r8i1p1f1.Amon.pr.gn.v20220126',
 'c3s-cmip6-decadal.DCPP.DWD.MPI-ESM1-2-LR.dcppA-hindcast.s2018-r14i1p1f1.Amon.pr.gn.v20220126',
 'c3s-cmip6-decadal.DCPP.DWD.MPI-ESM1-2-LR.dcppA-hindcast.s2018-r4i1p1f1.Amon.pr.gn.v20220126',
 'c3s-cmip6-decadal.DCPP.DWD.MPI-ESM1-2-LR.dcppA-hindcast.s2018-r11i1p1f1.Amon.pr.gn.v20220126',
 'c3s-cmip6-decadal.DCPP.DWD.MPI-ESM1-2-LR.dcppA-hindcast.s2018-r15i1p1f1.Amon.pr.gn.v20220126',
 'c3s-cmip6-decadal.DCPP.DWD.MPI-ESM1-2-LR.dcppA-hindcast.s2018-r10i1p1f1.Amon.pr.gn.v20220126',
 'c3s-cmip6-decadal.DCPP.DWD.MPI-ESM1-2-LR.dcppA-hindcast.s2018-r13i1p1f1.Amon.pr.gn.v20220126',
 'c3s-cmip6-decadal.DCPP.DWD.MPI-ESM1-2-LR.dcppA-hindcast.s2018-r12i1p1f1.Amon.pr.gn.v20220126',
 'c3s-cmip6-decadal.DCPP.DWD.MPI-ESM1-2-LR.dcppA-hindcast.s2018-r7i1p1f1.Amon.pr.gn.v20220126',
 'c3s-cmip6-decadal.DCPP.DWD.MPI-ESM1-2-LR.dcppA-hindcast.s2018-r16i1p1f1.Amon.pr.gn.v20220126',
 'c3s-cmip6-decadal.DCPP.DWD.MPI-ESM1-2-LR.dcppA-hindcast.s2018-r1i1p1f1.Amon.pr.gn.v20220126',
 'c3s-cmip6-decadal.DCPP.DWD.MPI-ESM1-2-LR.dcppA-hindcast.s2018-r3i1p1f1.Amon.pr.gn.v20220126',
 'c3s-cmip6-decadal.DCPP.DWD.MPI-ESM1-2-LR.dcppA-hindcast.s2018-r9i1p1f1.Amon.pr.gn.v20220126']

Run concat¶

https://docs.xarray.dev/en/stable/generated/xarray.concat.html

In [5]:

start = time.time()

t = "2019/2019"
#tc = "month:dec|day:01"

wf = ops.Concat(
        ops.Input('ds', dsets),
        time=t,
        #time_components=tc,
        dims="realization",
        #apply_average=1
)

# optional average
wf = ops.Average(wf, dims="realization")

# optional subset
wf = ops.Subset(
    wf, time=t, 
    # time_components=tc
)

resp = wf.orchestrate()

end = time.time()
print(f"duration: {int(end - start)} secs")
print(resp)

resp.ok

duration: 12 secs
Metalink URL: http://rook4.cloud.dkrz.de:80/outputs/rook/cbe2f91a-140c-11ee-be07-fa163ed6c06f/input.meta4, num files: 1

Out[5]:

True

open with xarray

In [6]:

resp.size_in_mb

Out[6]:

0.8807840347290039

In [7]:

resp.download_urls()

Out[7]:

['http://rook4.cloud.dkrz.de:80/outputs/rook/d1b99826-140c-11ee-a82b-fa163ed6c06f/pr_Amon_MPI-ESM1-2-LR_dcppA-hindcast_r10i1p1f1_gn_20190131-20191231.nc']

In [8]:

dsets = resp.datasets()

Downloading to /var/folders/qb/mg0csz190wd4rxybhhnwjln80000gn/T/metalink_3_p_smm3/pr_Amon_MPI-ESM1-2-LR_dcppA-hindcast_r10i1p1f1_gn_20190131-20191231.nc.

In [9]:

ds = dsets[0]
ds

provenance

In [10]:

prov_plot_url = resp.provenance_image()
prov_plot_url

Out[10]:

'http://rook4.cloud.dkrz.de:80/outputs/rook/cbe2f91a-140c-11ee-be07-fa163ed6c06f/provenance.png'

In [11]:

# from IPython.display import Image
# Image(prov_plot_url)

In [12]:

resp.provenance()

Out[12]:

'http://rook4.cloud.dkrz.de:80/outputs/rook/cbe2f91a-140c-11ee-be07-fa163ed6c06f/provenance.json'