%%time
import glob
import json
import os
os.sys.path.insert(0, "/p/user_pub/xclim/persist/software/xsearch/")
import xcdat as xc
import xsearch as xs
import logging
logging.getLogger("xcdat").setLevel(logging.ERROR)
CPU times: user 5.92 s, sys: 1.62 s, total: 7.54 s Wall time: 5.89 s
%%time
path = "/home/durack1/git/CMIP6_CVs"
file = "CMIP6_source_id.json"
with open(os.path.join(path, file), 'r') as f:
js = json.load(f)
js.keys()
srcIds = js["source_id"]
srcIds.keys()
CPU times: user 1.15 ms, sys: 2.04 ms, total: 3.19 ms Wall time: 13 ms
dict_keys(['4AOP-v1-5', 'ACCESS-CM2', 'ACCESS-ESM1-5', 'ACCESS-OM2', 'ACCESS-OM2-025', 'ARTS-2-3', 'AWI-CM-1-1-HR', 'AWI-CM-1-1-LR', 'AWI-CM-1-1-MR', 'AWI-ESM-1-1-LR', 'AWI-ESM-1-REcoM', 'AWI-ESM-2-1-LR', 'BCC-CSM2-HR', 'BCC-CSM2-MR', 'BCC-ESM1', 'CAM-MPAS-HR', 'CAM-MPAS-LR', 'CAMS-CSM1-0', 'CAS-ESM2-0', 'CESM1-1-CAM5-CMIP5', 'CESM1-CAM5-SE-HR', 'CESM1-CAM5-SE-LR', 'CESM1-WACCM-SC', 'CESM2', 'CESM2-FV2', 'CESM2-WACCM', 'CESM2-WACCM-FV2', 'CIESM', 'CMCC-CM2-HR4', 'CMCC-CM2-SR5', 'CMCC-CM2-VHR4', 'CMCC-ESM2', 'CNRM-CM6-1', 'CNRM-CM6-1-HR', 'CNRM-ESM2-1', 'CanESM5', 'CanESM5-1', 'CanESM5-CanOE', 'E3SM-1-0', 'E3SM-1-1', 'E3SM-1-1-ECA', 'E3SM-2-0', 'E3SM-2-0-NARRM', 'E3SM-2-1', 'EC-Earth3', 'EC-Earth3-AerChem', 'EC-Earth3-CC', 'EC-Earth3-ESM-1', 'EC-Earth3-GrIS', 'EC-Earth3-HR', 'EC-Earth3-LR', 'EC-Earth3-Veg', 'EC-Earth3-Veg-LR', 'EC-Earth3P', 'EC-Earth3P-HR', 'EC-Earth3P-VHR', 'ECMWF-IFS-HR', 'ECMWF-IFS-LR', 'ECMWF-IFS-MR', 'FGOALS-f3-H', 'FGOALS-f3-L', 'FGOALS-g3', 'FIO-ESM-2-0', 'GFDL-AM4', 'GFDL-CM4', 'GFDL-CM4C192', 'GFDL-ESM2M', 'GFDL-ESM4', 'GFDL-GRTCODE', 'GFDL-OM4p5B', 'GFDL-RFM-DISORT', 'GISS-E2-1-G', 'GISS-E2-1-G-CC', 'GISS-E2-1-H', 'GISS-E2-2-G', 'GISS-E2-2-H', 'GISS-E3-G', 'HadGEM3-GC31-HH', 'HadGEM3-GC31-HM', 'HadGEM3-GC31-LL', 'HadGEM3-GC31-LM', 'HadGEM3-GC31-MH', 'HadGEM3-GC31-MM', 'HiRAM-SIT-HR', 'HiRAM-SIT-LR', 'ICON-ESM-LR', 'IITM-ESM', 'INM-CM4-8', 'INM-CM5-0', 'INM-CM5-H', 'IPSL-CM5A2-INCA', 'IPSL-CM6A-ATM-HR', 'IPSL-CM6A-ATM-ICO-HR', 'IPSL-CM6A-ATM-ICO-LR', 'IPSL-CM6A-ATM-ICO-MR', 'IPSL-CM6A-ATM-ICO-VHR', 'IPSL-CM6A-ATM-LR-REPROBUS', 'IPSL-CM6A-LR', 'IPSL-CM6A-LR-INCA', 'IPSL-CM6A-MR1', 'KACE-1-0-G', 'KIOST-ESM', 'LBLRTM-12-8', 'MCM-UA-1-0', 'MIROC-ES2H', 'MIROC-ES2H-NB', 'MIROC-ES2L', 'MIROC6', 'MPI-ESM-1-2-HAM', 'MPI-ESM1-2-HR', 'MPI-ESM1-2-LR', 'MPI-ESM1-2-XR', 'MRI-AGCM3-2-H', 'MRI-AGCM3-2-S', 'MRI-ESM2-0', 'NESM3', 'NICAM16-7S', 'NICAM16-8S', 'NICAM16-9S', 'NorCPM1', 'NorESM1-F', 'NorESM2-LM', 'NorESM2-MM', 'PCMDI-test-1-0', 'RRTMG-LW-4-91', 'RRTMG-SW-4-02', 'RTE-RRTMGP-181204', 'SAM0-UNICON', 'TaiESM1', 'TaiESM1-TIMCOM', 'TaiESM1-TIMCOM2', 'UKESM1-0-LL', 'UKESM1-1-LL', 'UKESM1-ice-LL'])
native = {}
for count, srcId in enumerate(srcIds):
tmp = srcIds[srcId]["model_component"]["ocean"]["description"]
tmp = tmp.split(";")
for val in tmp:
if "levels" in val:
#print(srcId, val[:3])
native[srcId] = int(val[:3])
%%time
dpaths = xs.findPaths('historical', 'thetao', 'mon', realm='ocean', cmipTable='Omon', deduplicate=True)
#for count, sim in enumerate(dpaths):
# print("{:03d}".format(count), sim)
#print(dpaths.keys())
#sims = xs.getValuesForFacet(dpaths, 'model', 'IPSL-CM5A2-INCA') ## 'E3SM-2-0') # also E3SM-1-0 (E3SM-Project (1-5), UCSB, E3SM-Project), E3SM-1-1 + ECA, E3SM-2-0-NARRM
#sims
CPU times: user 49.3 ms, sys: 5.09 ms, total: 54.4 ms Wall time: 95.2 ms
%%time
exp = "historical"
var = "thetao"
freq = "mon"
cmipTable = "Omon"
mipEra = "CMIP6"
firstMemberOnly = True
dpaths = xs.findPaths(exp, var, freq, cmipTable=cmipTable, mip_era=mipEra)
models = xs.natural_sort(xs.getGroupValues(dpaths, 'model'))
#print('models:', models)
#print('number of models:', len(models))
badFilesTime = [
"/p/css03/esgf_publish/CMIP6/CMIP/EC-Earth-Consortium/EC-Earth3-AerChem/historical/r1i1p1f1/Omon/thetao/gn/v20200624/thetao_Omon_EC-Earth3-AerChem_historical_r1i1p1f1_gn_185001-185012.nc",
"/p/css03/esgf_publish/CMIP6/CMIP/EC-Earth-Consortium/EC-Earth3-CC/historical/r1i1p1f1/Omon/thetao/gn/v20210113/thetao_Omon_EC-Earth3-CC_historical_r1i1p1f1_gn_185001-185012.nc",
"/p/css03/esgf_publish/CMIP6/CMIP/EC-Earth-Consortium/EC-Earth3-Veg-LR/historical/r1i1p1f1/Omon/thetao/gn/v20200919/thetao_Omon_EC-Earth3-Veg-LR_historical_r1i1p1f1_gn_185001-185012.nc",
"/p/css03/esgf_publish/CMIP6/CMIP/EC-Earth-Consortium/EC-Earth3-Veg/historical/r1i1p1f1/Omon/thetao/gn/v20211207/thetao_Omon_EC-Earth3-Veg_historical_r1i1p1f1_gn_185001-185012.nc",
"/p/css03/scratch/cmip6/CMIP/CAS/FGOALS-f3-L/historical/r1i1p1f1/Omon/thetao/gn/v20191007/thetao_Omon_FGOALS-f3-L_historical_r1i1p1f1_gn_185001-189912.nc",
];
aveLvls = []
for count, model in enumerate(models):
dpaths_model = xs.retainDataByFacetValue(dpaths, 'model', model)
members = xs.natural_sort(xs.getGroupValues(dpaths_model, 'member'))
if firstMemberOnly:
members = members[0:1]
#print(count, model, members)
for member in members:
dpaths_model_member_list = xs.getValuesForFacet(dpaths_model, 'member', member)
if len(dpaths_model_member_list) > 1:
print('Error: multiple paths detected for ', model, member, ': ', dpaths_model_member_list)
else:
dpath = dpaths_model_member_list[0]
ncfile = xs.natural_sort(glob.glob(os.path.join(dpath, '*.nc')))[0]
# now poll vertical axis
ds = xc.open_dataset(ncfile, decode_times=False)
z = xc.get_dim_coords(ds, "Z")
natZ = int(native[model])
if natZ < len(z):
s = "-- misreported --"
elif len(z) != natZ:
s = "-- decimated --"
else:
s = ""
print(count, model, members, "output:", len(z), "reported native:", natZ, s)
aveLvls.append(len(z))
print(aveLvls)
0 ACCESS-CM2 ['r1i1p1f1'] output: 50 reported native: 50 1 ACCESS-ESM1-5 ['r1i1p1f1'] output: 50 reported native: 50 2 AWI-CM-1-1-MR ['r1i1p1f1'] output: 46 reported native: 46 3 AWI-ESM-1-1-LR ['r1i1p1f1'] output: 46 reported native: 46 4 BCC-CSM2-MR ['r1i1p1f1'] output: 40 reported native: 40 5 BCC-ESM1 ['r1i1p1f1'] output: 40 reported native: 40 6 CAMS-CSM1-0 ['r1i1p1f1'] output: 50 reported native: 50 7 CanESM5 ['r1i1p1f1'] output: 45 reported native: 45 8 CanESM5-1 ['r1i1p1f1'] output: 45 reported native: 45 9 CanESM5-CanOE ['r1i1p2f1'] output: 45 reported native: 45 10 CAS-ESM2-0 ['r1i1p1f1'] output: 30 reported native: 30 11 CESM2 ['r1i1p1f1'] output: 33 reported native: 60 -- decimated --
/home/durack1/mambaforge/envs/xcd061nctax/lib/python3.11/site-packages/xarray/conventions.py:436: SerializationWarning: variable 'thetao' has multiple fill values {1e+20, 1e+20}, decoding all values to NaN. new_vars[k] = decode_cf_variable(
12 CESM2-FV2 ['r1i1p1f1'] output: 33 reported native: 60 -- decimated -- 13 CESM2-WACCM ['r1i1p1f1'] output: 33 reported native: 60 -- decimated -- 14 CESM2-WACCM-FV2 ['r1i1p1f1'] output: 33 reported native: 60 -- decimated -- 15 CIESM ['r1i1p1f1'] output: 60 reported native: 46 -- misreported -- 16 CMCC-CM2-HR4 ['r1i1p1f1'] output: 50 reported native: 50 17 CMCC-CM2-SR5 ['r1i1p1f1'] output: 50 reported native: 50 18 CMCC-ESM2 ['r1i1p1f1'] output: 50 reported native: 50 19 CNRM-CM6-1 ['r1i1p1f2'] output: 75 reported native: 75 20 CNRM-CM6-1-HR ['r1i1p1f2'] output: 75 reported native: 75 21 CNRM-ESM2-1 ['r1i1p1f2'] output: 75 reported native: 75 22 E3SM-1-0 ['r1i1p1f1'] output: 60 reported native: 60 23 E3SM-1-1 ['r1i1p1f1'] output: 60 reported native: 60 24 E3SM-1-1-ECA ['r1i1p1f1'] output: 60 reported native: 60 25 E3SM-2-0 ['r1i1p1f1'] output: 60 reported native: 60 26 E3SM-2-0-NARRM ['r1i1p1f1'] output: 60 reported native: 60 27 EC-Earth3 ['r1i1p1f1'] output: 75 reported native: 75 28 EC-Earth3-AerChem ['r1i1p1f1'] output: 75 reported native: 75 29 EC-Earth3-CC ['r1i1p1f1'] output: 75 reported native: 75 30 EC-Earth3-Veg ['r1i1p1f1'] output: 75 reported native: 75 31 EC-Earth3-Veg-LR ['r1i1p1f1'] output: 75 reported native: 75 32 FGOALS-f3-L ['r1i1p1f1'] output: 30 reported native: 30 33 FGOALS-g3 ['r1i1p1f1'] output: 30 reported native: 30 34 FIO-ESM-2-0 ['r1i1p1f1'] output: 60 reported native: 60 35 GFDL-CM4 ['r1i1p1f1'] output: 35 reported native: 75 -- decimated -- 36 GFDL-ESM4 ['r1i1p1f1'] output: 35 reported native: 75 -- decimated -- 37 GISS-E2-1-G ['r1i1p1f1'] output: 40 reported native: 40 38 GISS-E2-1-H ['r1i1p1f1'] output: 33 reported native: 33 39 GISS-E2-2-G ['r1i1p1f1'] output: 40 reported native: 40 40 GISS-E2-2-H ['r1i1p1f1'] output: 33 reported native: 33 41 HadGEM3-GC31-LL ['r1i1p1f3'] output: 75 reported native: 75 42 HadGEM3-GC31-MM ['r1i1p1f3'] output: 75 reported native: 75 43 ICON-ESM-LR ['r1i1p1f1'] output: 64 reported native: 40 -- misreported -- 44 INM-CM4-8 ['r1i1p1f1'] output: 33 reported native: 40 -- decimated -- 45 INM-CM5-0 ['r1i1p1f1'] output: 33 reported native: 40 -- decimated -- 46 IPSL-CM5A2-INCA ['r1i1p1f1'] output: 31 reported native: 31 47 IPSL-CM6A-LR ['r1i1p1f1'] output: 75 reported native: 75 48 IPSL-CM6A-LR-INCA ['r1i1p1f1'] output: 75 reported native: 75 49 KIOST-ESM ['r1i1p1f1'] output: 52 reported native: 52 50 MCM-UA-1-0 ['r1i1p1f1'] output: 18 reported native: 18 51 MIROC6 ['r1i1p1f1'] output: 63 reported native: 63 52 MIROC-ES2H ['r1i1p4f2'] output: 63 reported native: 63 53 MIROC-ES2L ['r1i1p1f2'] output: 63 reported native: 63 54 MPI-ESM1-2-HR ['r1i1p1f1'] output: 40 reported native: 40 55 MPI-ESM1-2-LR ['r1i1p1f1'] output: 40 reported native: 40 56 MPI-ESM-1-2-HAM ['r1i1p1f1'] output: 40 reported native: 40 57 MRI-ESM2-0 ['r1i1p1f1'] output: 61 reported native: 61 58 NESM3 ['r1i1p1f1'] output: 46 reported native: 46 59 NorCPM1 ['r1i1p1f1'] output: 35 reported native: 53 -- decimated -- 60 NorESM2-LM ['r1i1p1f1'] output: 70 reported native: 70 61 NorESM2-MM ['r1i1p1f1'] output: 53 reported native: 70 -- decimated -- 62 SAM0-UNICON ['r1i1p1f1'] output: 60 reported native: 60 63 TaiESM1 ['r1i1p1f1'] output: 60 reported native: 60 64 UKESM1-0-LL ['r1i1p1f2'] output: 75 reported native: 75 65 UKESM1-1-LL ['r1i1p1f2'] output: 75 reported native: 75 [50, 50, 46, 46, 40, 40, 50, 45, 45, 45, 30, 33, 33, 33, 33, 60, 50, 50, 50, 75, 75, 75, 60, 60, 60, 60, 60, 75, 75, 75, 75, 75, 30, 30, 60, 35, 35, 40, 33, 40, 33, 75, 75, 64, 33, 33, 31, 75, 75, 52, 18, 63, 63, 63, 40, 40, 40, 61, 46, 35, 70, 53, 60, 60, 75, 75] CPU times: user 2.37 s, sys: 616 ms, total: 2.99 s Wall time: 41.3 s
print("Average ocean levels:", (sum(aveLvls) // len(aveLvls) + 1))
Average ocean levels: 53
native = {}
counter = 1
for count, srcId in enumerate(srcIds):
tmp = srcIds[srcId]["model_component"]["atmos"]["description"]
if tmp == "none":
print(count, srcId, "none")
continue
tmp = tmp.split(";")
for val in tmp:
#print(srcId, val)
if "levels" in val:
#print(count, val)
native[srcId] = int(val[:3])
elif "72 vertical layers" in val: # fudge E3SM-2-0, E3SM-2-1
#print(count, srcId, val)
native[srcId] = 72
elif ". 72 vertical layers " in val: # E3SM-2-0-NARRM
#print(count, srcId, val)
native[srcId] = 72
0 4AOP-v1-5 none 3 ACCESS-OM2 none 4 ACCESS-OM2-025 none 5 ARTS-2-3 none 68 GFDL-GRTCODE none 69 GFDL-OM4p5B none 70 GFDL-RFM-DISORT none 102 LBLRTM-12-8 none 124 RRTMG-LW-4-91 none 125 RRTMG-SW-4-02 none 126 RTE-RRTMGP-181204 none
%%time
exp = "historical"
var = "cl"
freq = "mon"
cmipTable = "Amon"
mipEra = "CMIP6"
firstMemberOnly = True
dpaths = xs.findPaths(exp, var, freq, cmipTable=cmipTable, mip_era=mipEra)
models = xs.natural_sort(xs.getGroupValues(dpaths, 'model'))
#print('models:', models)
#print('number of models:', len(models))
badFilesTime = [
];
aveLvls = []
for count, model in enumerate(models):
#if model in ["E3SM-2-0", "E3SM-2-0-NARRM"]:
# continue # problem with variable
dpaths_model = xs.retainDataByFacetValue(dpaths, 'model', model)
members = xs.natural_sort(xs.getGroupValues(dpaths_model, 'member'))
if firstMemberOnly:
members = members[0:1]
#print(count, model, members)
for member in members:
dpaths_model_member_list = xs.getValuesForFacet(dpaths_model, 'member', member)
if len(dpaths_model_member_list) > 1:
print('Error: multiple paths detected for ', model, member, ': ', dpaths_model_member_list)
else:
dpath = dpaths_model_member_list[0]
ncfile = xs.natural_sort(glob.glob(os.path.join(dpath, '*.nc')))[0]
# now poll vertical axis
ds = xc.open_dataset(ncfile, decode_times=False)
# deal with IPSL-CM6A-LR
if model == "IPSL-CM6A-LR":
z = ds["presnivs"]
else:
z = xc.get_dim_coords(ds, "Z")
natZ = int(native[model])
if natZ < len(z):
s = "-- misreported --"
elif len(z) != natZ:
s = "-- decimated --"
else:
s = ""
print(count, model, members, "output:", len(z), "reported native:", natZ, s)
aveLvls.append(len(z))
print(aveLvls)
0 ACCESS-CM2 ['r1i1p1f1'] output: 85 reported native: 85 1 ACCESS-ESM1-5 ['r1i1p1f1'] output: 38 reported native: 38 2 AWI-ESM-1-1-LR ['r1i1p1f1'] output: 47 reported native: 47 3 BCC-CSM2-MR ['r1i1p1f1'] output: 46 reported native: 46 4 BCC-ESM1 ['r1i1p1f1'] output: 26 reported native: 26 5 CAMS-CSM1-0 ['r1i1p1f1'] output: 31 reported native: 31 6 CanESM5 ['r1i1p1f1'] output: 49 reported native: 49 7 CanESM5-1 ['r1i1p1f1'] output: 49 reported native: 49 8 CAS-ESM2-0 ['r1i1p1f1'] output: 35 reported native: 35 9 CESM2 ['r1i1p1f1'] output: 32 reported native: 32
/home/durack1/mambaforge/envs/xcd061nctax/lib/python3.11/site-packages/xarray/conventions.py:436: SerializationWarning: variable 'cl' has multiple fill values {1e+20, 1e+20}, decoding all values to NaN. new_vars[k] = decode_cf_variable(
10 CESM2-FV2 ['r1i1p1f1'] output: 32 reported native: 32 11 CESM2-WACCM ['r1i1p1f1'] output: 70 reported native: 70 12 CESM2-WACCM-FV2 ['r1i1p1f1'] output: 70 reported native: 70 13 CIESM ['r1i1p1f1'] output: 30 reported native: 30 14 CMCC-CM2-HR4 ['r1i1p1f1'] output: 26 reported native: 26 15 CMCC-CM2-SR5 ['r1i1p1f1'] output: 30 reported native: 30 16 CMCC-ESM2 ['r1i1p1f1'] output: 30 reported native: 30 17 CNRM-CM6-1 ['r1i1p1f2'] output: 91 reported native: 91 18 CNRM-CM6-1-HR ['r1i1p1f2'] output: 91 reported native: 91 19 CNRM-ESM2-1 ['r1i1p1f2'] output: 91 reported native: 91 20 E3SM-1-0 ['r1i1p1f1'] output: 72 reported native: 72 21 E3SM-1-1 ['r1i1p1f1'] output: 72 reported native: 72 22 E3SM-1-1-ECA ['r1i1p1f1'] output: 72 reported native: 72 23 E3SM-2-0 ['r1i1p1f1'] output: 72 reported native: 72 24 E3SM-2-0-NARRM ['r1i1p1f1'] output: 72 reported native: 72 25 EC-Earth3 ['r10i1p1f1'] output: 91 reported native: 91 26 EC-Earth3-AerChem ['r1i1p1f1'] output: 91 reported native: 91 27 FGOALS-f3-L ['r1i1p1f1'] output: 32 reported native: 32 28 FGOALS-g3 ['r1i1p1f1'] output: 26 reported native: 26 29 GFDL-CM4 ['r1i1p1f1'] output: 33 reported native: 33 30 GFDL-ESM4 ['r1i1p1f1'] output: 49 reported native: 49 31 GISS-E2-1-G ['r1i1p1f1'] output: 29 reported native: 40 -- decimated -- 32 GISS-E2-1-H ['r1i1p1f1'] output: 29 reported native: 40 -- decimated -- 33 GISS-E2-2-G ['r1i1p1f1'] output: 102 reported native: 10 -- misreported -- 34 GISS-E2-2-H ['r1i1p1f1'] output: 102 reported native: 10 -- misreported -- 35 HadGEM3-GC31-LL ['r1i1p1f3'] output: 85 reported native: 85 36 HadGEM3-GC31-MM ['r1i1p1f3'] output: 85 reported native: 85 37 IPSL-CM6A-LR ['r1i1p1f1'] output: 79 reported native: 79 38 IPSL-CM6A-LR-INCA ['r1i1p1f1'] output: 79 reported native: 79 39 KACE-1-0-G ['r1i1p1f1'] output: 85 reported native: 85 40 MCM-UA-1-0 ['r1i1p1f1'] output: 19 reported native: 14 -- misreported -- 41 MIROC6 ['r1i1p1f1'] output: 81 reported native: 81 42 MIROC-ES2H ['r1i1p1f2'] output: 81 reported native: 81 43 MIROC-ES2L ['r1i1p1f2'] output: 40 reported native: 40 44 MPI-ESM1-2-HR ['r1i1p1f1'] output: 95 reported native: 95 45 MPI-ESM1-2-LR ['r1i1p1f1'] output: 47 reported native: 47 46 MPI-ESM-1-2-HAM ['r1i1p1f1'] output: 47 reported native: 47 47 MRI-ESM2-0 ['r1i1p1f1'] output: 80 reported native: 80 48 NESM3 ['r1i1p1f1'] output: 47 reported native: 47 49 NorESM2-LM ['r1i1p1f1'] output: 32 reported native: 32 50 NorESM2-MM ['r1i1p1f1'] output: 32 reported native: 32 51 SAM0-UNICON ['r1i1p1f1'] output: 30 reported native: 30 52 TaiESM1 ['r1i1p1f1'] output: 30 reported native: 30 53 UKESM1-0-LL ['r1i1p1f2'] output: 85 reported native: 85 [85, 38, 47, 46, 26, 31, 49, 49, 35, 32, 32, 70, 70, 30, 26, 30, 30, 91, 91, 91, 72, 72, 72, 72, 72, 91, 91, 32, 26, 33, 49, 29, 29, 102, 102, 85, 85, 79, 79, 85, 19, 81, 81, 40, 95, 47, 47, 80, 47, 32, 32, 30, 30, 85] CPU times: user 1.53 s, sys: 521 ms, total: 2.05 s Wall time: 47.3 s
print("Average atmos levels:", (sum(aveLvls) // len(aveLvls) + 1))
Average atmos levels: 58