This notebook demonstates a method to open all the variables in the latest HRRR forecast in a single xarray-datatree object using kerchunk to create and edit the necessary reference files.
import datetime as dt
import fsspec
import ujson
import kerchunk
from kerchunk.grib2 import scan_grib
from kerchunk.combine import MultiZarrToZarr
import dask.bag as db
import pickle
Much of this script can be run in parallel
from dask.distributed import Client
client = Client()
First let's create two file systems using fsspec
one to read the HRRR data from the S3 bucket and another to save the reference files.
fs_read = fsspec.filesystem('s3', anon=True, skip_instance_cache=True)
fs_write = fsspec.filesystem('')
The we generate a list containing paths to the latest forecast data:
days_avail = fs_read.glob('s3://noaa-hrrr-bdp-pds/hrrr.*')
runs = fs_read.glob(f's3://{days_avail[-1]}/conus/*wrfsfcf01.grib2')
files = fs_read.glob(f's3://{runs[-2][:-8]}*.grib2') #select second last run to ensure it is a complete forecast
#please note only 00,06,12 and 18z HRRR forecasts are 48 hours the remainder are 18 hours
files = sorted(['s3://'+f for f in files])
files
['s3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf00.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf01.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf02.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf03.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf04.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf05.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf06.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf07.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf08.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf09.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf10.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf11.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf12.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf13.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf14.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf15.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf16.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf17.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf18.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf19.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf20.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf21.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf22.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf23.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf24.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf25.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf26.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf27.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf28.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf29.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf30.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf31.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf32.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf33.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf34.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf35.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf36.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf37.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf38.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf39.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf40.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf41.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf42.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf43.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf44.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf45.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf46.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf47.grib2', 's3://noaa-hrrr-bdp-pds/hrrr.20220902/conus/hrrr.t12z.wrfsfcf48.grib2']
Create folder path to save reference files to.
json_dir = 'jsons/'
fs_write.mkdir(json_dir)
We use the below two functions to run scan_grib
on each HRRR forecast file and save a reference file for each grib message within these files
def make_json_name(file_url, message_number): #create a unique name for each reference file
date = file_url.split('/')[3].split('.')[1]
name = file_url.split('/')[5].split('.')[1:3]
return f'{json_dir}{date}_{name[0]}_{name[1]}_message{message_number}.json'
def gen_json(file_url):
out = scan_grib(file_url, storage_options={"anon": True}) #create the reference using scan_grib
for i, message in enumerate(out): # scan_grib outputs a list containing one reference file per grib message
out_file_name = make_json_name(file_url, i) #get name
with fs_write.open(out_file_name, "w") as f:
f.write(ujson.dumps(message)) #write to file
Run the computation, each file takes about 5-10 minutes to scan. I recommend only doing two files at first to check everything runs smoothly.
%%time
bag = db.from_sequence(files[:2])
bag_map = bag.map(gen_json)
_ = bag_map.compute()
CPU times: user 23.5 s, sys: 6.58 s, total: 30.1 s Wall time: 8min 3s
There are some variables within the HRRR dataset that will return the error message:
" MissingDimensionsError: 'x' has more than 1-dimension and the same name as one of its dimensions ('x', 'y'). xarray disallows such variables because they conflict with the coordinates used to label dimensions." see https://github.com/pydata/xarray/issues/2233
To avoid this issue I have created a dictionary containing the grib message and variable names which cause this issue and use preprocess
in MultiZarrtoZarr
to drop these variables. These grib messages are primarily 1 dimensional arrays describing their vertical component.
to_drop = {'103.json': 'surface', '106.json': 'isobaricLayer', '107.json': 'surface', '108.json': 'surface', '109.json': 'atmosphereSingleLayer', '110.json': 'atmosphereSingleLayer','111.json': 'atmosphereSingleLayer','112.json': 'atmosphere','113.json': 'atmosphere','114.json': 'boundaryLayerCloudLayer','115.json': 'lowCloudLayer','116.json': 'middleCloudLayer','117.json': 'highCloudLayer','118.json': 'atmosphere','119.json': 'cloudCeiling','120.json': 'cloudBase','121.json': 'cloudBase','122.json': 'cloudTop','123.json': 'cloudTop','124.json': 'nominalTop','129.json': 'surface','130.json': 'surface','131.json': 'surface','132.json': 'nominalTop','138.json': 'heightAboveGroundLayer','139.json': 'heightAboveGroundLayer','140.json': 'heightAboveGroundLayer','141.json': 'isothermZero','142.json': 'isothermZero','143.json': 'isothermZero','144.json': 'highestTroposphericFreezing','145.json': 'highestTroposphericFreezing','146.json': 'highestTroposphericFreezing','147.json': 'isothermal','148.json': 'isothermal','149.json': 'pressureFromGroundLayer','150.json': 'pressureFromGroundLayer','152.json': 'surface','153.json': 'adiabaticCondensation','155.json': 'pressureFromGroundLayer','157.json': 'pressureFromGroundLayer','158.json': 'equilibrium','159.json': 'pressureFromGroundLayer','162.json': 'surface','163.json': 'heightAboveGroundLayer','164.json': 'unknown','165.json': 'heightAboveGroundLayer','166.json': 'atmosphere','167.json': 'surface','168.json': 'surface'}
Next we combine the reference messages along the valid time dimension using MultiZarrtoZarr
.
fs_write.mkdir(f'{json_dir}combined/')
def combine(message_number):
jsons = fs_write.glob(f'{json_dir}*message{message_number}.json')
file_name = f'{json_dir}combined/{message_number}.json'
if file_name.split('/')[-1] in list(to_drop):
preprocess = kerchunk.combine.drop(to_drop[file_name.split('/')[-1]])
mzz = MultiZarrToZarr(jsons,concat_dims = ['valid_time'], preprocess = preprocess)
else:
mzz = MultiZarrToZarr(jsons,concat_dims = ['valid_time'])
d = mzz.translate(filename = f'{json_dir}combined/{message_number}.json')
And run this combine computation across all 169 grib messages
%%time
bag = db.from_sequence(range(170)) #there are 169 grib messages within an HRRR grib2 file
bag_map = bag.map(combine)
_ = bag_map.compute()
CPU times: user 1.78 s, sys: 0 ns, total: 1.78 s Wall time: 7.11 s
The next step is to group the grib messages that can be opened as a single xarray dataset together. I have again created a dictionary that contains the compatible groupings:
groups = {'atmosphere': [0, 2, 3, 52, 58, 109, 110, 115, 163], 'cloudTop': [1, 119, 120], 'surface': [4, 8,54, 61, 62, 63, 64, 66, 67, 68, 69, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 104, 105, 122, 123, 124, 125, 126, 127, 128, 149, 159, 164, 165], 'heightAboveGround': [5, 6, 42, 56, 57, 59, 60, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80], 'isothermal': [7, 43, 144, 145], 'isobaricInhPa': [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 41], 'nominalTop': [129, 166, 167, 168, 169, 121], 'heightAboveGroundLayer': [130, 131, 132, 133, 134, 135, 136, 137, 157, 160, 162, 44, 45, 46, 47, 48, 49, 50, 51], 'isothermZero': [138, 139, 140], 'highestTroposphericFreezing': [141, 142, 143], 'pressureFromGroundLayer': [146, 147, 148, 151, 152, 153, 154, 156, 37, 38], 'adiabaticCondensation': [150], 'equilibrium': [155], 'depthBelowLand': [65], 'unknown': [161, 158], 'isobaricLayer': [103], 'sigmaLayer': [39], 'meanSea': [40], 'atmosphereSingleLayer': [106, 107, 108, 55], 'boundaryLayerCloudLayer': [111], 'lowCloudLayer': [112], 'middleCloudLayer': [113], 'highCloudLayer': [114], 'cloudBase': [117, 118], 'cloudCeiling': [116], 'sigma': [53]}
Create list of reference files to group together
Use MultiZarrtoZarr
to combine each group into a single reference file. We use zstd
compression when saving to reduce the size of the reference files.
def make_groups(group):
jsons = [f'{json_dir}combined/{m}.json' for m in groups[group]]
if group == 'unknown':
preprocess = kerchunk.combine.drop('unknown')
mzz = MultiZarrToZarr(jsons,concat_dims = ['valid_time'], preprocess = preprocess)
else:
mzz = MultiZarrToZarr(jsons,concat_dims = ['valid_time'])
ref = mzz.translate(filename=f'{group}.json.zst', storage_options=dict(compression='zstd'))
run the computation:
%%time
bag = db.from_sequence(list(groups))
bag_map = bag.map(make_groups)
_ = bag_map.compute()
CPU times: user 202 ms, sys: 0 ns, total: 202 ms Wall time: 647 ms
xarray-datatree
¶Function to open the references we have just created as zarr object using fsspec reference file system and xarray:
import xarray as xr
from datatree import DataTree
import hvplot.xarray
import panel as pn
from geoviews import tile_sources as gvts
def return_ds(d):
fs_ = fsspec.filesystem("reference", fo=d, remote_protocol='s3', remote_options={'anon':True},ref_storage_args={"compression": "zstd"})
m = fs_.get_mapper("")
return xr.open_dataset(m, engine="zarr", backend_kwargs=dict(consolidated=False))
reference_files = fs_write.glob('*.json.zst')
datasets = {ref.split('/')[-1].split('.')[0]:return_ds(ref) for ref in reference_files}
dt = DataTree.from_dict(datasets)
Xarray datatree provides a lot of functionality like performing .sel()
across all datasets, but here we will just use it as a convenient way to view all the variables available from HRRR
print(dt)
DataTree('None', parent=None) ├── DataTree('adiabaticCondensation') │ Dimensions: (adiabaticCondensation: 1, valid_time: 2, x: 1059, │ y: 1799, step: 1, time: 1) │ Coordinates: │ * adiabaticCondensation (adiabaticCondensation) int64 0 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 20... │ Dimensions without coordinates: x, y │ Data variables: │ gh (valid_time, x, y) float64 ... │ latitude (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('atmosphere') │ Dimensions: (atmosphere: 1, valid_time: 2, x: 1059, y: 1799, step: 1, │ time: 1) │ Coordinates: │ * atmosphere (atmosphere) int64 0 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 2022-09-02T13... │ Dimensions without coordinates: x, y │ Data variables: │ hail (valid_time, x, y) float64 ... │ latitude (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ ltng (valid_time, x, y) float64 ... │ refc (valid_time, x, y) float64 ... │ tcc (valid_time, x, y) float64 ... │ unknown (valid_time, x, y) float64 ... │ veril (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('atmosphereSingleLayer') │ Dimensions: (atmosphereSingleLayer: 1, valid_time: 2, x: 1059, │ y: 1799, step: 1, time: 1) │ Coordinates: │ * atmosphereSingleLayer (atmosphereSingleLayer) int64 0 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 20... │ Dimensions without coordinates: x, y │ Data variables: │ latitude (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ pwat (valid_time, x, y) float64 ... │ unknown (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('boundaryLayerCloudLayer') │ Dimensions: (boundaryLayerCloudLayer: 1, valid_time: 2, │ x: 1059, y: 1799, step: 1, time: 1) │ Coordinates: │ * boundaryLayerCloudLayer (boundaryLayerCloudLayer) int64 0 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 ... │ Dimensions without coordinates: x, y │ Data variables: │ latitude (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ tcc (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('cloudBase') │ Dimensions: (cloudBase: 1, valid_time: 2, x: 1059, y: 1799, step: 1, time: 1) │ Coordinates: │ * cloudBase (cloudBase) int64 0 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 2022-09-02T13... │ Dimensions without coordinates: x, y │ Data variables: │ gh (valid_time, x, y) float64 ... │ latitude (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ pres (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('cloudCeiling') │ Dimensions: (cloudCeiling: 1, valid_time: 2, x: 1059, y: 1799, step: 1, │ time: 1) │ Coordinates: │ * cloudCeiling (cloudCeiling) int64 0 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 2022-09-02T... │ Dimensions without coordinates: x, y │ Data variables: │ gh (valid_time, x, y) float64 ... │ latitude (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('cloudTop') │ Dimensions: (cloudTop: 1, valid_time: 2, x: 1059, y: 1799, step: 1, time: 1) │ Coordinates: │ * cloudTop (cloudTop) int64 0 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 2022-09-02T13... │ Dimensions without coordinates: x, y │ Data variables: │ gh (valid_time, x, y) float64 ... │ latitude (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ pres (valid_time, x, y) float64 ... │ unknown (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('depthBelowLand') │ Dimensions: (depthBelowLand: 1, valid_time: 2, x: 1059, y: 1799, │ step: 1, time: 1) │ Coordinates: │ * depthBelowLand (depthBelowLand) int64 0 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 2022-09-0... │ Dimensions without coordinates: x, y │ Data variables: │ latitude (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ mstav (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('equilibrium') │ Dimensions: (equilibrium: 1, valid_time: 2, x: 1059, y: 1799, step: 1, │ time: 1) │ Coordinates: │ * equilibrium (equilibrium) int64 0 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 2022-09-02T1... │ Dimensions without coordinates: x, y │ Data variables: │ gh (valid_time, x, y) float64 ... │ latitude (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('heightAboveGround') │ Dimensions: (valid_time: 2, x: 1059, y: 1799, heightAboveGround: 1, │ step: 1, time: 1) │ Coordinates: │ * heightAboveGround (heightAboveGround) int64 1000 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 2022-0... │ Dimensions without coordinates: x, y │ Data variables: (12/14) │ 10si (valid_time, x, y) float64 ... │ 10u (valid_time, x, y) float64 ... │ 10v (valid_time, x, y) float64 ... │ 2d (valid_time, x, y) float64 ... │ 2r (valid_time, x, y) float64 ... │ 2sh (valid_time, x, y) float64 ... │ ... ... │ longitude (valid_time, x, y) float64 ... │ pt (valid_time, x, y) float64 ... │ refd (valid_time, x, y) float64 ... │ u (valid_time, x, y) float64 ... │ unknown (valid_time, x, y) float64 ... │ v (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('heightAboveGroundLayer') │ Dimensions: (valid_time: 2, x: 1059, y: 1799, │ heightAboveGroundLayer: 1, step: 1, time: 1) │ Coordinates: │ * heightAboveGroundLayer (heightAboveGroundLayer) int64 3000 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 2... │ Dimensions without coordinates: x, y │ Data variables: │ cape (valid_time, x, y) float64 ... │ hlcy (valid_time, x, y) float64 ... │ latitude (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ unknown (valid_time, x, y) float64 ... │ ustm (valid_time, x, y) float64 ... │ vo (valid_time, x, y) float64 ... │ vstm (valid_time, x, y) float64 ... │ vucsh (valid_time, x, y) float64 ... │ vvcsh (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('highCloudLayer') │ Dimensions: (valid_time: 2, x: 1059, y: 1799, highCloudLayer: 1, │ step: 1, time: 1) │ Coordinates: │ * highCloudLayer (highCloudLayer) int64 0 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 2022-09-0... │ Dimensions without coordinates: x, y │ Data variables: │ hcc (valid_time, x, y) float64 ... │ latitude (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('highestTroposphericFreezing') │ Dimensions: (valid_time: 2, x: 1059, y: 1799, │ highestTroposphericFreezing: 1, step: 1, │ time: 1) │ Coordinates: │ * highestTroposphericFreezing (highestTroposphericFreezing) int64 0 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00... │ Dimensions without coordinates: x, y │ Data variables: │ gh (valid_time, x, y) float64 ... │ latitude (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ pres (valid_time, x, y) float64 ... │ r (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('isobaricInhPa') │ Dimensions: (valid_time: 2, x: 1059, y: 1799, isobaricInhPa: 1, step: 1, │ time: 1) │ Coordinates: │ * isobaricInhPa (isobaricInhPa) int64 250 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 2022-09-02... │ Dimensions without coordinates: x, y │ Data variables: │ dpt (valid_time, x, y) float64 ... │ gh (valid_time, x, y) float64 ... │ latitude (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ t (valid_time, x, y) float64 ... │ u (valid_time, x, y) float64 ... │ v (valid_time, x, y) float64 ... │ wz (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('isobaricLayer') │ Dimensions: (isobaricLayer: 1, valid_time: 2, x: 1059, y: 1799, step: 1, │ time: 1) │ Coordinates: │ * isobaricLayer (isobaricLayer) int64 500 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 2022-09-02... │ Dimensions without coordinates: x, y │ Data variables: │ latitude (valid_time, x, y) float64 ... │ lftx (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('isothermZero') │ Dimensions: (valid_time: 2, x: 1059, y: 1799, isothermZero: 1, step: 1, │ time: 1) │ Coordinates: │ * isothermZero (isothermZero) int64 0 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 2022-09-02T... │ Dimensions without coordinates: x, y │ Data variables: │ gh (valid_time, x, y) float64 ... │ latitude (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ pres (valid_time, x, y) float64 ... │ r (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('isothermal') │ Dimensions: (valid_time: 2, x: 1059, y: 1799, isothermal: 1, step: 1, │ time: 1) │ Coordinates: │ * isothermal (isothermal) int64 263 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 2022-09-02T13... │ Dimensions without coordinates: x, y │ Data variables: │ gh (valid_time, x, y) float64 ... │ latitude (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ refd (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('lowCloudLayer') │ Dimensions: (valid_time: 2, x: 1059, y: 1799, lowCloudLayer: 1, step: 1, │ time: 1) │ Coordinates: │ * lowCloudLayer (lowCloudLayer) int64 0 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 2022-09-02... │ Dimensions without coordinates: x, y │ Data variables: │ latitude (valid_time, x, y) float64 ... │ lcc (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('meanSea') │ Dimensions: (valid_time: 2, x: 1059, y: 1799, meanSea: 1, step: 1, time: 1) │ Coordinates: │ * meanSea (meanSea) int64 0 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 2022-09-02T13... │ Dimensions without coordinates: x, y │ Data variables: │ latitude (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ mslma (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('middleCloudLayer') │ Dimensions: (valid_time: 2, x: 1059, y: 1799, middleCloudLayer: 1, │ step: 1, time: 1) │ Coordinates: │ * middleCloudLayer (middleCloudLayer) int64 0 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 2022-09... │ Dimensions without coordinates: x, y │ Data variables: │ latitude (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ mcc (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('nominalTop') │ Dimensions: (valid_time: 2, x: 1059, y: 1799, nominalTop: 1, step: 1, │ time: 1) │ Coordinates: │ * nominalTop (nominalTop) int64 0 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 2022-09-02T13... │ Dimensions without coordinates: x, y │ Data variables: │ SBT113 (valid_time, x, y) float64 ... │ SBT114 (valid_time, x, y) float64 ... │ SBT123 (valid_time, x, y) float64 ... │ SBT124 (valid_time, x, y) float64 ... │ latitude (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ ulwrf (valid_time, x, y) float64 ... │ uswrf (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('pressureFromGroundLayer') │ Dimensions: (valid_time: 2, x: 1059, y: 1799, │ pressureFromGroundLayer: 1, step: 1, time: 1) │ Coordinates: │ * pressureFromGroundLayer (pressureFromGroundLayer) int64 18000 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 ... │ Dimensions without coordinates: x, y │ Data variables: │ 4lftx (valid_time, x, y) float64 ... │ cape (valid_time, x, y) float64 ... │ cin (valid_time, x, y) float64 ... │ latitude (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ plpl (valid_time, x, y) float64 ... │ unknown (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('sigma') │ Dimensions: (valid_time: 2, x: 1059, y: 1799, sigma: 1, step: 1, time: 1) │ Coordinates: │ * sigma (sigma) int64 0 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 2022-09-02T13... │ Dimensions without coordinates: x, y │ Data variables: │ hail (valid_time, x, y) float64 ... │ latitude (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('sigmaLayer') │ Dimensions: (valid_time: 2, x: 1059, y: 1799, sigmaLayer: 1, step: 1, │ time: 1) │ Coordinates: │ * sigmaLayer (sigmaLayer) int64 1 │ * step (step) timedelta64[ns] 00:00:00 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 2022-09-02T13... │ Dimensions without coordinates: x, y │ Data variables: │ latitude (valid_time, x, y) float64 ... │ longitude (valid_time, x, y) float64 ... │ wz (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 ├── DataTree('surface') │ Dimensions: (valid_time: 2, x: 1059, y: 1799, step: 1, surface: 1, time: 1) │ Coordinates: │ * step (step) timedelta64[ns] 00:00:00 │ * surface (surface) int64 0 │ * time (time) datetime64[ns] 2022-09-02T12:00:00 │ * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 2022-09-02T13... │ Dimensions without coordinates: x, y │ Data variables: (12/44) │ asnow (valid_time, x, y) float64 ... │ bgrun (valid_time, x, y) float64 ... │ cape (valid_time, x, y) float64 ... │ cfnsf (valid_time, x, y) float64 ... │ cfrzr (valid_time, x, y) float64 ... │ ci (valid_time, x, y) float64 ... │ ... ... │ uswrf (valid_time, x, y) float64 ... │ vbdsf (valid_time, x, y) float64 ... │ vddsf (valid_time, x, y) float64 ... │ veg (valid_time, x, y) float64 ... │ vgtyp (valid_time, x, y) float64 ... │ vis (valid_time, x, y) float64 ... │ Attributes: │ centre: kwbc │ centreDescription: US National Weather Service - NCEP │ edition: 2 │ subCentre: 0 └── DataTree('unknown') Dimensions: (valid_time: 2, x: 1059, y: 1799, step: 1, time: 1) Coordinates: * step (step) timedelta64[ns] 00:00:00 * time (time) datetime64[ns] 2022-09-02T12:00:00 * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 2022-09-02T13... Dimensions without coordinates: x, y Data variables: gh (valid_time, x, y) float64 ... latitude (valid_time, x, y) float64 ... layth (valid_time, x, y) float64 ... longitude (valid_time, x, y) float64 ... Attributes: centre: kwbc centreDescription: US National Weather Service - NCEP edition: 2 subCentre: 0
ds = dt['surface'].to_dataset()
ds = ds.assign_coords(longitude=(((ds.longitude + 180) % 360) - 180), latitude=ds.latitude)
ds
<xarray.Dataset> Dimensions: (valid_time: 2, x: 1059, y: 1799, step: 1, surface: 1, time: 1) Coordinates: latitude (valid_time, x, y) float64 ... longitude (valid_time, x, y) float64 -122.7 -122.7 ... -60.95 -60.92 * step (step) timedelta64[ns] 00:00:00 * surface (surface) int64 0 * time (time) datetime64[ns] 2022-09-02T12:00:00 * valid_time (valid_time) datetime64[ns] 2022-09-02T12:00:00 2022-09-02T13... Dimensions without coordinates: x, y Data variables: (12/42) asnow (valid_time, x, y) float64 ... bgrun (valid_time, x, y) float64 ... cape (valid_time, x, y) float64 ... cfnsf (valid_time, x, y) float64 ... cfrzr (valid_time, x, y) float64 ... ci (valid_time, x, y) float64 ... ... ... uswrf (valid_time, x, y) float64 ... vbdsf (valid_time, x, y) float64 ... vddsf (valid_time, x, y) float64 ... veg (valid_time, x, y) float64 ... vgtyp (valid_time, x, y) float64 ... vis (valid_time, x, y) float64 ... Attributes: centre: kwbc centreDescription: US National Weather Service - NCEP edition: 2 subCentre: 0
sel_time = ds['valid_time'][0]
ds['vis'].sel(valid_time=sel_time).hvplot.quadmesh(x='longitude', y='latitude', geo=True, tiles='OSM',
rasterize=True, cmap='viridis_r', title = str(sel_time.values))