Explore Kerchunking Harmonie data

Explore zipped test data from: https://stackoverflow.com/questions/70952226/how-to-merge-different-shaped-netcdf4-files

import fsspec
import xarray as xr
from kerchunk.hdf import SingleHdf5ToZarr 
from pathlib import Path
import ujson

fs = fsspec.filesystem('file')

flist = fs.glob('data/test_data_stackoverflow/*.nc')
flist[:5]

['/shared/users/rsignell/notebooks/testing/data/test_data_stackoverflow/harmonie_knmi_2021_03_20_18_30_18.nc',
 '/shared/users/rsignell/notebooks/testing/data/test_data_stackoverflow/harmonie_knmi_2021_03_20_18_30_19.nc',
 '/shared/users/rsignell/notebooks/testing/data/test_data_stackoverflow/harmonie_knmi_2021_03_20_18_30_20.nc',
 '/shared/users/rsignell/notebooks/testing/data/test_data_stackoverflow/harmonie_knmi_2021_03_20_18_30_21.nc',
 '/shared/users/rsignell/notebooks/testing/data/test_data_stackoverflow/harmonie_knmi_2021_03_20_18_30_22.nc']

flist[0]

'/shared/users/rsignell/notebooks/testing/data/test_data_stackoverflow/harmonie_knmi_2021_03_20_18_30_18.nc'

for f in flist:
    print(xr.open_dataset(flist[0]).x.values)

[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]
[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 24 25 26 27 28 29 30]

for f in flist:
    print(xr.open_dataset(flist[0]).dt_calc.values)

2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000
2021-03-20T18:00:00.000000000

for f in flist:
    print(xr.open_dataset(flist[0]).dt_fore[0].values)

1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000
1616263200000000000

json_dir = 'jsons'

#generate a json from a netCDF
def gen_json(u):
    #open the file (u) with desiganted parameters (so), call it infile
    with fs.open(u, **so) as infile:
        #inline_threshold: chunks smaller than 300 (I'm guessing mb?) are included in output
        h5chunks = SingleHdf5ToZarr(infile, u, inline_threshold=300)
        #Path().stem is from the pathlib library; returns the file name w/o the extension
        fstem = Path(u).stem
        #here we create the output file path
        outf = f'{json_dir}/{fstem}.json'
        #print(outf)
        #wb: write binary file
        with fs.open(outf, 'wb') as f:
            #create json file and write it to the path specified above (f)
            #dumps: dictionary to string
            #translate: translate contents of HDF5 to Zarr
            f.write(ujson.dumps(h5chunks.translate()).encode());

so = dict(mode='rb')

#loop through list of netCDFs and put them through our function for generating jsons
for f in flist:
    gen_json(f)

#create and view a sorted list of all jsons created in the previous step
json_list = fs.glob(f'{json_dir}/*.json')
json_list[:5]

['/shared/users/rsignell/notebooks/testing/jsons/harmonie_knmi_2021_03_20_18_30_18.json',
 '/shared/users/rsignell/notebooks/testing/jsons/harmonie_knmi_2021_03_20_18_30_19.json',
 '/shared/users/rsignell/notebooks/testing/jsons/harmonie_knmi_2021_03_20_18_30_20.json',
 '/shared/users/rsignell/notebooks/testing/jsons/harmonie_knmi_2021_03_20_18_30_21.json',
 '/shared/users/rsignell/notebooks/testing/jsons/harmonie_knmi_2021_03_20_18_30_22.json']

rpath = json_list[0]
fs3 = fsspec.filesystem("reference", fo=rpath)

m = fs3.get_mapper("")
ds = xr.open_dataset(m, engine="zarr",chunks={}, backend_kwargs={'consolidated':False})

ds

<xarray.Dataset>
Dimensions:                       (dt_fore: 49, x: 31, y: 54)
Coordinates:
    dt_calc                       datetime64[ns] ...
  * dt_fore                       (dt_fore) float64 1.616e+18 ... 1.616e+18
    lat                           (x, y) float64 dask.array<chunksize=(31, 54), meta=np.ndarray>
    lon                           (x, y) float64 dask.array<chunksize=(31, 54), meta=np.ndarray>
Dimensions without coordinates: x, y
Data variables: (12/27)
    air_temperature_2m            (dt_fore, x, y) float16 dask.array<chunksize=(13, 16, 27), meta=np.ndarray>
    air_temperature_500           (dt_fore, x, y) float16 dask.array<chunksize=(13, 16, 27), meta=np.ndarray>
    air_temperature_850           (dt_fore, x, y) float16 dask.array<chunksize=(13, 16, 27), meta=np.ndarray>
    dewpoint_2m                   (dt_fore, x, y) float16 dask.array<chunksize=(13, 16, 27), meta=np.ndarray>
    geopotential_height_500       (dt_fore, x, y) float64 dask.array<chunksize=(13, 8, 14), meta=np.ndarray>
    geopotential_height_850       (dt_fore, x, y) float64 dask.array<chunksize=(13, 8, 14), meta=np.ndarray>
    ...                            ...
    wind_direction_50             (dt_fore, x, y) float64 dask.array<chunksize=(13, 8, 14), meta=np.ndarray>
    wind_speed_10                 (dt_fore, x, y) float64 dask.array<chunksize=(13, 8, 14), meta=np.ndarray>
    wind_speed_100                (dt_fore, x, y) float64 dask.array<chunksize=(13, 8, 14), meta=np.ndarray>
    wind_speed_200                (dt_fore, x, y) float64 dask.array<chunksize=(13, 8, 14), meta=np.ndarray>
    wind_speed_300                (dt_fore, x, y) float64 dask.array<chunksize=(13, 8, 14), meta=np.ndarray>
    wind_speed_50                 (dt_fore, x, y) float64 dask.array<chunksize=(13, 8, 14), meta=np.ndarray>

xarray.Dataset

• Dimensions:
  • dt_fore: 49
  • x: 31
  • y: 54
• Coordinates: (4)
  • dt_calc
    ()
    datetime64[ns]
    ...

    array('2021-03-20T18:00:00.000000000', dtype='datetime64[ns]')

  • dt_fore
    (dt_fore)
    float64
    1.616e+18 1.616e+18 ... 1.616e+18

    array([1.616263e+18, 1.616267e+18, 1.616270e+18, 1.616274e+18, 1.616278e+18,
           1.616281e+18, 1.616285e+18, 1.616288e+18, 1.616292e+18, 1.616296e+18,
           1.616299e+18, 1.616303e+18, 1.616306e+18, 1.616310e+18, 1.616314e+18,
           1.616317e+18, 1.616321e+18, 1.616324e+18, 1.616328e+18, 1.616332e+18,
           1.616335e+18, 1.616339e+18, 1.616342e+18, 1.616346e+18, 1.616350e+18,
           1.616353e+18, 1.616357e+18, 1.616360e+18, 1.616364e+18, 1.616368e+18,
           1.616371e+18, 1.616375e+18, 1.616378e+18, 1.616382e+18, 1.616386e+18,
           1.616389e+18, 1.616393e+18, 1.616396e+18, 1.616400e+18, 1.616404e+18,
           1.616407e+18, 1.616411e+18, 1.616414e+18, 1.616418e+18, 1.616422e+18,
           1.616425e+18, 1.616429e+18, 1.616432e+18, 1.616436e+18])

  • lat
    (x, y)
    float64
    dask.array<chunksize=(31, 54), meta=np.ndarray>

    Array Chunk Bytes 13.08 kiB 13.08 kiB Shape (31, 54) (31, 54) Count 2 Tasks 1 Chunks Type float64 numpy.ndarray

  • lon
    (x, y)
    float64
    dask.array<chunksize=(31, 54), meta=np.ndarray>

    Array Chunk Bytes 13.08 kiB 13.08 kiB Shape (31, 54) (31, 54) Count 2 Tasks 1 Chunks Type float64 numpy.ndarray

• Data variables: (27)
  • air_temperature_2m
    (dt_fore, x, y)
    float16
    dask.array<chunksize=(13, 16, 27), meta=np.ndarray>

    Array Chunk Bytes 160.21 kiB 10.97 kiB Shape (49, 31, 54) (13, 16, 27) Count 17 Tasks 16 Chunks Type float16 numpy.ndarray

  • air_temperature_500
    (dt_fore, x, y)
    float16
    dask.array<chunksize=(13, 16, 27), meta=np.ndarray>

    Array Chunk Bytes 160.21 kiB 10.97 kiB Shape (49, 31, 54) (13, 16, 27) Count 17 Tasks 16 Chunks Type float16 numpy.ndarray

  • air_temperature_850
    (dt_fore, x, y)
    float16
    dask.array<chunksize=(13, 16, 27), meta=np.ndarray>

    Array Chunk Bytes 160.21 kiB 10.97 kiB Shape (49, 31, 54) (13, 16, 27) Count 17 Tasks 16 Chunks Type float16 numpy.ndarray

  • dewpoint_2m
    (dt_fore, x, y)
    float16
    dask.array<chunksize=(13, 16, 27), meta=np.ndarray>

    Array Chunk Bytes 160.21 kiB 10.97 kiB Shape (49, 31, 54) (13, 16, 27) Count 17 Tasks 16 Chunks Type float16 numpy.ndarray

  • geopotential_height_500
    (dt_fore, x, y)
    float64
    dask.array<chunksize=(13, 8, 14), meta=np.ndarray>

    Array Chunk Bytes 640.83 kiB 11.38 kiB Shape (49, 31, 54) (13, 8, 14) Count 65 Tasks 64 Chunks Type float64 numpy.ndarray

  • geopotential_height_850
    (dt_fore, x, y)
    float64
    dask.array<chunksize=(13, 8, 14), meta=np.ndarray>

    Array Chunk Bytes 640.83 kiB 11.38 kiB Shape (49, 31, 54) (13, 8, 14) Count 65 Tasks 64 Chunks Type float64 numpy.ndarray

  • global_horizontal_irradiance
    (dt_fore, x, y)
    float64
    dask.array<chunksize=(13, 8, 14), meta=np.ndarray>

    Array Chunk Bytes 640.83 kiB 11.38 kiB Shape (49, 31, 54) (13, 8, 14) Count 65 Tasks 64 Chunks Type float64 numpy.ndarray

  • high_level_clouds
    (dt_fore, x, y)
    float64
    dask.array<chunksize=(13, 8, 14), meta=np.ndarray>

    Array Chunk Bytes 640.83 kiB 11.38 kiB Shape (49, 31, 54) (13, 8, 14) Count 65 Tasks 64 Chunks Type float64 numpy.ndarray

  • low_level_clouds
    (dt_fore, x, y)
    float64
    dask.array<chunksize=(13, 8, 14), meta=np.ndarray>

    Array Chunk Bytes 640.83 kiB 11.38 kiB Shape (49, 31, 54) (13, 8, 14) Count 65 Tasks 64 Chunks Type float64 numpy.ndarray

  • max_wind_gust_10m
    (dt_fore, x, y)
    float64
    dask.array<chunksize=(13, 8, 14), meta=np.ndarray>

    Array Chunk Bytes 640.83 kiB 11.38 kiB Shape (49, 31, 54) (13, 8, 14) Count 65 Tasks 64 Chunks Type float64 numpy.ndarray

  • medium_level_clouds
    (dt_fore, x, y)
    float64
    dask.array<chunksize=(13, 8, 14), meta=np.ndarray>

    Array Chunk Bytes 640.83 kiB 11.38 kiB Shape (49, 31, 54) (13, 8, 14) Count 65 Tasks 64 Chunks Type float64 numpy.ndarray

  • pressure_mean_sea_level
    (dt_fore, x, y)
    float64
    dask.array<chunksize=(13, 8, 14), meta=np.ndarray>

    Array Chunk Bytes 640.83 kiB 11.38 kiB Shape (49, 31, 54) (13, 8, 14) Count 65 Tasks 64 Chunks Type float64 numpy.ndarray

  • relative_humidity_2m
    (dt_fore, x, y)
    float64
    dask.array<chunksize=(13, 8, 14), meta=np.ndarray>

    Array Chunk Bytes 640.83 kiB 11.38 kiB Shape (49, 31, 54) (13, 8, 14) Count 65 Tasks 64 Chunks Type float64 numpy.ndarray

  • snow_depth
    (dt_fore, x, y)
    float16
    dask.array<chunksize=(13, 16, 27), meta=np.ndarray>

    Array Chunk Bytes 160.21 kiB 10.97 kiB Shape (49, 31, 54) (13, 16, 27) Count 17 Tasks 16 Chunks Type float16 numpy.ndarray

  • surface_temperature
    (dt_fore, x, y)
    float16
    dask.array<chunksize=(13, 16, 27), meta=np.ndarray>

    Array Chunk Bytes 160.21 kiB 10.97 kiB Shape (49, 31, 54) (13, 16, 27) Count 17 Tasks 16 Chunks Type float16 numpy.ndarray

  • total_cloud_cover
    (dt_fore, x, y)
    float64
    dask.array<chunksize=(13, 8, 14), meta=np.ndarray>

    Array Chunk Bytes 640.83 kiB 11.38 kiB Shape (49, 31, 54) (13, 8, 14) Count 65 Tasks 64 Chunks Type float64 numpy.ndarray

  • total_precipitation
    (dt_fore, x, y)
    float16
    dask.array<chunksize=(13, 16, 27), meta=np.ndarray>

    Array Chunk Bytes 160.21 kiB 10.97 kiB Shape (49, 31, 54) (13, 16, 27) Count 17 Tasks 16 Chunks Type float16 numpy.ndarray

  • wind_direction_10
    (dt_fore, x, y)
    float64
    dask.array<chunksize=(13, 8, 14), meta=np.ndarray>

    Array Chunk Bytes 640.83 kiB 11.38 kiB Shape (49, 31, 54) (13, 8, 14) Count 65 Tasks 64 Chunks Type float64 numpy.ndarray

  • wind_direction_100
    (dt_fore, x, y)
    float64
    dask.array<chunksize=(13, 8, 14), meta=np.ndarray>

    Array Chunk Bytes 640.83 kiB 11.38 kiB Shape (49, 31, 54) (13, 8, 14) Count 65 Tasks 64 Chunks Type float64 numpy.ndarray

  • wind_direction_200
    (dt_fore, x, y)
    float64
    dask.array<chunksize=(13, 8, 14), meta=np.ndarray>

    Array Chunk Bytes 640.83 kiB 11.38 kiB Shape (49, 31, 54) (13, 8, 14) Count 65 Tasks 64 Chunks Type float64 numpy.ndarray

  • wind_direction_300
    (dt_fore, x, y)
    float64
    dask.array<chunksize=(13, 8, 14), meta=np.ndarray>

    Array Chunk Bytes 640.83 kiB 11.38 kiB Shape (49, 31, 54) (13, 8, 14) Count 65 Tasks 64 Chunks Type float64 numpy.ndarray

  • wind_direction_50
    (dt_fore, x, y)
    float64
    dask.array<chunksize=(13, 8, 14), meta=np.ndarray>

    Array Chunk Bytes 640.83 kiB 11.38 kiB Shape (49, 31, 54) (13, 8, 14) Count 65 Tasks 64 Chunks Type float64 numpy.ndarray

  • wind_speed_10
    (dt_fore, x, y)
    float64
    dask.array<chunksize=(13, 8, 14), meta=np.ndarray>

    Array Chunk Bytes 640.83 kiB 11.38 kiB Shape (49, 31, 54) (13, 8, 14) Count 65 Tasks 64 Chunks Type float64 numpy.ndarray

  • wind_speed_100
    (dt_fore, x, y)
    float64
    dask.array<chunksize=(13, 8, 14), meta=np.ndarray>

    Array Chunk Bytes 640.83 kiB 11.38 kiB Shape (49, 31, 54) (13, 8, 14) Count 65 Tasks 64 Chunks Type float64 numpy.ndarray

  • wind_speed_200
    (dt_fore, x, y)
    float64
    dask.array<chunksize=(13, 8, 14), meta=np.ndarray>

    Array Chunk Bytes 640.83 kiB 11.38 kiB Shape (49, 31, 54) (13, 8, 14) Count 65 Tasks 64 Chunks Type float64 numpy.ndarray

  • wind_speed_300
    (dt_fore, x, y)
    float64
    dask.array<chunksize=(13, 8, 14), meta=np.ndarray>

    Array Chunk Bytes 640.83 kiB 11.38 kiB Shape (49, 31, 54) (13, 8, 14) Count 65 Tasks 64 Chunks Type float64 numpy.ndarray

  • wind_speed_50
    (dt_fore, x, y)
    float64
    dask.array<chunksize=(13, 8, 14), meta=np.ndarray>

    Array Chunk Bytes 640.83 kiB 11.38 kiB Shape (49, 31, 54) (13, 8, 14) Count 65 Tasks 64 Chunks Type float64 numpy.ndarray

• Attributes: (0)