Use TIFFfile to create a Zarr virtual datacube¶

In [1]:

from tifffile import TiffSequence
import zarr
import fsspec

Try local tiff files first:¶

In [2]:

fs = fsspec.filesystem('file')
flist = fs.ls('/caldera/projects/usgs/hazards/cmgp/woodshole/rsignell/lcmap/cog/')
flist

Out[2]:

['/caldera/projects/usgs/hazards/cmgp/woodshole/rsignell/lcmap/cog/LCMAP_CU_2019_V12_LCPRI.tif',
 '/caldera/projects/usgs/hazards/cmgp/woodshole/rsignell/lcmap/cog/LCMAP_CU_2020_V12_LCPRI.tif']

In [3]:

import re
ex = re.compile(r".*_(\d+)_V12_LCPRI.tif")
ex.search(flist[0]).groups()[0]

Out[3]:

'2019'

In [4]:

image_sequence = TiffSequence(flist, pattern=r'.*_(\d+)_V12_LCPRI.tif')

In [5]:

image_sequence.shape

Out[5]:

(2,)

In [6]:

with image_sequence.aszarr() as store:
    dz = zarr.open(store, mode='r')

In [7]:

    image_sequence.close()

In [8]:

with image_sequence.aszarr() as store:
    store.write_fsspec('temp.json', url='file://')

In [9]:

import fsspec
import tifffile.numcodecs
tifffile.numcodecs.register_codec()
mapper = fsspec.get_mapper('reference://', fo='temp.json', target_protocol='file')

In [10]:

ds = zarr.open(mapper, mode='r')

In [11]:

ds.info

Out[11]:

Type	zarr.core.Array
Data type	uint8
Shape	(2, 105000, 160000)
Chunk shape	(1, 105000, 160000)
Order	C
Read-only	True
Compressor	Tiff(bigtiff=None, byteorder=None, compression=None, extratags=(), imagej=False, key=None, level=None, maxworkers=None, metadata={}, ome=None, photometric=None, planarconfig=None, predictor=None, rowsperstrip=None, series=None, subsampling=None, tile=None, truncate=False, volumetric=None)
Store type	zarr.storage.KVStore
No. bytes	33600000000 (31.3G)
No. bytes stored	215
Storage ratio	156279069.8
Chunks initialized	2/2

Try accessing files Open Storage Network¶

The Open Storage Network is an NSF-supported program that provides about one PB of locally installed, remotely managed, s3-compatible object storage for about $30K/year.

In [12]:

fs = fsspec.filesystem('s3', anon=True, client_kwargs={'endpoint_url': 'https://mghp.osn.xsede.org'})

In [13]:

flist = fs.ls('/rsignellbucket1/lcmap/cog')
flist

Out[13]:

['rsignellbucket1/lcmap/cog/LCMAP_CU_2019_V12_LCPRI.tif',
 'rsignellbucket1/lcmap/cog/LCMAP_CU_2020_V12_LCPRI.tif']

In [14]:

s3_flist = [f's3://{f}' for f in flist]

In [15]:

image_sequence = TiffSequence(s3_flist, pattern=r'.*_(\d+)_V12_LCPRI.tif')

In [16]:

with image_sequence.aszarr() as store:
    dz = zarr.open(store, mode='r')

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
Input In [16], in <cell line: 1>()
----> 1 with image_sequence.aszarr() as store:
      2     dz = zarr.open(store, mode='r')

File ~/miniconda3/envs/pangeo/lib/python3.9/site-packages/tifffile/tifffile.py:10292, in FileSequence.aszarr(self, **kwargs)
  10290 def aszarr(self, **kwargs):
  10291     """Return image data from files as zarr storage."""
> 10292     return ZarrFileSequenceStore(self, **kwargs)

File ~/miniconda3/envs/pangeo/lib/python3.9/site-packages/tifffile/tifffile.py:9828, in ZarrFileSequenceStore.__init__(self, arg, fillvalue, chunkmode, chunkshape, dtype, axestiled, zattrs, **kwargs)
   9825 self._commonpath = arg.commonpath()
   9827 if chunkshape is None or dtype is None:
-> 9828     chunk = arg.imread(arg.files[0], **kwargs)
   9829     self._chunks = chunk.shape
   9830     self._dtype = chunk.dtype

File ~/miniconda3/envs/pangeo/lib/python3.9/site-packages/tifffile/tifffile.py:820, in imread(files, aszarr, **kwargs)
    817     files = files[0]
    819 if isinstance(files, (str, os.PathLike)) or hasattr(files, 'seek'):
--> 820     with TiffFile(files, **kwargs_file) as tif:
    821         if aszarr:
    822             return tif.aszarr(**kwargs)

File ~/miniconda3/envs/pangeo/lib/python3.9/site-packages/tifffile/tifffile.py:3142, in TiffFile.__init__(self, arg, mode, name, offset, size, _multifile, _useframes, _parent, **kwargs)
   3139 if mode not in (None, 'rb', 'r+b'):
   3140     raise ValueError(f'invalid mode {mode!r}')
-> 3142 fh = FileHandle(arg, mode=mode, name=name, offset=offset, size=size)
   3143 self._fh = fh
   3144 self._multifile = bool(_multifile)

File ~/miniconda3/envs/pangeo/lib/python3.9/site-packages/tifffile/tifffile.py:10509, in FileHandle.__init__(self, file, mode, name, offset, size)
  10507 self.is_file = None
  10508 self._lock = NullContext()
> 10509 self.open()

File ~/miniconda3/envs/pangeo/lib/python3.9/site-packages/tifffile/tifffile.py:10522, in FileHandle.open(self)
  10520 self._file = os.path.realpath(self._file)
  10521 self._dir, self._name = os.path.split(self._file)
> 10522 self._fh = open(self._file, self._mode)
  10523 self._close = True
  10524 if self._offset is None:

FileNotFoundError: [Errno 2] No such file or directory: '/home/rsignell/EarthMap/Projects/esip-qhub-notebooks/LCMAP/s3:/rsignellbucket1/lcmap/cog/LCMAP_CU_2019_V12_LCPRI.tif'

In [17]:

file_like_object_list = [fs.open(f) for f in flist]

In [18]:

image_sequence = TiffSequence(file_like_object_list, pattern=r'.*_(\d+)_V12_LCPRI.tif')

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Input In [18], in <cell line: 1>()
----> 1 image_sequence = TiffSequence(file_like_object_list, pattern=r'.*_(\d+)_V12_LCPRI.tif')

File ~/miniconda3/envs/pangeo/lib/python3.9/site-packages/tifffile/tifffile.py:10308, in TiffSequence.__init__(self, files, imread, **kwargs)
  10306 def __init__(self, files=None, imread=imread, **kwargs):
  10307     """Initialize instance from multiple TIFF files."""
> 10308     super().__init__(imread, '*.tif' if files is None else files, **kwargs)

File ~/miniconda3/envs/pangeo/lib/python3.9/site-packages/tifffile/tifffile.py:10105, in FileSequence.__init__(self, imread, files, container, sort, parse, **kwargs)
  10102     if sort:
  10103         files = sort(files)
> 10105 files = [os.fspath(f) for f in files]
  10106 if not files:
  10107     raise ValueError('no files found')

File ~/miniconda3/envs/pangeo/lib/python3.9/site-packages/tifffile/tifffile.py:10105, in <listcomp>(.0)
  10102     if sort:
  10103         files = sort(files)
> 10105 files = [os.fspath(f) for f in files]
  10106 if not files:
  10107     raise ValueError('no files found')

TypeError: expected str, bytes or os.PathLike object, not S3File