In [1]:

import xarray as xr
import requests
import boto3
import rasterio as rio
from rasterio.session import AWSSession
import os
from netrc import netrc
from subprocess import Popen
from platform import system
from getpass import getpass

import warnings
warnings.filterwarnings('ignore')

In [2]:

# Generates authentication token
# Asks for your Earthdata username and password for first time, if netrc does not exists.

urs = 'urs.earthdata.nasa.gov'    # Earthdata URL endpoint for authentication
prompts = ['Enter NASA Earthdata Login Username: ',
           'Enter NASA Earthdata Login Password: ']

# Determine the OS (Windows machines usually use an '_netrc' file)
netrc_name = "_netrc" if system()=="Windows" else ".netrc"

# Determine if netrc file exists, and if so, if it includes NASA Earthdata Login Credentials
try:
    netrcDir = os.path.expanduser(f"~/{netrc_name}")
    netrc(netrcDir).authenticators(urs)[0]

# Below, create a netrc file and prompt user for NASA Earthdata Login Username and Password
except FileNotFoundError:
    homeDir = os.path.expanduser("~")
    Popen('touch {0}{2} | echo machine {1} >> {0}{2}'.format(homeDir + os.sep, urs, netrc_name), shell=True)
    Popen('echo login {} >> {}{}'.format(getpass(prompt=prompts[0]), homeDir + os.sep, netrc_name), shell=True)
    Popen('echo \'password {} \'>> {}{}'.format(getpass(prompt=prompts[1]), homeDir + os.sep, netrc_name), shell=True)
    # Set restrictive permissions
    Popen('chmod 0600 {0}{1}'.format(homeDir + os.sep, netrc_name), shell=True)

    # Determine OS and edit netrc file if it exists but is not set up for NASA Earthdata Login
except TypeError:
    homeDir = os.path.expanduser("~")
    Popen('echo machine {1} >> {0}{2}'.format(homeDir + os.sep, urs, netrc_name), shell=True)
    Popen('echo login {} >> {}{}'.format(getpass(prompt=prompts[0]), homeDir + os.sep, netrc_name), shell=True)
    Popen('echo \'password {} \'>> {}{}'.format(getpass(prompt=prompts[1]), homeDir + os.sep, netrc_name), shell=True)

In [3]:

# Generates the temporary
s3_cred_endpoint = 'https://archive.podaac.earthdata.nasa.gov/s3credentials'
def get_temp_creds():
    temp_creds_url = s3_cred_endpoint
    return requests.get(temp_creds_url).json()
temp_creds_req = get_temp_creds()

session = boto3.Session(aws_access_key_id=temp_creds_req['accessKeyId'], 
                        aws_secret_access_key=temp_creds_req['secretAccessKey'],
                        aws_session_token=temp_creds_req['sessionToken'],
                        region_name='us-west-2')

rio_env = rio.Env(AWSSession(session),
                  GDAL_DISABLE_READDIR_ON_OPEN='EMPTY_DIR',
                  CPL_VSIL_CURL_ALLOWED_EXTENSIONS='TIF, TIFF',
                  GDAL_HTTP_COOKIEFILE=os.path.expanduser('~/cookies.txt'),
                  GDAL_HTTP_COOKIEJAR=os.path.expanduser('~/cookies.txt'))
rio_env.__enter__()

Out[3]:

<rasterio.env.Env at 0x7f9ad0c71e10>

In [4]:

# HTTPS links work
B01_T42RUQ_HTTPS = xr.open_dataset('https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V0/OPERA_L3_DSWx-HLS_T42RUQ_20220930T055641Z_20230112T024042Z_S2A_30_v0.0_B01_WTR.tif', engine='rasterio')
B01_T42RUQ_HTTPS

In [5]:

## S3 links does not work
B01_T42RUQ_S3 = xr.open_dataset('s3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V0/OPERA_L3_DSWx-HLS_T42RUQ_20220930T055641Z_20230112T024042Z_S2A_30_v0.0_B01_WTR.tif', engine='rasterio')
B01_T42RUQ_S3 

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
~/opt/anaconda3/envs/opera_app/lib/python3.7/site-packages/xarray/backends/file_manager.py in _acquire_with_cache_info(self, needs_lock)
    198             try:
--> 199                 file = self._cache[self._key]
    200             except KeyError:

~/opt/anaconda3/envs/opera_app/lib/python3.7/site-packages/xarray/backends/lru_cache.py in __getitem__(self, key)
     52         with self._lock:
---> 53             value = self._cache[key]
     54             self._cache.move_to_end(key)

KeyError: [<function open at 0x7f9af12ddef0>, ('s3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V0/OPERA_L3_DSWx-HLS_T42RUQ_20220930T055641Z_20230112T024042Z_S2A_30_v0.0_B01_WTR.tif',), 'r', (('sharing', False),)]

During handling of the above exception, another exception occurred:

CPLE_OpenFailedError                      Traceback (most recent call last)
rasterio/_base.pyx in rasterio._base.DatasetBase.__init__()

rasterio/_shim.pyx in rasterio._shim.open_dataset()

rasterio/_err.pyx in rasterio._err.exc_wrap_pointer()

CPLE_OpenFailedError: '/vsis3/podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V0/OPERA_L3_DSWx-HLS_T42RUQ_20220930T055641Z_20230112T024042Z_S2A_30_v0.0_B01_WTR.tif' not recognized as a supported file format.

During handling of the above exception, another exception occurred:

RasterioIOError                           Traceback (most recent call last)
/var/folders/1j/j4ygy1w57njg5mq703dy9df00000gq/T/ipykernel_47195/1335951124.py in <module>
      1 ## S3 links does not work
----> 2 B01_T42RUQ_S3 = xr.open_dataset('s3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V0/OPERA_L3_DSWx-HLS_T42RUQ_20220930T055641Z_20230112T024042Z_S2A_30_v0.0_B01_WTR.tif', engine='rasterio')
      3 B01_T42RUQ_S3

~/opt/anaconda3/envs/opera_app/lib/python3.7/site-packages/xarray/backends/api.py in open_dataset(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, backend_kwargs, *args, **kwargs)
    497         drop_variables=drop_variables,
    498         **decoders,
--> 499         **kwargs,
    500     )
    501     ds = _dataset_from_backend_dataset(

~/opt/anaconda3/envs/opera_app/lib/python3.7/site-packages/rioxarray/xarray_plugin.py in open_dataset(self, filename_or_obj, drop_variables, parse_coordinates, chunks, cache, lock, masked, mask_and_scale, variable, group, default_name, decode_times, decode_timedelta, open_kwargs)
     66             decode_times=decode_times,
     67             decode_timedelta=decode_timedelta,
---> 68             **open_kwargs,
     69         )
     70         if isinstance(rds, xr.DataArray):

~/opt/anaconda3/envs/opera_app/lib/python3.7/site-packages/rioxarray/_io.py in open_rasterio(filename, parse_coordinates, chunks, cache, lock, masked, mask_and_scale, variable, group, default_name, decode_times, decode_timedelta, **open_kwargs)
    831         else:
    832             manager = URIManager(rasterio.open, filename, mode="r", kwargs=open_kwargs)
--> 833         riods = manager.acquire()
    834         captured_warnings = rio_warnings.copy()
    835 

~/opt/anaconda3/envs/opera_app/lib/python3.7/site-packages/xarray/backends/file_manager.py in acquire(self, needs_lock)
    179             An open file object, as returned by ``opener(*args, **kwargs)``.
    180         """
--> 181         file, _ = self._acquire_with_cache_info(needs_lock)
    182         return file
    183 

~/opt/anaconda3/envs/opera_app/lib/python3.7/site-packages/xarray/backends/file_manager.py in _acquire_with_cache_info(self, needs_lock)
    203                     kwargs = kwargs.copy()
    204                     kwargs["mode"] = self._mode
--> 205                 file = self._opener(*self._args, **kwargs)
    206                 if self._mode == "w":
    207                     # ensure file doesn't get overriden when opened again

~/opt/anaconda3/envs/opera_app/lib/python3.7/site-packages/rasterio/env.py in wrapper(*args, **kwds)
    433 
    434         with env_ctor(session=session):
--> 435             return f(*args, **kwds)
    436 
    437     return wrapper

~/opt/anaconda3/envs/opera_app/lib/python3.7/site-packages/rasterio/__init__.py in open(fp, mode, driver, width, height, count, crs, transform, dtype, nodata, sharing, **kwargs)
    218         # None.
    219         if mode == 'r':
--> 220             s = DatasetReader(path, driver=driver, sharing=sharing, **kwargs)
    221         elif mode == "r+":
    222             s = get_writer_for_path(path, driver=driver)(

rasterio/_base.pyx in rasterio._base.DatasetBase.__init__()

RasterioIOError: '/vsis3/podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V0/OPERA_L3_DSWx-HLS_T42RUQ_20220930T055641Z_20230112T024042Z_S2A_30_v0.0_B01_WTR.tif' not recognized as a supported file format.

In [ ]:

rio_env.__exit__()