Development of Algorithms for ERDDAP Dataset from ONC CTD Data¶

from collections import OrderedDict
import os

import arrow
from lxml import etree
import matplotlib.pyplot as plt
import numpy as np
import xarray as xr

from salishsea_tools import data_tools
from salishsea_tools.places import PLACES

%matplotlib inline

TOKEN = os.environ['ONC_USER_TOKEN']

onc_data = data_tools.get_onc_data(
    'scalardata', 'getByStation', TOKEN,
    station='SCVIP', deviceCategory='CTD',
    sensors='salinity,temperature',
    dateFrom=data_tools.onc_datetime('2015-12-27 00:00', 'utc'),
)

ctd_data = data_tools.onc_json_to_dataset(onc_data)

salinity_qaqc_mask = ctd_data.salinity.attrs['qaqcFlag'] == 1
salinity = xr.DataArray(
    name='salinity',
    data=ctd_data.salinity[salinity_qaqc_mask].values,
    coords={'time': ctd_data.salinity.sampleTime[salinity_qaqc_mask].values},
)

temperature_qaqc_mask = ctd_data.temperature.attrs['qaqcFlag'] == 1
temperature = xr.DataArray(
    name='temperature',
    data=ctd_data.temperature[temperature_qaqc_mask].values,
    coords={'time': ctd_data.temperature.sampleTime[temperature_qaqc_mask].values},
)

xr_metadata = {
    'SCVIP': {
        'place_name': 'Central node',
        'ONC_station': 'Central',
        'ONC_stationCode': PLACES['Central node']['ONC stationCode'],
        'ONC_stationDescription': 
            'Pacific, Salish Sea, Strait of Georgia, Central, Strait of Georgia VENUS Instrument Platform',
        'ONC_data_product_url': 'http://dmas.uvic.ca/DataSearch?location=SCVIP&deviceCategory=CTD',
    },
    'SEVIP': {
        'place_name': 'East node',
        'ONC_station': 'East',
        'ONC_stationCode': PLACES['East node']['ONC stationCode'],
        'ONC_stationDescription': 
            'Pacific, Salish Sea, Strait of Georgia, East, Strait of Georgia VENUS Instrument Platform',
        'ONC_data_product_url': 'http://dmas.uvic.ca/DataSearch?location=SEVIP&deviceCategory=CTD',
    },
}

def count(values, axis):
    return values.size

onc_station = 'SCVIP'

ds = xr.Dataset(
    data_vars={
        'salinity': xr.DataArray(
            name='salinity',
            data=salinity.resample('15Min', 'time', how='mean'),
            attrs={
                'ioos_category': 'Salinity',
                'standard_name': 'sea_water_reference_salinity',
                'long_name': 'reference salinity',
                'units': 'g/kg',
                'aggregation_operation': 'mean',
                'aggregation_interval': 15*60,
                'aggregation_interval_units': 'seconds',
            },
        ),
        'salinity_std_dev': xr.DataArray(
            name='salinity_std_dev',
            data=salinity.resample('15Min', 'time', how='std'),
            attrs={
                'ioos_category': 'Salinity',
                'standard_name':b 'sea_water_reference_salinity_standard_deviation',
                'long_name': 'reference salinity standard deviation',
                'units': 'g/kg',
                'aggregation_operation': 'standard deviation',
                'aggregation_interval': 15*60,
                'aggregation_interval_units': 'seconds',
            },
        ),
        'salinity_sample_count': xr.DataArray(
            name='salinity_sample_count',
            data=salinity.resample('15Min', 'time', how=count),
            attrs={
                'standard_name': 'sea_water_reference_salinity_sample_count',
                'long_name': 'reference salinity sample count',
                'aggregation_operation': 'count',
                'aggregation_interval': 15*60,
                'aggregation_interval_units': 'seconds',
            },
        ),
        'temperature': xr.DataArray(
            name='temperature',
            data=temperature.resample('15Min', 'time', how='mean'),
            attrs={
                'ioos_category': 'Temperature',
                'standard_name': 'sea_water_temperature',
                'long_name': 'temperature',
                'units': 'degrees_Celcius',
                'aggregation_operation': 'mean',
                'aggregation_interval': 15*60,
                'aggregation_interval_units': 'seconds',
            },
        ),
        'temperature_std_dev': xr.DataArray(
            name='temperature_std_dev',
            data=temperature.resample('15Min', 'time', how='std'),
            attrs={
                'ioos_category': 'Temperature',
                'standard_name': 'sea_water_temperature_standard_deviation',
                'long_name': 'temperature standard deviation',
                'units': 'degrees_Celcius',
                'aggregation_operation': 'standard deviation',
                'aggregation_interval': 15*60,
                'aggregation_interval_units': 'seconds',
            },
        ),
        'temperature_sample_count': xr.DataArray(
            name='temperature_sample_count',
            data=temperature.resample('15Min', 'time', how=count),
            attrs={
                'standard_name': 'sea_water_temperature_sample_count',
                'long_name': 'temperature sample count',
                'aggregation_operation': 'count',
                'aggregation_interval': 15*60,
                'aggregation_interval_units': 'seconds',
            },
        ),
    },
    coords={
        'depth': PLACES[xr_metadata[onc_station]['place_name']]['depth'],
        'longitude': PLACES[xr_metadata[onc_station]['place_name']]['lon lat'][0],
        'latitude': PLACES[xr_metadata[onc_station]['place_name']]['lon lat'][1],
    },
    attrs={
        'history': """
{0} Download raw data from ONC scalardata API.
{0} Filter to exclude data with qaqcFlag != 1.
{0} Resample data to 15 minute intervals using mean, standard deviation and count as aggregation functions.
{0} Store as netCDF4 file.
        """.format(arrow.now().format('YYYY-MM-DD HH:mm:ss')),
        'ONC_station': xr_metadata[onc_station]['ONC_station'],
        'ONC_stationCode': PLACES[xr_metadata[onc_station]['place_name']]['ONC stationCode'],
        'ONC_stationDescription': xr_metadata[onc_station]['ONC_stationDescription'],
        'ONC_data_product_url': xr_metadata[onc_station]['ONC_data_product_url'],
    },
)

ds.salinity_sample_count.values = np.nan_to_num(ds.salinity_sample_count.values)
ds.temperature_sample_count.values = np.nan_to_num(ds.temperature_sample_count.values)

ds.to_netcdf(
    '/results/observations/ONC/CTD/{station}/{station}_CTD_15m_20151227.nc'
    .format(station=onc_station),
    encoding={'time': {'units': 'minutes since 1970-01-01 00:00'}})

metadata = OrderedDict([
    ('cdm_data_type', {'text': 'TimeSeries'}),
    ('cdm_timeseries_variables', {
        'text': 'depth, longitude, latitude',
        'after': 'cdm_data_type',
    }),
    ('institution_fullname', {
        'text': 'Earth, Ocean & Atmospheric Sciences, University of British Columbia',
        'after': 'institution',
    }),
    ('license', {
        'text': '''The Salish Sea MEOPAR observation datasets are copyright 2013 – present
by the Salish Sea MEOPAR Project Contributors, The University of British Columbia, and Ocean Networks Canada.

They are licensed under the Apache License, Version 2.0. http://www.apache.org/licenses/LICENSE-2.0

Raw instrument data on which this dataset is based were provided by Ocean Networks Canada.''',
    }),
    ('project', {
        'text':'Salish Sea MEOPAR NEMO Model',
        'after': 'title',
    }),
    ('creator_name', {
        'text': 'Salish Sea MEOPAR Project Contributors',
    }),
    ('creator_email', {
        'text': 'sallen@eos.ubc.ca',
        'after': 'creator_name',
    }),
    ('creator_url', {
        'text': 'https://salishsea-meopar-docs.readthedocs.org/',
    }),
    ('acknowledgement', {
        'text': 'MEOPAR, ONC, Compute Canada',
        'after': 'creator_url',
    }),
    ('drawLandMask', {
        'text': 'over',
        'after': 'acknowledgement',
    }),
])

datasets = {
    'ubcONCSCVIPCTD15mV1' :{
        'type': 'resampled CTD',
        'title': 'ONC, Strait of Georgia, Central Node, Salinity and Temperature, 15min, v1',
        'summary':'''Temperature and salinity data from the Ocean Networks Canada (ONC)
Strait of Georgia Central Node VENUS Instrument Platform CTD.
The data are resampled from the raw instrument data to 15 minute mean values.
They are accompanied by standard deviations and sample counts for each of the 15 minute
aggregation intervals.

v1: reference salinity, reference salinity standard deviation, reference salinity sample counts, 
temperature, temperature standard deviation, temperature sample counts variables''',
        'keywords': '''15min aggregation, ONC Central Node VENUS Instrument Platform, Ocean Networks Canada,
depth, UBC EOAS, Strait of Georgia, latitude, longitude, ocean, SCVIP, observations, CTD,
Oceans > Ocean Temperature > Water Temperature,
reference salinity, salinity_sample_count, salinity_std_dev, sea_water_reference_salinity, 
sea_water_reference_salinity_sample_count, sea_water_reference_salinity_standard_deviation, 
sea_water_temperature, sea_water_temperature_sample_count, sea_water_temperature_standard_deviation, 
temperature, temperature_sample_count, temperature_std_dev, time''',
        'fileNameRegex': '.*SCVIP_CTD_15m_\d{8}\.nc$'
    },
    
    'ubcONCSEVIPCTD15mV1' :{
        'type': 'resampled CTD',
        'title': 'ONC, Strait of Georgia, East Node, Salinity and Temperature, 15min, v1',
        'summary':'''Temperature and salinity data from the Ocean Networks Canada (ONC)
Strait of Georgia East Node VENUS Instrument Platform CTD.
The data are resampled from the raw instrument data to 15 minute mean values.
They are accompanied by standard deviations and sample counts for each of the 15 minute
aggregation intervals.

v1: reference salinity, reference salinity standard deviation, reference salinity sample counts, 
temperature, temperature standard deviation, temperature sample counts variables''',
        'keywords': '''15min aggregation, ONC East Node VENUS Instrument Platform, Ocean Networks Canada,
depth, UBC EOAS, Strait of Georgia, latitude, longitude, ocean, SEVIP, observations, CTD,
Oceans > Ocean Temperature > Water Temperature,
reference salinity, salinity_sample_count, salinity_std_dev, sea_water_reference_salinity, 
sea_water_reference_salinity_sample_count, sea_water_reference_salinity_standard_deviation, 
sea_water_temperature, sea_water_temperature_sample_count, sea_water_temperature_standard_deviation, 
temperature, temperature_sample_count, temperature_std_dev, time''',
        'fileNameRegex': '.*SEVIP_CTD_15m_\d{8}\.nc$'
    },
}

def print_tree(root):
    """Display an XML tree fragment with indentation.
    """
    print(etree.tostring(root, pretty_print=True).decode('ascii'))

def find_att(root, att):
    """Return the dataset attribute element named att
    or raise a ValueError exception if it cannot be found.
    """
    e = root.find('.//att[@name="{}"]'.format(att))
    if e is None:
        raise ValueError('{} attribute element not found'.format(att))
    return e

parser = etree.XMLParser(remove_blank_text=True)
tree = etree.parse('/results/erddap/logs/GenerateDatasetsXml.out', parser)
root = tree.getroot()

datasetID = 'ubcONCSEVIPCTD15mV1'

root.attrib['datasetID'] = datasetID
root.find('.//recursive').text = 'false'
root.find('.//fileNameRegex').text = datasets[datasetID]['fileNameRegex']

e = etree.Element('att', name='cf_role')
e.text = 'timeseries_id'
tree.find('//dataVariable[{}]/addAttributes'.format(vars.index('time')+1)).append(e)

for att, info in metadata.items():
    e = etree.Element('att', name=att)
    e.text = info['text']
    try:
        root.find('.//att[@name="{}"]'.format(info['after'])).addnext(e)
    except KeyError:
        find_att(root, att).text = info['text']

title = datasets[datasetID]['title']
find_att(root, 'title').text = title
find_att(root, 'summary').text = '{0}\n\n{1}'.format(title, datasets[datasetID]['summary'])
find_att(root, 'keywords').text = datasets[datasetID]['keywords']

# Salinity colour map limits
e = tree.find(
    '//dataVariable[{}]/addAttributes/att[@name="colorBarMinimum"]'
    .format(vars.index('salinity')+1))
e.text = '0.0'
e = tree.find(
    '//dataVariable[{}]/addAttributes/att[@name="colorBarMaximum"]'
    .format(vars.index('salinity')+1))
e.text = '34.0'

# Temperature colour map limits
e = tree.find(
    '//dataVariable[{}]/addAttributes/att[@name="colorBarMinimum"]'
    .format(vars.index('temperature')+1))
e.text = '4.0'
e = tree.find(
    '//dataVariable[{}]/addAttributes/att[@name="colorBarMaximum"]'
    .format(vars.index('temperature')+1))
e.text = '20.0'

# Depth colour map limits
e = tree.find(
    '//dataVariable[{}]/addAttributes/att[@name="colorBarMinimum"]'
    .format(vars.index('depth')+1))
e.text = '0.0'
e = tree.find(
    '//dataVariable[{}]/addAttributes/att[@name="colorBarMaximum"]'
    .format(vars.index('depth')+1))
e.text = '450.0'

# Delete nonsensical colourBar* attributes
no_cbar_vars = [
    'temperature_sample_count', 'temperature_std_dev',
    'salinity_sample_count', 'salinity_std_dev']
for var in no_cbar_vars:
    for att in ('colorBarMinimum', 'colorBarMaximum'):
        e = tree.find(
            '//dataVariable[{0}]/addAttributes/att[@name="{1}"]'
            .format(vars.index(var)+1, att))
        e.getparent().remove(e)

with open('/results/erddap_datasets_xml/{}.xml'.format(datasetID), 'wb') as f:
    f.write(etree.tostring(root, pretty_print=True))