In [11]:
import numpy as np
import matplotlib.pyplot as plt
import netCDF4 as nc
import pandas as pd
import datetime as dt
from salishsea_tools import places
import glob
from pandas.plotting import register_matplotlib_converters
register_matplotlib_converters()
%matplotlib inline
In [10]:
f=nc.Dataset('/data/sallen/results/MEOPAR/tuning/01nov14/S3_201501-201501.nc')
In [11]:
f.variables.keys()
Out[11]:
dict_keys(['nav_lat', 'nav_lon', 'bounds_nav_lon', 'bounds_nav_lat', 'area', 'deptht_15m', 'deptht_15m_bounds', 'time_centered', 'time_centered_bounds', 'time_counter', 'time_counter_bounds', 'nitrate', 'silicon', 'diatoms', 'flagellates', 'ciliates'])
In [12]:
f.variables['diatoms']
Out[12]:
<class 'netCDF4._netCDF4.Variable'>
float32 diatoms(time_counter, deptht_15m, y, x)
standard_name: mole_concentration_of_diatoms_expressed_as_nitrogen_in_sea_water
long_name: Diatoms Concentration
units: mmol m-3
online_operation: average
interval_operation: 40 s
interval_write: 1 h
cell_methods: time: mean (interval: 40 s)
cell_measures: area: area
_FillValue: 1e+20
missing_value: 1e+20
coordinates: time_centered nav_lat nav_lon
unlimited dimensions: time_counter
current shape = (744, 15, 1, 1)
filling on
In [2]:
# load ferry chl
llon=places.PLACES['S3']['lon lat'][0]-.01
ulon=places.PLACES['S3']['lon lat'][0]+.01
llat=places.PLACES['S3']['lon lat'][1]-.01
ulat=places.PLACES['S3']['lon lat'][1]+.01
dfs=list()
#2015
df=pd.read_csv('/ocean/eolson/MEOPAR/obs/ONC/turbidity/nearSurface/search3928586/BritishColumbiaFerries_Tsawwassen-DukePoint_Turbidity-ChlorophyllandFluorescence_20140804T234330Z_20150604T070614Z-clean.csv',
skiprows=78,header=None,
names=('TimeUTC','CDOM','CDOMQC','Chlorophyll_ug','ChlQC','Turbidity_NTU','TurbQC','Lat','LatQC','Lon','LongQC'))
df['Lat']=pd.to_numeric(df['Lat'],errors='coerce')
df['Lon']=pd.to_numeric(df['Lon'],errors='coerce')
df['Chlorophyll_ug']=pd.to_numeric(df['Chlorophyll_ug'],errors='coerce')
iidfnd=(df.Lon>llon)&(df.Lon<ulon)&(df.Lat>llat)&(df.Lat<ulat)&(df.Chlorophyll_ug>=0)
dfs.append(df.loc[iidfnd].reset_index().copy(deep=True))
df=pd.read_csv('/ocean/eolson/MEOPAR/obs/ONC/turbidity/nearSurface/search3928586/BritishColumbiaFerries_Tsawwassen-DukePoint_Turbidity-ChlorophyllandFluorescence_20150604T070624Z_20160307T160206Z-clean.csv',
skiprows=78,header=None,
names=('TimeUTC','CDOM','CDOMQC','Chlorophyll_ug','ChlQC','Turbidity_NTU','TurbQC','Lat','LatQC','Lon','LongQC'))
df['Lat']=pd.to_numeric(df['Lat'],errors='coerce')
df['Lon']=pd.to_numeric(df['Lon'],errors='coerce')
df['Chlorophyll_ug']=pd.to_numeric(df['Chlorophyll_ug'],errors='coerce')
iidfnd=(df.Lon>llon)&(df.Lon<ulon)&(df.Lat>llat)&(df.Lat<ulat)&(df.Chlorophyll_ug>=0)
dfs.append(df.loc[iidfnd].reset_index().copy(deep=True))
#2016
df=pd.read_csv('/ocean/eolson/MEOPAR/obs/ONC/turbidity/nearSurface/search3928586/BritishColumbiaFerries_Tsawwassen-DukePoint_Turbidity-ChlorophyllandFluorescence_20150604T070624Z_20160307T160206Z-clean.csv',
skiprows=78,header=None,
names=('TimeUTC','CDOM','CDOMQC','Chlorophyll_ug','ChlQC','Turbidity_NTU','TurbQC','Lat','LatQC','Lon','LongQC'))
df['Lat']=pd.to_numeric(df['Lat'],errors='coerce')
df['Lon']=pd.to_numeric(df['Lon'],errors='coerce')
df['Chlorophyll_ug']=pd.to_numeric(df['Chlorophyll_ug'],errors='coerce')
iidfnd=(df.Lon>llon)&(df.Lon<ulon)&(df.Lat>llat)&(df.Lat<ulat)&(df.Chlorophyll_ug>=0)
dfs.append(df.loc[iidfnd].reset_index().copy(deep=True))
df=pd.read_csv('/ocean/eolson/MEOPAR/obs/ONC/turbidity/nearSurface/search3928586/BritishColumbiaFerries_Tsawwassen-DukePoint_Turbidity-ChlorophyllandFluorescence_20160307T160215Z_20161019T064047Z-clean.csv',
skiprows=78,header=None,
names=('TimeUTC','CDOM','CDOMQC','Chlorophyll_ug','ChlQC','Turbidity_NTU','TurbQC','Lat','LatQC','Lon','LongQC'))
df['Lat']=pd.to_numeric(df['Lat'],errors='coerce')
df['Lon']=pd.to_numeric(df['Lon'],errors='coerce')
df['Chlorophyll_ug']=pd.to_numeric(df['Chlorophyll_ug'],errors='coerce')
iidfnd=(df.Lon>llon)&(df.Lon<ulon)&(df.Lat>llat)&(df.Lat<ulat)&(df.Chlorophyll_ug>=0)
dfs.append(df.loc[iidfnd].reset_index().copy(deep=True))
df=pd.read_csv('/ocean/eolson/MEOPAR/obs/ONC/turbidity/nearSurface/search3928586/BritishColumbiaFerries_Tsawwassen-DukePoint_Turbidity-ChlorophyllandFluorescence_20161019T064056Z_20161115T220747Z-clean.csv',
skiprows=78,header=None,
names=('TimeUTC','CDOM','CDOMQC','Chlorophyll_ug','ChlQC','Turbidity_NTU','TurbQC','Lat','LatQC','Lon','LongQC'))
df['Lat']=pd.to_numeric(df['Lat'],errors='coerce')
df['Lon']=pd.to_numeric(df['Lon'],errors='coerce')
df['Chlorophyll_ug']=pd.to_numeric(df['Chlorophyll_ug'],errors='coerce')
iidfnd=(df.Lon>llon)&(df.Lon<ulon)&(df.Lat>llat)&(df.Lat<ulat)&(df.Chlorophyll_ug>=0)
dfs.append(df.loc[iidfnd].reset_index().copy(deep=True))
#2017
flist=glob.glob('/ocean/eolson/MEOPAR/obs/ONC/turbidity/nearSurface/search4979680/*Tsawwassen-DukePoint_Turbidity*')
for ii in range(0,len(flist)):
df=pd.read_csv(flist[ii],
skiprows=67,header=None,usecols=[0,1,2,3,4,5,6,7,8,9,10],
names=('TimeUTC','CDOM','CDOMQC','Chlorophyll_ug','ChlQC','Turbidity_NTU','TurbQC','Lat','LatQC','Lon','LongQC'))
df['Lat']=pd.to_numeric(df['Lat'],errors='coerce')
df['Lon']=pd.to_numeric(df['Lon'],errors='coerce')
df['Chlorophyll_ug']=pd.to_numeric(df['Chlorophyll_ug'],errors='coerce')
iidfnd=(df.Lon>llon)&(df.Lon<ulon)&(df.Lat>llat)&(df.Lat<ulat)&(df.Chlorophyll_ug>=0)
dfs.append(df.loc[iidfnd].reset_index().copy(deep=True))
#2017-2018
flist=glob.glob('/ocean/eolson/MEOPAR/obs/ONC/turbidity/nearSurface/search7672208/*Tsawwassen-DukePoint_Turbidity*')
for ii in range(0,len(flist)):
df=pd.read_csv(flist[ii],
skiprows=59,header=None,usecols=[0,1,2,3,4,5,6,7,8,9,10],
names=('TimeUTC','CDOM','CDOMQC','Chlorophyll_ug','ChlQC','Turbidity_NTU','TurbQC','Lat','LatQC','Lon','LongQC'))
df['Lat']=pd.to_numeric(df['Lat'],errors='coerce')
df['Lon']=pd.to_numeric(df['Lon'],errors='coerce')
df['Chlorophyll_ug']=pd.to_numeric(df['Chlorophyll_ug'],errors='coerce')
iidfnd=(df.Lon>llon)&(df.Lon<ulon)&(df.Lat>llat)&(df.Lat<ulat)&(df.Chlorophyll_ug>=0)
dfs.append(df.loc[iidfnd].reset_index().copy(deep=True))
#2018-
flist=glob.glob('/ocean/eolson/MEOPAR/obs/ONC/turbidity/nearSurface/search13411521/*Tsawwassen-DukePoint_Turbidity*')
for ii in range(0,len(flist)):
df=pd.read_csv(flist[ii],
skiprows=61,header=None,usecols=[0,1,2,3,4,5,6,7,8,9,10],
names=('TimeUTC','CDOM','CDOMQC','Chlorophyll_ug','ChlQC','Turbidity_NTU','TurbQC','Lat','LatQC','Lon','LongQC'))
df['Lat']=pd.to_numeric(df['Lat'],errors='coerce')
df['Lon']=pd.to_numeric(df['Lon'],errors='coerce')
df['Chlorophyll_ug']=pd.to_numeric(df['Chlorophyll_ug'],errors='coerce')
iidfnd=(df.Lon>llon)&(df.Lon<ulon)&(df.Lat>llat)&(df.Lat<ulat)&(df.Chlorophyll_ug>=0)
dfs.append(df.loc[iidfnd].reset_index().copy(deep=True))
df0=pd.concat(dfs,ignore_index=True)
dts=[dt.datetime(int(r[0:4]),int(r[5:7]),int(r[8:10]),int(r[11:13]),int(r[14:16]),int(r[17:19])) for r in df0['TimeUTC']]
df0=df0.assign(dts=dts)
df0['Year']=[int(ii.year) for ii in df0['dts']]
/home/eolson/anaconda3/envs/py38_ct_pmill/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3062: DtypeWarning: Columns (3) have mixed types. Specify dtype option on import or set low_memory=False. has_raised = await self.run_ast_nodes(code_ast.body, cell_name, /home/eolson/anaconda3/envs/py38_ct_pmill/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3062: DtypeWarning: Columns (1,3,5) have mixed types. Specify dtype option on import or set low_memory=False. has_raised = await self.run_ast_nodes(code_ast.body, cell_name, /home/eolson/anaconda3/envs/py38_ct_pmill/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3062: DtypeWarning: Columns (7,9) have mixed types. Specify dtype option on import or set low_memory=False. has_raised = await self.run_ast_nodes(code_ast.body, cell_name, /home/eolson/anaconda3/envs/py38_ct_pmill/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3062: DtypeWarning: Columns (1) have mixed types. Specify dtype option on import or set low_memory=False. has_raised = await self.run_ast_nodes(code_ast.body, cell_name, /home/eolson/anaconda3/envs/py38_ct_pmill/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3062: DtypeWarning: Columns (3,5,7,9) have mixed types. Specify dtype option on import or set low_memory=False. has_raised = await self.run_ast_nodes(code_ast.body, cell_name, /home/eolson/anaconda3/envs/py38_ct_pmill/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3062: DtypeWarning: Columns (1,3,5,7,9) have mixed types. Specify dtype option on import or set low_memory=False. has_raised = await self.run_ast_nodes(code_ast.body, cell_name, /home/eolson/anaconda3/envs/py38_ct_pmill/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3062: DtypeWarning: Columns (5) have mixed types. Specify dtype option on import or set low_memory=False. has_raised = await self.run_ast_nodes(code_ast.body, cell_name, /home/eolson/anaconda3/envs/py38_ct_pmill/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3062: DtypeWarning: Columns (1,3,7,9) have mixed types. Specify dtype option on import or set low_memory=False. has_raised = await self.run_ast_nodes(code_ast.body, cell_name, /home/eolson/anaconda3/envs/py38_ct_pmill/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3062: DtypeWarning: Columns (3,7,9) have mixed types. Specify dtype option on import or set low_memory=False. has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
In [19]:
lcol={'diatoms':'b','flagellates':'m','ciliates':'r'}
with nc.Dataset('/ocean/eolson/MEOPAR/NEMO-forcing/grid/mesh_mask201702_noLPE.nc') as fm:
e3t_0=np.copy(fm.variables['e3t_0'])
def plotModChl(ax,f):
torig=dt.datetime.strptime(f.variables['time_centered'].time_origin,'%Y-%m-%d %H:%M:%S')
times=np.array([torig + dt.timedelta(seconds=ii) for ii in f.variables['time_centered'][:]])
for var in ('diatoms','flagellates','ciliates'):
try:
ax.plot(times,2.0*np.sum(f.variables[var][:,:3,0,0]*f.variables['e3t'][:,:3,0,0],1)/np.sum(f.variables['e3t'][:,:3,0,0],1),
linestyle='--',color=lcol[var],alpha=.5,label=var)
except:
ax.plot(times,2.0*np.sum(f.variables[var][:,:3,0,0]*np.tile(e3t_0[:,:3,0,0],(len(times),1)),1)/np.sum(np.tile(e3t_0[:,:3,0,0],(len(times),1)),1),
linestyle='--',color=lcol[var],alpha=.5,label=var)
try:
ax.plot(times,2.0*np.sum((f.variables['diatoms'][:,:3,0,0]+f.variables['flagellates'][:,:3,0,0]+f.variables['ciliates'][:,:3,0,0])*f.variables['e3t'][:,:3,0,0],1)/np.sum(f.variables['e3t'][:,:3,0,0],1),
linestyle='-',color='k',alpha=.6,label='total')
except:
ax.plot(times,2.0*np.sum((f.variables['diatoms'][:,:3,0,0]+f.variables['flagellates'][:,:3,0,0]+f.variables['ciliates'][:,:3,0,0])*np.tile(e3t_0[:,:3,0,0],(len(times),1)),1)/np.sum(np.tile(e3t_0[:,:3,0,0],(len(times),1)),1),
linestyle='-',color='k',alpha=.6,label='total')
In [20]:
base='/data/sallen/results/MEOPAR/tuning/'
flist15=('01nov14/S3_201501-201501.nc',
'01nov14/S3_201502-201502.nc',
'01nov14/S3_201503-201503.nc',
'01nov14/S3_201504-201504.nc',
'01may15/S3_201505-201505.nc',
'01may15/S3_201506-201506.nc',
'01jul15/S3_201507-201507.nc',
'01jul15/S3_201508-201508.nc',
'01jul15/S3_201509-201509.nc',
'01jul15/S3_201510-201510.nc',)
flist16=('01mar16_secondtime/S3_201603-201603.nc',
'01mar16_secondtime/S3_201604-201604.nc',)
flist17=('01nov16/S3_201701-201701.nc',
'01nov16/S3_201702-201702.nc',
'01nov16/S3_201703-201703.nc',
'01nov16/S3_201704-201704.nc',
'31mar17/S3_201703-201704.nc',
'31mar17/S3_201704-201705.nc',)
In [25]:
fig,ax=plt.subplots(3,1,figsize=(12,12))
fer15=df0.loc[df0.Year==2015]
ax[0].plot(fer15['dts'],fer15['Chlorophyll_ug'],'g*',label='ferry')
for file in flist15:
with nc.Dataset(base+file) as f:
plotModChl(ax[0],f)
fer16=df0.loc[df0.Year==2016]
ax[1].plot(fer16['dts'],fer16['Chlorophyll_ug'],'g*',label='ferry')
for file in flist16:
with nc.Dataset(base+file) as f:
plotModChl(ax[1],f)
fer17=df0.loc[df0.Year==2017]
ax[2].plot(fer17['dts'],fer17['Chlorophyll_ug'],'g*',label='ferry')
for file in flist17:
with nc.Dataset(base+file) as f:
plotModChl(ax[2],f)
for iax in ax:
iax.set_ylim(0,40)
In [ ]: