import requests
import numpy as np
import pandas as pd
import cmocean
import matplotlib.pylab as plt
from scipy.interpolate import griddata
from scipy import interpolate
from datetime import datetime
import pdb
import os
import csv
from datetime import datetime, timedelta
import calendar
import matplotlib
matplotlib.font_manager._rebuild()
#used for map projections
from cartopy import config
import cartopy.crs as ccrs
import matplotlib.patches as mpatches
%matplotlib inline
#sets plot styles
import seaborn as sns
from matplotlib import rc
from matplotlib import rcParams
import matplotlib.ticker as mtick
rc('text', usetex=False)
rcStyle = {"font.size": 10,
"axes.titlesize": 20,
"axes.labelsize": 20,
'xtick.labelsize': 16,
'ytick.labelsize': 16}
sns.set_context("paper", rc=rcStyle)
sns.set_style("whitegrid", {'axes.grid' : False})
myColors = ["windows blue", "amber", "dusty rose", "prussian blue", "faded green", "dusty purple", "gold", "dark pink", "green", "red", "brown"]
colorsBW = ["black", "grey"]
sns.set_palette(sns.xkcd_palette(myColors))
curDir = os.getcwd()
dataDir = os.path.join(curDir, 'data')
if not os.path.exists(dataDir):
os.mkdir(dataDir)
import warnings
warnings.filterwarnings('ignore')
If you know that a profile contains BGC parameters, the standard profile api contains the bgc measurements under the field bgcMeas.
def get_profile(profile_number):
url = 'https://argovis.colorado.edu/catalog/profiles/{}'.format(profile_number)
resp = requests.get(url)
# Consider any status other than 2xx an error
if not resp.status_code // 100 == 2:
return "Error: Unexpected response {}".format(resp)
profile = resp.json()
return profile
def json2dataframe(profiles, measKey='measurements'):
""" convert json data to Pandas DataFrame """
# Make sure we deal with a list
if isinstance(profiles, list):
data = profiles
else:
data = [profiles]
# Transform
rows = []
for profile in data:
keys = [x for x in profile.keys() if x not in ['measurements', 'bgcMeas']]
meta_row = dict((key, profile[key]) for key in keys)
for row in profile[measKey]:
row.update(meta_row)
rows.append(row)
df = pd.DataFrame(rows)
return df
profileId = "5901069_270"
profile = get_profile(profileId)
df = json2dataframe([profile], 'bgcMeas')
> <ipython-input-44-da136a77d156>(20)json2dataframe() -> for profile in data:
df.head(5)
pres | pres_qc | psal | psal_qc | temp | temp_qc | doxy_qc | bgcMeasKeys | station_parameters | station_parameters_in_nc | ... | jcommopsPlatform | euroargoPlatform | formatted_station_parameters | roundLat | roundLon | strLat | strLon | date_formatted | id | doxy | |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | 5.400000 | 1 | 35.058998 | 1 | 19.774000 | 1 | 4 | [pres, psal, temp, doxy] | [pres, psal, temp] | [PRES, PSAL, TEMP, DOXY] | ... | http://www.jcommops.org/board/wa/Platform?ref=... | https://fleetmonitoring.euro-argo.eu/float/590... | [ pres, psal, temp] | 29.931 | -173.291 | 29.931 N | 173.291 W | 2009-12-31 | 5901069_270 | NaN |
1 | 9.300000 | 1 | 35.057999 | 1 | 19.778000 | 1 | 1 | [pres, psal, temp, doxy] | [pres, psal, temp] | [PRES, PSAL, TEMP, DOXY] | ... | http://www.jcommops.org/board/wa/Platform?ref=... | https://fleetmonitoring.euro-argo.eu/float/590... | [ pres, psal, temp] | 29.931 | -173.291 | 29.931 N | 173.291 W | 2009-12-31 | 5901069_270 | 227.421417 |
2 | 19.500000 | 1 | 35.057999 | 1 | 19.777000 | 1 | 1 | [pres, psal, temp, doxy] | [pres, psal, temp] | [PRES, PSAL, TEMP, DOXY] | ... | http://www.jcommops.org/board/wa/Platform?ref=... | https://fleetmonitoring.euro-argo.eu/float/590... | [ pres, psal, temp] | 29.931 | -173.291 | 29.931 N | 173.291 W | 2009-12-31 | 5901069_270 | 222.475571 |
3 | 29.700001 | 1 | 35.056000 | 1 | 19.768999 | 1 | 1 | [pres, psal, temp, doxy] | [pres, psal, temp] | [PRES, PSAL, TEMP, DOXY] | ... | http://www.jcommops.org/board/wa/Platform?ref=... | https://fleetmonitoring.euro-argo.eu/float/590... | [ pres, psal, temp] | 29.931 | -173.291 | 29.931 N | 173.291 W | 2009-12-31 | 5901069_270 | 219.328979 |
4 | 39.400002 | 1 | 35.055000 | 1 | 19.768999 | 1 | 1 | [pres, psal, temp, doxy] | [pres, psal, temp] | [PRES, PSAL, TEMP, DOXY] | ... | http://www.jcommops.org/board/wa/Platform?ref=... | https://fleetmonitoring.euro-argo.eu/float/590... | [ pres, psal, temp] | 29.931 | -173.291 | 29.931 N | 173.291 W | 2009-12-31 | 5901069_270 | 216.657944 |
5 rows × 49 columns
Platform metadata is queried separatly from the BGC data. This is to keep the payload small enough for the server to operate efficiently. Platform BGC data is queried by two parameters at a time.
def get_platform_profile_metadata(platform_number):
url = f'https://argovis.colorado.edu/catalog/platform_profile_metadata/{platform_number}'
print(url)
resp = requests.get(url)
# Consider any status other than 2xx an error
if not resp.status_code // 100 == 2:
return "Error: Unexpected response {}".format(resp)
platformMetadata = resp.json()
return platformMetadata
def get_platform_profile_data(platform_number, xaxis='doxy', yaxis='pres'):
url = 'https://argovis.colorado.edu/catalog/bgc_platform_data/{0}/?xaxis={1}&yaxis={2}'.format(platform_number, xaxis, yaxis)
print(url)
resp = requests.get(url)
# Consider any status other than 2xx an error
if not resp.status_code // 100 == 2:
return "Error: Unexpected response {}".format(resp)
platformData = resp.json()
return platformData
def join_platform_data(platformMetadata, platformData):
platforms = []
for idx, platform in enumerate(platformMetadata):
metadata_id = platform['_id']
data_id = platformData[idx]['_id']
if (metadata_id == data_id) and ('bgcMeas' in platformData[idx].keys()) and isinstance(platformData[idx]['bgcMeas'], list):
platform['bgcMeas'] = platformData[idx]['bgcMeas']
platforms.append(platform)
return platforms
We merge the metadata and data and convert it into a dataframe
platformMetadata = get_platform_profile_metadata(5901464)
platformData = get_platform_profile_data(5901464, 'doxy', 'pres')
platforms = join_platform_data(platformMetadata, platformData)
df = json2dataframe(platforms, 'bgcMeas')
https://argovis.colorado.edu/catalog/platform_profile_metadata/5901464 https://argovis.colorado.edu/catalog/bgc_platform_data/5901464/?xaxis=doxy&yaxis=pres
df.head()
doxy | doxy_qc | pres | pres_qc | _id | POSITIONING_SYSTEM | DATA_CENTRE | PI_NAME | WMO_INST_TYPE | DATA_MODE | ... | cycle_number | dac | platform_number | station_parameters_in_nc | nc_url | PARAMETER_DATA_MODE | bgcMeasKeys | containsBGC | DIRECTION | BASIN | |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | 71.110497 | 1 | 1797.489990 | 1 | 5901464_115 | GPS | AO | STEPHEN RISER, | 846 | D | ... | 115 | aoml | 5901464 | [PRES, PSAL, TEMP, DOXY] | ftp://ftp.ifremer.fr/ifremer/argo/dac/aoml/590... | [[D, D, D, D]] | [pres, psal, temp, doxy] | True | A | 2 |
1 | 75.924095 | 1 | 1848.199951 | 1 | 5901464_115 | GPS | AO | STEPHEN RISER, | 846 | D | ... | 115 | aoml | 5901464 | [PRES, PSAL, TEMP, DOXY] | ftp://ftp.ifremer.fr/ifremer/argo/dac/aoml/590... | [[D, D, D, D]] | [pres, psal, temp, doxy] | True | A | 2 |
2 | 79.144302 | 1 | 1897.660034 | 1 | 5901464_115 | GPS | AO | STEPHEN RISER, | 846 | D | ... | 115 | aoml | 5901464 | [PRES, PSAL, TEMP, DOXY] | ftp://ftp.ifremer.fr/ifremer/argo/dac/aoml/590... | [[D, D, D, D]] | [pres, psal, temp, doxy] | True | A | 2 |
3 | 81.345016 | 1 | 1947.329956 | 1 | 5901464_115 | GPS | AO | STEPHEN RISER, | 846 | D | ... | 115 | aoml | 5901464 | [PRES, PSAL, TEMP, DOXY] | ftp://ftp.ifremer.fr/ifremer/argo/dac/aoml/590... | [[D, D, D, D]] | [pres, psal, temp, doxy] | True | A | 2 |
4 | 84.040329 | 1 | 1997.800049 | 1 | 5901464_115 | GPS | AO | STEPHEN RISER, | 846 | D | ... | 115 | aoml | 5901464 | [PRES, PSAL, TEMP, DOXY] | ftp://ftp.ifremer.fr/ifremer/argo/dac/aoml/590... | [[D, D, D, D]] | [pres, psal, temp, doxy] | True | A | 2 |
5 rows × 34 columns
def get_bgc_selection_profiles(startDate, endDate, shape, xaxis, yaxis, presRange=None, printUrl=True):
url = 'https://argovis.colorado.edu/selection/bgc_data_selection'
url += '?startDate={}'.format(startDate)
url += '&endDate={}'.format(endDate)
url += '&shape={}'.format(shape)
url += '&xaxis={}'.format(xaxis)
url += '&yaxis={}'.format(yaxis)
if presRange:
pressRangeQuery = '&presRange='.format(presRange)
url += pressRangeQuery
url = url.replace(' ', '')
if printUrl:
print(url)
resp = requests.get(url)
# Consider any status other than 2xx an error
if not resp.status_code // 100 == 2:
return "Error: Unexpected response {}".format(resp)
selectionProfiles = resp.json()
return selectionProfiles
startDate = '2020-03-01'
endDate = '2020-03-11'
presRange = [0, 50]
shape = [[[-155.929898,27.683528],[-156.984448,13.752725],[-149.468316,8.819693],[-142.15318,3.741443],[-134.922845,-1.396838],\
[-127.660888,-6.512815],[-120.250934,-11.523088],[-110.056944,-2.811371],[-107.069051,12.039321],[-118.141833,20.303418],\
[-125.314828,22.509761],[-132.702476,24.389053],[-140.290513,25.90038],[-148.048372,27.007913],[-155.929898,27.683528]]]
xaxis='doxy'
yaxis='pres'
profiles = get_bgc_selection_profiles(startDate, endDate, shape, xaxis, yaxis, presRange, printUrl=True)
https://argovis.colorado.edu/selection/bgc_data_selection?startDate=2020-03-01&endDate=2020-03-11&shape=[[[-155.929898,27.683528],[-156.984448,13.752725],[-149.468316,8.819693],[-142.15318,3.741443],[-134.922845,-1.396838],[-127.660888,-6.512815],[-120.250934,-11.523088],[-110.056944,-2.811371],[-107.069051,12.039321],[-118.141833,20.303418],[-125.314828,22.509761],[-132.702476,24.389053],[-140.290513,25.90038],[-148.048372,27.007913],[-155.929898,27.683528]]]&xaxis=doxy&yaxis=pres&presRange=
df = json2dataframe(profiles, 'bgcMeas')
df.head()
doxy | doxy_qc | pres | pres_qc | _id | POSITIONING_SYSTEM | DATA_MODE | date | lat | lon | cycle_number | bgcMeasKeys | core_data_mode | roundLat | roundLon | strLat | strLon | |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | 203.949081 | 1 | 7.460000 | 8 | 5906046_32 | GPS | D | 2020-03-10T09:12:22.000Z | -5.061 | -118.6 | 32 | [temp, nitrate, pres, psal, ph_in_situ_total, ... | D | -5.061 | -118.600 | 5.061 S | 118.600 W |
1 | 203.940735 | 1 | 11.550000 | 8 | 5906046_32 | GPS | D | 2020-03-10T09:12:22.000Z | -5.061 | -118.6 | 32 | [temp, nitrate, pres, psal, ph_in_situ_total, ... | D | -5.061 | -118.600 | 5.061 S | 118.600 W |
2 | 203.849197 | 1 | 16.520000 | 8 | 5906046_32 | GPS | D | 2020-03-10T09:12:22.000Z | -5.061 | -118.6 | 32 | [temp, nitrate, pres, psal, ph_in_situ_total, ... | D | -5.061 | -118.600 | 5.061 S | 118.600 W |
3 | 203.733582 | 1 | 21.540001 | 8 | 5906046_32 | GPS | D | 2020-03-10T09:12:22.000Z | -5.061 | -118.6 | 32 | [temp, nitrate, pres, psal, ph_in_situ_total, ... | D | -5.061 | -118.600 | 5.061 S | 118.600 W |
4 | 203.752197 | 1 | 26.670000 | 8 | 5906046_32 | GPS | D | 2020-03-10T09:12:22.000Z | -5.061 | -118.6 | 32 | [temp, nitrate, pres, psal, ph_in_situ_total, ... | D | -5.061 | -118.600 | 5.061 S | 118.600 W |