In [1]:

import os
import intake
import pandas as pd
import xarray as xr
import colorcet as cc

df = intake.open_csv('./data/bird_migration/{species}.csv').read()

def fill_day(v):
    next_year = v.assign(day=v.day + v.day.max())
    last_year = v.assign(day=v.day - v.day.max())
    surrounding_years = pd.concat([last_year, v, next_year])
    filled = surrounding_years.assign(
        lat=surrounding_years.lat.interpolate(), 
        lon=surrounding_years.lon.interpolate())
    this_year = filled[filled.day.isin(v.day)]
    return this_year

df = pd.concat([fill_day(v) for k, v in df.groupby('species')])

species_cmap = dict(zip(df.species.cat.categories, cc.glasbey))

data_url = 'http://www.esrl.noaa.gov/psd/thredds/dodsC/Datasets/ncep/air.day.ltm.nc'

# I downloaded the file locally because I was hitting rate limits.
local_file = './data/air.day.ltm.nc'
if os.path.isfile(local_file):
    data_url = local_file

ds = xr.open_dataset(data_url)
ds = ds.rename(time='day').sel(level=1000)
ds['day'] = list(range(1,366))

## convert to F
ds = ds.assign(air_F = (ds['air'] - 273.15) * 9/5 + 32)

/Users/jsignell/conda/envs/birds/lib/python3.7/site-packages/xarray/coding/times.py:419: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range
  self.use_cftime)
/Users/jsignell/conda/envs/birds/lib/python3.7/site-packages/numpy/core/numeric.py:538: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range
  return array(a, dtype, copy=False, order=order)

Panel¶

Panel provides the framework and interactivity to make dashboards that work in the notebook and can be deployed as standalone apps. Just to remind ourselves - we have our target dashboard.

In [2]:

import numpy as np

import hvplot.pandas
import hvplot.xarray

import colorcet as cc
import holoviews as hv
from holoviews import opts
import geoviews as gv
import geoviews.tile_sources as gts
import cartopy.crs as ccrs

from holoviews.streams import Selection1D, Params
import panel as pn

hv.extension('bokeh', width=90)
pn.extension()

Make some widgets¶

One of the things that panel provides is an easy way to instantiate widgets that work both inside and outside the notebook. We'll set up a Player widget, a MultiSelect widget, and a Toggle.

In [3]:

species = pn.widgets.MultiSelect(options=df.species.cat.categories.tolist(), size=10)
species

Out[3]:

Select an item from the list above or cmd/ctrl + click to select multiple species

In [4]:

species.value

Out[4]:

[]

In [5]:

day = pn.widgets.Player(value=1, start=1, end=365, loop_policy='loop', name='day', step=5)
day

Out[5]:

In [6]:

day.value

Out[6]:

In [7]:

toggle = pn.widgets.Toggle(name='Air Temperature Layer', value=True)
toggle

Out[7]:

In [8]:

toggle.value

Out[8]:

True

Now we can capture the streams from these widgets so we can use them in our dynamic maps.

In [9]:

species_stream = Params(species, ['value'], rename={'value': 'species'})
day_stream = Params(day, ['value'], rename={'value': 'day'})
toggle_stream = Params(toggle, ['value'])

In [10]:

def sanity_checker(species):
    return hv.Text(0.5, 0.5, '\n'.join(species))

hv.util.DynamicMap(sanity_checker, streams=[species_stream])

Out[10]:

In [11]:

species

Out[11]:

We'll make a reset button to reset all these widgets.

In [12]:

def reset(arg=None):
    day_stream.update(value=1)
    species_stream.update(value=[])
    toggle_stream.update(value=True)
    
reset_button = pn.widgets.Button(name='Reset')
reset_button.param.watch(reset, 'clicks')
reset_button

Out[12]:

NOTE: Click the button and then go back and look at the "sanity_checker"

Constructing pieces¶

We'll start by setting up a map of bird locations grouped by day of year. This plot is the same as we've set up in prior notebooks.

In [13]:

birds = df.hvplot.points('lon', 'lat', color='species', groupby='day', geo=True,
                         cmap=species_cmap, legend=False, size=100, 
                         tools=['tap', 'hover', 'box_select'], width=500, height=600)

In [14]:

tiles = gts.EsriImagery()
tiles.extents = df.lon.min(), df.lat.min(), df.lon.max(), df.lat.max()

In [15]:

birds * tiles

Out[15]:

Now we can take the player widget that we've defined above and use that one instead of the slider.

In [16]:

bird_dmap = birds.clone(streams=[day_stream])
row = pn.Row(bird_dmap * tiles, day)
row

Out[16]:

That doesn't look quite how we want it. So we can inspect the structure of the panel object.

In [17]:

print(row)

Row
    [0] Row
        [0] HoloViews(DynamicMap)
        [1] Column
            [0] DiscreteSlider(formatter='%d', name='day', options=OrderedDict([('1', ...]), value=1)
    [1] Player(end=365, loop_policy='loop', name='day', start=1, step=5, value=1)

Each of the items has an index, so we can access the components individually. Note that the components are still linked (try wiggling the slider).

In [18]:

row[0][1]

Out[18]:

In [19]:

pn.Row(row[0][0], row[1])

Out[19]:

Adding another layer¶

Let's add the temperature data now. To speed things up we can subset the air temperature layer to the region of interest, and then persist that in memory.

In [20]:

extents = df.lon.min(), df.lon.max(), df.lat.min(), df.lat.max()
extents

Out[20]:

(-151.845358820379, -52.496901215892, -50.8305029978382, 72.1362543082153)

In [21]:

360+extents[0], 360+extents[1]

Out[21]:

(208.154641179621, 307.503098784108)

One tricky thing is figuring out the right order for the slices. We'll do it by inspection, but there is probably a more clever way.

In [22]:

ROI = ds.sel(lon=slice(205, 310), lat=slice(75, -55)).persist()
ROI

Out[22]:

<xarray.Dataset>
Dimensions:  (day: 365, lat: 53, lon: 43)
Coordinates:
  * lon      (lon) float32 205.0 207.5 210.0 212.5 ... 302.5 305.0 307.5 310.0
  * lat      (lat) float32 75.0 72.5 70.0 67.5 65.0 ... -47.5 -50.0 -52.5 -55.0
    level    float32 1000.0
  * day      (day) int64 1 2 3 4 5 6 7 8 9 ... 358 359 360 361 362 363 364 365
Data variables:
    air      (day, lat, lon) float32 255.02 255.06999 ... 278.99 278.33002
    air_F    (day, lat, lon) float32 -0.6339798 -0.54400253 ... 41.324043
Attributes:
    title:          Once daily NCEP temperature ltm
    delta_time:     once daily
    supplier:       NCEP
    producer:       NCEP
    history:        created 12/21/95 by C. Smith (netCDF2.3)
    description:    Data is from NCEP initialized analysis\n(2x/day).  It con...
    platform:       Model
    Conventions:    CF-1.2
    References:     https://www.esrl.noaa.gov/psd/data/gridded/data.ncep.html
    dataset_title:  NCEP Global Data Assimilation System GDAS

In [23]:

p = ROI.hvplot.quadmesh('lon', 'lat', 'air_F', groupby='day', geo=True, height=600, width=500)
p

Out[23]:

You'll notice that as you slide around the days, the colorbar hops around to accommodate the range of cell values. We can fix that by taking a look at the min and max and using those values to set the allowable range for air temperature.

In [24]:

ds.air_F.min().item(), ds.air_F.max().item()

Out[24]:

(-32.40399932861328, 114.80000305175781)

So we'll do some plot tweaking and then clone the air plot to accept the day_stream as we did above for birds:

In [25]:

grouped_air = p.redim.range(air_F=(-20, 100))
air_dmap = grouped_air.clone(streams=[day_stream])

The last step for the temperature layer is to add in the toggle. For that we'll create a function that accept the dynamic map and the active stream from the toggle. Then we'll create a new dynamic map that wraps our air_dmap.

In [26]:

def toggle_temp(layer, value=True):
    return layer.options(fill_alpha=int(value))

temp_layer = hv.util.Dynamic(air_dmap, operation=toggle_temp, streams=[toggle_stream])

row = pn.Row(
    pn.Column(toggle, day, width=450), 
    pn.Row(tiles * temp_layer * gv.feature.coastline * bird_dmap)[0][0])
row

Out[26]:

In [27]:

print(row)

Row
    [0] Column(width=450)
        [0] Toggle(name='Air Temperature Layer', value=True)
        [1] Player(end=365, loop_policy='loop', name='day', start=1, step=5, value=1)
    [1] HoloViews(DynamicMap)

It's kind of hard to see those birds so let's set the line color to white and add another toggle to turn that on and off.

In [28]:

highlight = pn.widgets.Toggle(name='Highlight Birds', value=False)
highlight_stream = Params(highlight, ['value'])

def do_highlight(points, value=True):
    return points.opts(opts.Points(line_alpha=(0.5 if value else 0), 
                                   selection_line_alpha=value))

bird_dmap = hv.util.Dynamic(bird_dmap.opts(line_color='white'), 
                            operation=do_highlight, 
                            streams=[highlight_stream])

In [29]:

pn.Row(highlight, bird_dmap * tiles)

Out[29]:

Doing a little more computation¶

One of the things that we'd like to display on our dashboard is bird speed. We can calculate the speed for each bird for each day using pyproj.

In [30]:

import pyproj
import numpy as np

g = pyproj.Geod(ellps='WGS84')

def calculate_speed(v):
    today_lat = v['lat'].values
    today_lon = v['lon'].values
    tomorrow_lat = np.append(v['lat'][1:].values, v['lat'][0])
    tomorrow_lon = np.append(v['lon'][1:].values, v['lon'][0])
    _, _, dist = g.inv(today_lon, today_lat, tomorrow_lon, tomorrow_lat)
    return v.assign(speed=dist/1000.)

df = pd.concat([calculate_speed(v) for k, v in df.groupby('species')])
df.head()

Out[30]:

	day	lon	lat	species	speed
0	1	-68.585528	-38.489467	Baird_s_Sandpiper	5.812429
1	2	-68.566917	-38.439191	Baird_s_Sandpiper	6.663629
2	3	-68.548347	-38.380966	Baird_s_Sandpiper	7.527409
3	4	-68.529948	-38.314720	Baird_s_Sandpiper	8.403018
4	5	-68.511848	-38.240375	Baird_s_Sandpiper	9.290238

Defining functions¶

A more common way to link streams with plots is using a function that accepts some values from streams as input and returns some holoviews object. Here we will define a function that plots lat vs day for all species or a select list of species depending on the input.

In [31]:

def timeseries(species=None, y='lat'):
    data = df[df.species.isin(species)] if species else df
    plots = [
        (data.groupby(['day', 'species'], observed=True)[y]
            .mean()
            .groupby('day').agg([np.min, np.max])
            .hvplot.area('day', 'amin', 'amax', alpha=0.2, fields={'amin': y}))]
    if not species or len(species) > 7:
        plots.append(data.groupby('day')[y].mean().hvplot().relabel('mean'))
    else:
        gb = df.groupby('species', observed=True)
        
        plots.extend([v.hvplot('day', y, color=species_cmap[k]).relabel(k) for k, v in gb if k in species])
    return hv.Overlay(plots).opts(width=900, height=250, toolbar='below', 
                                  legend_position='right', legend_offset=(20, 0), label_width=150)

In [32]:

timeseries()

Out[32]:

In [33]:

timeseries(['American_Redstart', 'Chimney_Swift'])

Out[33]:

Now that we have a sense of how the function works, we can set up a holoviews.DynamicMap with species and day as the streams and our function as the callable.

In [34]:

ts_lat = hv.DynamicMap(lambda species: timeseries(species, 'lat'), streams=[species_stream])
ts_speed = hv.DynamicMap(lambda species: timeseries(species, 'speed'), streams=[species_stream])

col = pn.Column('**Species**', species, ts_speed, ts_lat)
col

Out[34]:

In [35]:

species

Out[35]:

In [36]:

print(col)

Column
    [0] Markdown(str)
    [1] MultiSelect(options=['Baird_s_Sandpiper', ...], size=10)
    [2] HoloViews(DynamicMap)
    [3] HoloViews(DynamicMap)

Setting up the Table¶

We can get the air temp for a particular bird by selecting the nearest grid cell.

In [37]:

def temp_calc(ds, row):
    lat_lon_day = row[['lat', 'lon', 'day']]
    return round(ds.sel(**lat_lon_day, method='nearest')['air_F'].item())

In [38]:

temp_calc(ds, df.iloc[100])

Out[38]:

Now we can set up a holoviews.Table element to report back the temperature. This function has a similar structure to the one above for calculating timeseries.

In [39]:

def daily_table(species=None, day=None):
    if not species or not day:
        return hv.Table(pd.DataFrame(columns=['Species', 'Air [F]', 'Speed [km/day]'])).relabel('No species selected')
    
    subset = df[df.species.isin(species)]
    subset = subset[subset.day==day]
    temps = [temp_calc(ds, row) for _, row in subset.iterrows()]
    
    return hv.Table(pd.DataFrame({'Species': species, 'Air [F]': temps, 'Speed [km/day]': subset['speed']})).relabel('day: {}'.format(day))

In [40]:

daily_table().opts(opts.Table(height=100))

Out[40]:

In [41]:

daily_table(['Veery'],5).opts(opts.Table(height=100))

Out[41]:

In [42]:

table = hv.DynamicMap(daily_table, streams=[species_stream, day_stream])

Hook up the map selector to the MultiSelect¶

The goal is to make it so that if you change the species selected on the map, then it will update species_stream. This will trigger the all the DynamicMaps that depend on species_stream to change as well.

In [43]:

def on_map_select(index):
    if index:
        species = df.species.cat.categories[index].tolist()
        if set(species_stream.contents['species']) != set(species):
            species_stream.update(value=species)
        
map_selected_stream = Selection1D(source=bird_dmap)
map_selected_stream.param.watch_values(on_map_select, ['index']);

Putting the pieces together¶

In [55]:

dashboard = pn.Column(
    pn.Row('### Bird Migration Dashboard'),
    pn.Row(
        pn.Column(
            pn.Row(
                pn.Row(tiles * temp_layer * gv.feature.coastline * bird_dmap)[0][0], 
                pn.Spacer(width=30),
                pn.Column(
                    '**Day of Year**', day, 
                    '**Species**:',
                    'This selector does not affect the map. Use plot selectors.', species, 
                    toggle, highlight,
                    'This reset button only resets widgets - otherwise use the plot reset 🔄',
                    reset_button
                ),
            ),
            pn.Row(pn.layout.Tabs(('Latitude', ts_lat), ('Speed', ts_speed))),
        ),
        pn.Column(table.opts(opts.Table(width=300, height=850)))
    )
)

In [56]:

dashboard.servable()

Out[56]:

In [46]:

print(dashboard)

Column
    [0] Row
        [0] Markdown(str)
    [1] Row
        [0] Column
            [0] Row
                [0] HoloViews(DynamicMap)
                [1] Spacer(width=30)
                [2] Column
                    [0] Markdown(str)
                    [1] Player(end=365, loop_policy='loop', name='day', start=1, step=5, value=1)
                    [2] Markdown(str)
                    [3] Markdown(str)
                    [4] MultiSelect(options=['Baird_s_Sandpiper', ...], size=10)
                    [5] Toggle(name='Air Temperature Layer', value=True)
                    [6] Toggle(name='Highlight Birds')
                    [7] Markdown(str)
                    [8] Button(name='Reset')
            [1] Row
                [0] Tabs
                    [0] HoloViews(DynamicMap, name='Latitude')
                    [1] HoloViews(DynamicMap, name='Speed')
        [1] Column
            [0] HoloViews(DynamicMap)

Deploy this dashboard from the CLI using:

$ panel serve 04_panel.ipynb