Who’s responsible?

The National Archives of Australia's RecordSearch database divides government activities up into a series of functions. Over time, different agencies have been made responsible for these functions, and it can be interesting to track how these responsibilities have shifted.

This notebook uses data about functions harvested from RecordSearch to create a a simple visualisation of the agencies responsible for a selected function.

In [ ]:
import ipywidgets as widgets
from IPython.display import display, HTML
import json
import altair as alt
from altair import datum
import pandas as pd
In [ ]:
%%javascript # This will give an error in Lab, just ignore it -- stops notebook from using scrollbars
IPython.OutputArea.prototype._should_scroll = function(lines) {return false;}
In [ ]:
# Load the harvested functions data from a JSON file
with open('data/agencies_by_function.json', 'r') as json_file:
    data = json.load(json_file)
def get_children(function, level):
    Gets the children of the supplied term.
    Formats/indents the terms for the dropdown.
    f_list = []
    if 'narrower' in function:
        level += 1
        for subf in function['narrower']:
            f_list.append(('{}{} {}'.format(level * '  ', level * '-', subf['term']), subf))
            f_list += get_children(subf, level=level)
    return f_list

def get_functions():
    # Load the JSON file of functions we've previously harvested
    with open('data/functions.json', 'r') as json_file:
        functions = json.load(json_file)

    # Make the list of options for the dropdown
    functions_list = []
    for function in functions:
        functions_list.append((function['term'], function))
        functions_list += get_children(function, level=0)
    return functions_list

def get_function_agencies(term):
    for f in data:
        if f['term'] == term:
            return f['agencies']
In [ ]:
def make_chart(change):
    # Clear current output
    # Get the currently selected term from the dropdown
    # term = change['new']
    function = term.value
    atype = agency_type.value
    # Get the agencies responsible for the selected function
    agencies = get_function_agencies(function['term'])
    if agencies:
        # Convert to a dataframe
        df = pd.DataFrame(agencies)
        # Set some defualts for missing dates
        missing = {'agency_status': 'Not recorded', 'function_start_date': '1901-01-01', 'function_end_date': '2018-12-31'}
        df = df.fillna(value=missing)
        df['url'] = df.apply(lambda x: 'http://www.naa.gov.au/cgi-bin/Search?O=S&Number={}'.format(x['agency_id']), axis=1)
        if change['owner'].description == 'Agency type:' and atype != 'All':
            df = df.loc[df['agency_status'] == atype]
            agency_type.value = 'All'
        # Create a Gannt style chart
        chart = alt.Chart(df).mark_bar(size=20).encode(
            x=alt.X('function_start_date:T', axis=alt.Axis(format='%Y', title='Dates agency was responsible for function'), scale=alt.Scale(nice=True)),
            y=alt.Y('title', scale=alt.Scale(), title='Agency'),
            color=alt.Color('agency_status', legend=alt.Legend(title='Agency type')),
                alt.Tooltip('agency_id', title='Identifier'), 
                alt.Tooltip('title', title='Agency'),
                alt.Tooltip('agency_status', title='Type'),
                alt.Tooltip('location', title='Location'),
                alt.Tooltip('function_start_date', title='From', timeUnit='year'), 
                alt.Tooltip('function_end_date', title='To', timeUnit='year')],
        with out:
            display(HTML('<h3>Agencies responsible for &lsquo;{}&rsquo;</h3>'.format(function['term'])))
        with out:
            display(HTML('<p>No agencies responsible for &lsquo;{}&rsquo;</p>'.format(function['term'])))

# This is where the chart will be displayed
out = widgets.Output(layout=widgets.Layout(width='100%'))

# Create the dropdown
term = widgets.Dropdown(

agency_type = widgets.Dropdown(
    options=['All', 'Department of State', 'Head Office', 'Regional or State Office', 'Local Office', 'Intergovernmental agency'],
    description='Agency type:'

# Making a selection from the dropdown will automatically run 'make_chart'
term.observe(make_chart, names='value')
agency_type.observe(make_chart, names='value')

display(widgets.HBox([widgets.Label('Select a function:'), term, agency_type]))

Created by Tim Sherratt as part of the GLAM Workbench.