Copyright (c) 2021, ETH Zurich, Computer Engineering Group (TEC)

STeC Health Visualisation (weekly)

This script visualizes the health data of the STeC deployments for the last week both on DH and ETZ (test deployment).

In [1]:
import datetime as dt
import pandas as pd
import sys
import plotly.express as px

sys.path.append('../')  # FIXME: Work around if not built as a package
from data_management.data_manager    import DataManager
from data_management.data_processing import DataProcessor

# Settings
TEST_SERVER = 'http://tpbl.permasense.ethz.ch/'

DEPLOYMENT_OUTDOOR = 'dirruhorn'
DEPLOYMENT_TEST    = 'etz'

VS_ACOUSTIC_METADATA = '_dpp_geophone_acq__conv'
VS_ACOUSTIC_AGGR     = '_dpp_geophone_acq_min_aggr__mapped'
VS_ACOUSTIC_DATA     = '_dpp_geophone_adcData__conv'
VS_HEALTH_MIN_DATA   = '_dpp_health_min__conv'

DATA_START_TIME = dt.datetime.today() - dt.timedelta(days=7)
DATA_END_TIME   = dt.datetime.today()

S_TO_US = 1000 * 1000

FALLBACK_BS_IDS_OUTDOOR = [106, 110]
FALLBACK_BS_IDS_TEST    = [103, 107, 109]

# Print settings
print('Going to fetch STeC Health data at deployment site {0:s} and {1:s} from {2:s} - {3:s}'.format(DEPLOYMENT_OUTDOOR.capitalize(), DEPLOYMENT_TEST.capitalize(), DATA_START_TIME.strftime('%d/%m/%Y'), DATA_END_TIME.strftime('%d/%m/%Y')))

# Adjust display width for variables
pd.set_option('display.max_rows',    500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width',       1000)
Going to fetch STeC Health data at deployment site Dirruhorn and Etz from 04/11/2021 - 11/11/2021
In [2]:
# Create necessary objects
DataMgr  = DataManager(deployment=DEPLOYMENT_OUTDOOR, config_file='../stec.conf', project_name='stec', start_time=DATA_START_TIME, end_time=DATA_END_TIME)
DataProc = DataProcessor(config_file='../stec.conf', project_name='stec')

# Fetch data

# Create URL including conditions
url_health_outdoor = DataMgr.assemble_gsn_url(VS_HEALTH_MIN_DATA)
url_aggr_outdoor   = DataMgr.assemble_gsn_url(VS_ACOUSTIC_AGGR)
url_health_test    = DataMgr.assemble_gsn_url(VS_HEALTH_MIN_DATA, server_url=TEST_SERVER, deployment=DEPLOYMENT_TEST)
url_aggr_test      = DataMgr.assemble_gsn_url(VS_ACOUSTIC_AGGR,   server_url=TEST_SERVER, deployment=DEPLOYMENT_TEST)

# Fetch data
df_health_outdoor = DataMgr.fetch_csv_data(url_health_outdoor, description="Health",      cache=False)
df_aggr_outdoor   = DataMgr.fetch_csv_data(url_aggr_outdoor,   description="Aggregation", cache=False)
df_health_test    = DataMgr.fetch_csv_data(url_health_test,    description="Health",      abort=False, cache=False)
df_aggr_test      = DataMgr.fetch_csv_data(url_aggr_test,      description="Aggregation", abort=False, cache=False)
17:40:05 - DataManager - INFO - No Aggregation data found for CSV from path:
http://data.permasense.ch/multidata?field[0]=All&vs[0]=dirruhorn_dpp_geophone_acq_min_aggr__mapped&time_format=iso&timeline=generation_time&from=04/11/2021+17:40:05&to=11/11/2021+17:40:05
17:40:05 - DataManager - INFO - No Health data found for CSV from path:
http://tpbl.permasense.ethz.ch//multidata?field[0]=All&vs[0]=etz_dpp_health_min__conv&time_format=iso&timeline=generation_time&from=04/11/2021+17:40:05&to=11/11/2021+17:40:05
17:40:05 - DataManager - INFO - No Aggregation data found for CSV from path:
http://tpbl.permasense.ethz.ch//multidata?field[0]=All&vs[0]=etz_dpp_geophone_acq_min_aggr__mapped&time_format=iso&timeline=generation_time&from=04/11/2021+17:40:05&to=11/11/2021+17:40:05
In [3]:
# Prepare data for plotting

# Formatting timestamps, add column indicating whether the packet was received over SF7 or SF10
if df_health_outdoor is not None:
    df_health_outdoor['generation_time'] = pd.to_datetime(df_health_outdoor['generation_time_microsec'], unit='us')
    df_health_outdoor['device_id_str']   = df_health_outdoor['device_id'].astype(str)  # Required for correct colouring of scatter plots
    df_health_outdoor['fallback']        = df_health_outdoor['target_id'].isin(FALLBACK_BS_IDS_OUTDOOR)
    df_health_outdoor['fallback_str']    = df_health_outdoor['fallback'].astype(str)  # Required for correct colouring of scatter plots

if df_aggr_outdoor is not None:
    df_aggr_outdoor['generation_time'] = pd.to_datetime(df_aggr_outdoor['generation_time_microsec'], unit='us')
    df_aggr_outdoor['device_id_str']   = df_aggr_outdoor['device_id'].astype(str)  # Required for correct colouring of scatter plots
    df_aggr_outdoor['fallback']        = df_aggr_outdoor['target_id'].isin(FALLBACK_BS_IDS_OUTDOOR)
    df_aggr_outdoor['fallback_str']    = df_aggr_outdoor['fallback'].astype(str)  # Required for correct colouring of scatter plots

if df_health_test is not None:
    df_health_test['generation_time'] = pd.to_datetime(df_health_test['generation_time_microsec'], unit='us')
    df_health_test['device_id_str']   = df_health_test['device_id'].astype(str)  # Required for correct colouring of scatter plots
    df_health_test['fallback']        = df_health_test['target_id'].isin(FALLBACK_BS_IDS_TEST)
    df_health_test['fallback_str']    = df_health_test['fallback'].astype(str)  # Required for correct colouring of scatter plots

if df_aggr_test is not None:
    df_aggr_test['generation_time'] = pd.to_datetime(df_aggr_test['generation_time_microsec'], unit='us')
    df_aggr_test['device_id_str']   = df_aggr_test['device_id'].astype(str)  # Required for correct colouring of scatter plots
    df_aggr_test['fallback']        = df_aggr_test['target_id'].isin(FALLBACK_BS_IDS_TEST)
    df_aggr_test['fallback_str']    = df_aggr_test['fallback'].astype(str)  # Required for correct colouring of scatter plots

# Color mapping for consistent display of fallback data
color_fallback_map = {'True': 'red', 'False': 'green'}

Plotting

After having pre-processed the data, we now plot the data for visual inspection.

In [4]:
# Plotting Outdoor

if df_health_outdoor is not None:
    # Health messages over time
    fig = px.scatter(df_health_outdoor.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Health from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['target_id', 'position'], title='Outdoor Deployment (Dirruhorn)')
    fig.update_yaxes(type='category')
    fig.show()

if df_aggr_outdoor is not None:
    # Aggregated messages over time
    fig = px.scatter(df_aggr_outdoor.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Aggregated Acq from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['block_cnt', 'seqnr', 'target_id'])
    fig.update_yaxes(type='category')
    fig.show()
else:
    print('Skipping plotting aggregation data as no co-detections have been observed in the selected time frame')

if df_health_outdoor is not None:
    # Uptime
    fig = px.line(df_health_outdoor, x='generation_time', y='uptime', labels={'generation_time': 'Time', 'uptime': 'Uptime [s]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
    fig.show()

    # Sequence number
    fig = px.line(df_health_outdoor, x='generation_time', y='seqnr', labels={'generation_time': 'Time', 'seqnr': 'Sequence Number [#]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
    fig.show()

    # Temperature
    fig = px.line(df_health_outdoor, x='generation_time', y='temperature', labels={'generation_time': 'Time', 'temperature': 'Temperature [°C]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
    fig.show()

    # Humidity
    fig = px.line(df_health_outdoor, x='generation_time', y='humidity', labels={'generation_time': 'Time', 'humidity': 'Humidity [%]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
    fig.show()

    # Radio RSSI
    fig = px.line(df_health_outdoor, x='generation_time', y='radio_rssi', labels={'generation_time': 'Time', 'radio_rssi': 'Radio RSSI [dBm]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
    fig.show()

    # Clock drift
    drift_data = DataProc.extract_clock_drift(df_health_outdoor, min_id=21006, reference_id=112)
    # fig = px.scatter(drift_data, x='generation_time', y='clock_drift_local', labels={'generation_time': 'Time', 'clock_drift_local': 'Local clock drift [us]', 'device_id': 'Device ID', 'device_id_str': 'Device ID'}, color='device_id_str', color_discrete_sequence=px.colors.sequential.matter)
    # fig.show()

    fig = px.scatter(drift_data, x='generation_time', y='clock_drift_global', labels={'generation_time': 'Time', 'clock_drift_global': 'Clock drift [us]', 'device_id': 'Device ID', 'device_id_str': 'Device ID'}, color='device_id_str', color_discrete_sequence=px.colors.sequential.matter)
    fig.show()

    # Base station stats
    bs_data = DataProc.extract_bs_data(df_health_outdoor)
    fig = px.line(bs_data, x='generation_time', y='nr_reports', labels={'generation_time': 'Time', 'nr_reports': 'Number of reports received [#]', 'bs_id': 'BaseStation ID'}, color='bs_id', color_discrete_sequence=px.colors.sequential.matter)
    fig.update_yaxes(dtick=1)
    fig.show()