STeC Health Visualisation (weekly)¶

import datetime as dt
import pandas as pd
import sys
import plotly.express as px

sys.path.append('../')  # FIXME: Work around if not built as a package
from data_management.data_manager    import DataManager
from data_management.data_processing import DataProcessor

# Settings
TEST_SERVER = 'http://tpbl.permasense.ethz.ch/'

DEPLOYMENT_OUTDOOR = 'dirruhorn'
DEPLOYMENT_TEST    = 'etz'

VS_ACOUSTIC_METADATA = '_dpp_geophone_acq__conv'
VS_ACOUSTIC_AGGR     = '_dpp_geophone_acq_min_aggr__mapped'
VS_ACOUSTIC_DATA     = '_dpp_geophone_adcData__conv'
VS_HEALTH_MIN_DATA   = '_dpp_health_min__conv'

DATA_START_TIME = dt.datetime.today() - dt.timedelta(days=7)
DATA_END_TIME   = dt.datetime.today()

S_TO_US = 1000 * 1000

FALLBACK_BS_IDS_OUTDOOR = [106, 110]
FALLBACK_BS_IDS_TEST    = [103, 107, 109]

# Print settings
print('Going to fetch STeC Health data at deployment site {0:s} and {1:s} from {2:s} - {3:s}'.format(DEPLOYMENT_OUTDOOR.capitalize(), DEPLOYMENT_TEST.capitalize(), DATA_START_TIME.strftime('%d/%m/%Y'), DATA_END_TIME.strftime('%d/%m/%Y')))

# Adjust display width for variables
pd.set_option('display.max_rows',    500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width',       1000)

# Create necessary objects
DataMgr  = DataManager(deployment=DEPLOYMENT_OUTDOOR, config_file='../stec.conf', project_name='stec', start_time=DATA_START_TIME, end_time=DATA_END_TIME)
DataProc = DataProcessor(config_file='../stec.conf', project_name='stec')

# Fetch data

# Create URL including conditions
url_health_outdoor = DataMgr.assemble_gsn_url(VS_HEALTH_MIN_DATA)
url_aggr_outdoor   = DataMgr.assemble_gsn_url(VS_ACOUSTIC_AGGR)
url_health_test    = DataMgr.assemble_gsn_url(VS_HEALTH_MIN_DATA, server_url=TEST_SERVER, deployment=DEPLOYMENT_TEST)
url_aggr_test      = DataMgr.assemble_gsn_url(VS_ACOUSTIC_AGGR,   server_url=TEST_SERVER, deployment=DEPLOYMENT_TEST)

# Fetch data
df_health_outdoor = DataMgr.fetch_csv_data(url_health_outdoor, description="Health",      cache=False)
df_aggr_outdoor   = DataMgr.fetch_csv_data(url_aggr_outdoor,   description="Aggregation", cache=False)
df_health_test    = DataMgr.fetch_csv_data(url_health_test,    description="Health",      abort=False, cache=False)
df_aggr_test      = DataMgr.fetch_csv_data(url_aggr_test,      description="Aggregation", abort=False, cache=False)

# Plotting Outdoor

if df_health_outdoor is not None:
    # Health messages over time
    fig = px.scatter(df_health_outdoor.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Health from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['target_id', 'position'], title='Outdoor Deployment (Dirruhorn)')
    fig.update_yaxes(type='category')
    fig.show()

if df_aggr_outdoor is not None:
    # Aggregated messages over time
    fig = px.scatter(df_aggr_outdoor.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Aggregated Acq from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['block_cnt', 'seqnr', 'target_id'])
    fig.update_yaxes(type='category')
    fig.show()
else:
    print('Skipping plotting aggregation data as no co-detections have been observed in the selected time frame')

if df_health_outdoor is not None:
    # Uptime
    fig = px.line(df_health_outdoor, x='generation_time', y='uptime', labels={'generation_time': 'Time', 'uptime': 'Uptime [s]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
    fig.show()

    # Sequence number
    fig = px.line(df_health_outdoor, x='generation_time', y='seqnr', labels={'generation_time': 'Time', 'seqnr': 'Sequence Number [#]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
    fig.show()

    # Temperature
    fig = px.line(df_health_outdoor, x='generation_time', y='temperature', labels={'generation_time': 'Time', 'temperature': 'Temperature [°C]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
    fig.show()

    # Humidity
    fig = px.line(df_health_outdoor, x='generation_time', y='humidity', labels={'generation_time': 'Time', 'humidity': 'Humidity [%]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
    fig.show()

    # Radio RSSI
    fig = px.line(df_health_outdoor, x='generation_time', y='radio_rssi', labels={'generation_time': 'Time', 'radio_rssi': 'Radio RSSI [dBm]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
    fig.show()

    # Clock drift
    drift_data = DataProc.extract_clock_drift(df_health_outdoor, min_id=21006, reference_id=112)
    # fig = px.scatter(drift_data, x='generation_time', y='clock_drift_local', labels={'generation_time': 'Time', 'clock_drift_local': 'Local clock drift [us]', 'device_id': 'Device ID', 'device_id_str': 'Device ID'}, color='device_id_str', color_discrete_sequence=px.colors.sequential.matter)
    # fig.show()

    fig = px.scatter(drift_data, x='generation_time', y='clock_drift_global', labels={'generation_time': 'Time', 'clock_drift_global': 'Clock drift [us]', 'device_id': 'Device ID', 'device_id_str': 'Device ID'}, color='device_id_str', color_discrete_sequence=px.colors.sequential.matter)
    fig.show()

    # Base station stats
    bs_data = DataProc.extract_bs_data(df_health_outdoor)
    fig = px.line(bs_data, x='generation_time', y='nr_reports', labels={'generation_time': 'Time', 'nr_reports': 'Number of reports received [#]', 'bs_id': 'BaseStation ID'}, color='bs_id', color_discrete_sequence=px.colors.sequential.matter)
    fig.update_yaxes(dtick=1)
    fig.show()

# Plotting Test Deployment

if df_health_test is not None:
    # Health messages over time
    fig = px.scatter(df_health_test.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Health from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['target_id'], title='Test Deployment (ETZ)')
    fig.update_yaxes(type='category')
    fig.show()

if df_aggr_test is not None:
    # Aggregated messages over time
    fig = px.scatter(df_aggr_test.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Aggregated Acq from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['block_cnt', 'seqnr', 'target_id'])
    fig.update_yaxes(type='category')
    fig.show()
else:
    print('Skipping plotting aggregation data as no co-detections have been observed in the selected time frame')

if df_health_test is not None:
    # Uptime
    fig = px.line(df_health_test, x='generation_time', y='uptime', labels={'generation_time': 'Time', 'uptime': 'Uptime [s]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.deep)
    fig.show()

    # Sequence number
    fig = px.line(df_health_test, x='generation_time', y='seqnr', labels={'generation_time': 'Time', 'seqnr': 'Sequence Number [#]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.deep)
    fig.show()

    # Temperature
    fig = px.line(df_health_test, x='generation_time', y='temperature', labels={'generation_time': 'Time', 'temperature': 'Temperature [°C]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.deep)
    fig.show()

    # Humidity
    fig = px.line(df_health_test, x='generation_time', y='humidity', labels={'generation_time': 'Time', 'humidity': 'Humidity [%]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
    fig.show()

    # Radio RSSI
    fig = px.line(df_health_test, x='generation_time', y='radio_rssi', labels={'generation_time': 'Time', 'radio_rssi': 'Radio RSSI [dBm]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.deep)
    fig.show()

    # Clock drift
    drift_data = DataProc.extract_clock_drift(df_health_test, min_id=21006, reference_id=108)
    #fig = px.scatter(drift_data, x='generation_time', y='clock_drift_local', labels={'generation_time': 'Time', 'clock_drift_local': 'Local clock drift [us]', 'device_id': 'Device ID', 'device_id_str': 'Device ID'}, color='device_id_str', color_discrete_sequence=px.colors.sequential.deep)
    #fig.show()

    fig = px.scatter(drift_data, x='generation_time', y='clock_drift_global', labels={'generation_time': 'Time', 'clock_drift_global': 'Clock drift [us]', 'device_id': 'Device ID', 'device_id_str': 'Device ID'}, color='device_id_str', color_discrete_sequence=px.colors.sequential.deep)
    fig.show()

    # Base station stats
    bs_data = DataProc.extract_bs_data(df_health_test)
    fig = px.line(bs_data, x='generation_time', y='nr_reports', labels={'generation_time': 'Time', 'nr_reports': 'Number of reports received [#]', 'bs_id': 'BaseStation ID'}, color='bs_id', color_discrete_sequence=px.colors.sequential.deep)
    fig.update_yaxes(dtick=1)
    fig.show()

if df_health_test is None and df_aggr_test is None:
    print('Skipping test data as not able to access from this machine (no connection to ETH-internal network)')