Copyright (c) 2021, ETH Zurich, Computer Engineering Group (TEC)
This script visualizes the health data of the STeC deployments for the last week both on DH and ETZ (test deployment).
import datetime as dt
import pandas as pd
import sys
import plotly.express as px
sys.path.append('../') # FIXME: Work around if not built as a package
from data_management.data_manager import DataManager
from data_management.data_processing import DataProcessor
# Settings
TEST_SERVER = 'http://tpbl.permasense.ethz.ch/'
DEPLOYMENT_OUTDOOR = 'dirruhorn'
DEPLOYMENT_TEST = 'etz'
VS_ACOUSTIC_METADATA = '_dpp_geophone_acq__conv'
VS_ACOUSTIC_AGGR = '_dpp_geophone_acq_min_aggr__mapped'
VS_ACOUSTIC_DATA = '_dpp_geophone_adcData__conv'
VS_HEALTH_MIN_DATA = '_dpp_health_min__conv'
DATA_START_TIME = dt.datetime.today() - dt.timedelta(days=7)
DATA_END_TIME = dt.datetime.today()
S_TO_US = 1000 * 1000
FALLBACK_BS_IDS_OUTDOOR = [106, 110]
FALLBACK_BS_IDS_TEST = [103, 107, 109]
# Print settings
print('Going to fetch STeC Health data at deployment site {0:s} and {1:s} from {2:s} - {3:s}'.format(DEPLOYMENT_OUTDOOR.capitalize(), DEPLOYMENT_TEST.capitalize(), DATA_START_TIME.strftime('%d/%m/%Y'), DATA_END_TIME.strftime('%d/%m/%Y')))
# Adjust display width for variables
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
# Create necessary objects
DataMgr = DataManager(deployment=DEPLOYMENT_OUTDOOR, config_file='../stec.conf', project_name='stec', start_time=DATA_START_TIME, end_time=DATA_END_TIME)
DataProc = DataProcessor(config_file='../stec.conf', project_name='stec')
# Fetch data
# Create URL including conditions
url_health_outdoor = DataMgr.assemble_gsn_url(VS_HEALTH_MIN_DATA)
url_aggr_outdoor = DataMgr.assemble_gsn_url(VS_ACOUSTIC_AGGR)
url_health_test = DataMgr.assemble_gsn_url(VS_HEALTH_MIN_DATA, server_url=TEST_SERVER, deployment=DEPLOYMENT_TEST)
url_aggr_test = DataMgr.assemble_gsn_url(VS_ACOUSTIC_AGGR, server_url=TEST_SERVER, deployment=DEPLOYMENT_TEST)
# Fetch data
df_health_outdoor = DataMgr.fetch_csv_data(url_health_outdoor, description="Health", cache=False)
df_aggr_outdoor = DataMgr.fetch_csv_data(url_aggr_outdoor, description="Aggregation", cache=False)
df_health_test = DataMgr.fetch_csv_data(url_health_test, description="Health", abort=False, cache=False)
df_aggr_test = DataMgr.fetch_csv_data(url_aggr_test, description="Aggregation", abort=False, cache=False)
# Prepare data for plotting
# Formatting timestamps, add column indicating whether the packet was received over SF7 or SF10
if df_health_outdoor is not None:
df_health_outdoor['generation_time'] = pd.to_datetime(df_health_outdoor['generation_time_microsec'], unit='us')
df_health_outdoor['device_id_str'] = df_health_outdoor['device_id'].astype(str) # Required for correct colouring of scatter plots
df_health_outdoor['fallback'] = df_health_outdoor['target_id'].isin(FALLBACK_BS_IDS_OUTDOOR)
df_health_outdoor['fallback_str'] = df_health_outdoor['fallback'].astype(str) # Required for correct colouring of scatter plots
if df_aggr_outdoor is not None:
df_aggr_outdoor['generation_time'] = pd.to_datetime(df_aggr_outdoor['generation_time_microsec'], unit='us')
df_aggr_outdoor['device_id_str'] = df_aggr_outdoor['device_id'].astype(str) # Required for correct colouring of scatter plots
df_aggr_outdoor['fallback'] = df_aggr_outdoor['target_id'].isin(FALLBACK_BS_IDS_OUTDOOR)
df_aggr_outdoor['fallback_str'] = df_aggr_outdoor['fallback'].astype(str) # Required for correct colouring of scatter plots
if df_health_test is not None:
df_health_test['generation_time'] = pd.to_datetime(df_health_test['generation_time_microsec'], unit='us')
df_health_test['device_id_str'] = df_health_test['device_id'].astype(str) # Required for correct colouring of scatter plots
df_health_test['fallback'] = df_health_test['target_id'].isin(FALLBACK_BS_IDS_TEST)
df_health_test['fallback_str'] = df_health_test['fallback'].astype(str) # Required for correct colouring of scatter plots
if df_aggr_test is not None:
df_aggr_test['generation_time'] = pd.to_datetime(df_aggr_test['generation_time_microsec'], unit='us')
df_aggr_test['device_id_str'] = df_aggr_test['device_id'].astype(str) # Required for correct colouring of scatter plots
df_aggr_test['fallback'] = df_aggr_test['target_id'].isin(FALLBACK_BS_IDS_TEST)
df_aggr_test['fallback_str'] = df_aggr_test['fallback'].astype(str) # Required for correct colouring of scatter plots
# Color mapping for consistent display of fallback data
color_fallback_map = {'True': 'red', 'False': 'green'}
After having pre-processed the data, we now plot the data for visual inspection.
# Plotting Outdoor
if df_health_outdoor is not None:
# Health messages over time
fig = px.scatter(df_health_outdoor.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Health from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['target_id', 'position'], title='Outdoor Deployment (Dirruhorn)')
fig.update_yaxes(type='category')
fig.show()
if df_aggr_outdoor is not None:
# Aggregated messages over time
fig = px.scatter(df_aggr_outdoor.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Aggregated Acq from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['block_cnt', 'seqnr', 'target_id'])
fig.update_yaxes(type='category')
fig.show()
else:
print('Skipping plotting aggregation data as no co-detections have been observed in the selected time frame')
if df_health_outdoor is not None:
# Uptime
fig = px.line(df_health_outdoor, x='generation_time', y='uptime', labels={'generation_time': 'Time', 'uptime': 'Uptime [s]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Sequence number
fig = px.line(df_health_outdoor, x='generation_time', y='seqnr', labels={'generation_time': 'Time', 'seqnr': 'Sequence Number [#]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Temperature
fig = px.line(df_health_outdoor, x='generation_time', y='temperature', labels={'generation_time': 'Time', 'temperature': 'Temperature [°C]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Humidity
fig = px.line(df_health_outdoor, x='generation_time', y='humidity', labels={'generation_time': 'Time', 'humidity': 'Humidity [%]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Radio RSSI
fig = px.line(df_health_outdoor, x='generation_time', y='radio_rssi', labels={'generation_time': 'Time', 'radio_rssi': 'Radio RSSI [dBm]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Clock drift
drift_data = DataProc.extract_clock_drift(df_health_outdoor, min_id=21006, reference_id=112)
# fig = px.scatter(drift_data, x='generation_time', y='clock_drift_local', labels={'generation_time': 'Time', 'clock_drift_local': 'Local clock drift [us]', 'device_id': 'Device ID', 'device_id_str': 'Device ID'}, color='device_id_str', color_discrete_sequence=px.colors.sequential.matter)
# fig.show()
fig = px.scatter(drift_data, x='generation_time', y='clock_drift_global', labels={'generation_time': 'Time', 'clock_drift_global': 'Clock drift [us]', 'device_id': 'Device ID', 'device_id_str': 'Device ID'}, color='device_id_str', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Base station stats
bs_data = DataProc.extract_bs_data(df_health_outdoor)
fig = px.line(bs_data, x='generation_time', y='nr_reports', labels={'generation_time': 'Time', 'nr_reports': 'Number of reports received [#]', 'bs_id': 'BaseStation ID'}, color='bs_id', color_discrete_sequence=px.colors.sequential.matter)
fig.update_yaxes(dtick=1)
fig.show()