Process SCADA and downtime data¶

# import libraries
import os
import glob
import itertools
import pandas as pd
import numpy as np

# create directory to store processed data
os.makedirs("data/processed/", exist_ok=True)

# read and view data
data = pd.read_excel("data/Melogale Downtime Categories.xlsx")

# drop first row
data = data.drop([0])

# function to filter data for each category type
catData = {}


def categorise_data(cat, number):
    catData[cat] = data.filter(
        items=[cat+" Categories", "Unnamed: "+str(number)]
    )
    catData[cat].rename(
        columns={
            cat+" Categories": "Category", "Unnamed: "+str(number): "Name"
        },
        inplace=True
    )
    catData[cat]["Type"] = cat
    catData[cat].dropna(inplace=True)

# filtering
categorise_data("Turbine", 1)
categorise_data("Environmental", 3)
categorise_data("Grid", 5)
categorise_data("Infrastructure", 7)
categorise_data("Availability", 9)

# concatenate data
data = pd.concat(catData.values(), ignore_index=True)

# drop duplicate columns
dt[1].drop(columns=list(dt[1].filter(regex=".1")), inplace=True)

# convert timestamps to datetime data type
for key in dt.keys():
    for col in list(dt[key].filter(regex="timestamp")):
        dt[key][col] = pd.to_datetime(dt[key][col])

# concatenate data
data = pd.concat(dt.values(), join="outer")

# rename ap_min in scada[1] to ap_max
scada[1].rename(columns={"ap_min": "ap_max"}, inplace=True)

# fixing rotor speed readings due to errors in data
def fix_rs(c):
    if c["turbine"] <= 20:
        return c["rs_av"]

for df in scada.keys():
    scada[df]["rs_av_old"] = scada[df].apply(fix_rs, axis=1)
    scada[df] = scada[df].drop("rs_av", axis=1)

# concatenate old datasets
scadaOld = pd.concat(scada.values())

# concatenate new datasets
scadaNew = pd.concat(scada.values())