If you are having trouble with Jupyter notebooks run this notebook to help identify where the problem might be.
Select the notebook menu item Cell->Run All
- check for any warnings or errors.
Read the text above the cell(s) that produce errors - the text contains links to resources that describe how to fix the error.
Note You can set the default Python version in Azure Notebooks project settings.
For details on how to do this see AzureNotebooks-ConfigurePythonVersion
If you are using a Data Science Virtual Machine as your Azure Notebooks compute you should read Provisioning a DSVM
import sys
from IPython.display import display, HTML, Markdown
MIN_REQ_PYTHON = (3, 6)
errors = []
warns = []
info = []
def setup_err(mssg):
display(Markdown("<h3><font color='red'>Setup Error</font></h3>"))
display(Markdown("<h4><font color='red'>%s</font></h4>" % mssg))
errors.append(mssg)
def setup_ok(mssg):
display(Markdown("<h4><font color='blue'>%s Ok</font></h4>" % mssg))
info.append(mssg)
def setup_warn(mssg):
display(Markdown("<h4><font color='orange'>%s</font></h4>" % mssg))
warns.append(mssg)
display(Markdown("#### Checking Python version..."))
if sys.version_info < MIN_REQ_PYTHON:
setup_err("Python version")
display(Markdown('Check the Kernel->Change Kernel menu and ensure that Python 3.6'))
display(Markdown('or later is selected as the active kernel.'))
else:
setup_ok(
"Python version {}.{}.{}".format(
sys.version_info[0], sys.version_info[1], sys.version_info[2]
)
)
This section checks the import of msticpy
and its dependent packages.
Note If you are repeatedly seeing packages going missing when working in Azure Notebooks this may be because the docker containers running the Python kernel are recycled after a few hours when not in use. This causes the environments to reset to defaults.
To prevent this you should configure you Azure Notebooks project with a requirements.txt file that is automatically run (and packages installed) when the contain is initialized.
For details on how to do this see AzureNotebooks-ConfigurePythonVersion
import importlib
import sys
import warnings
from IPython.display import display, HTML, Markdown
MSTICPY_REQ_VERSION = (0, 2, 7)
display(Markdown("#### Checking msticpy..."))
warn_mssg = []
err_mssg = []
restart_req = False
MISSING_PKG_ERR = """
<h3><font color='red'>Warning {package} is not installed or has an unsupported version</h3></font>
"""
need_update = False
try:
import msticpy
mp_version = tuple([int(v) for v in msticpy.__version__.split(".")])
if mp_version < MSTICPY_REQ_VERSION:
setup_err("msticpy %s.%s.%s or later is required." % MSTICPY_REQ_VERSION)
need_update = True
except ImportError:
display(HTML(MISSING_PKG_ERR.format(package="msticpy")))
need_update = True
else:
setup_ok(f"msticpy version {msticpy.__version__}")
if need_update:
resp = input("Install the package now? (y/n)")
if resp.casefold().startswith("y"):
!pip install --user --upgrade msticpy
if "msticpy" in sys.modules:
importlib.reload(msticpy)
else:
import msticpy
print(f"msticpy installed - version {msticpy.__version__}")
else:
setup_warn("msticpy missing or out-of-date.")
display(Markdown("Please run `pip install --user --upgrade msticpy` to upgrade/install msticpy"))
Many of the notebooks and msticpy features require a mininum pandas version of 0.25.0.
display(Markdown("#### Checking pandas..."))
PANDAS_REQ_VERSION = (0, 25, 0)
need_update = False
try:
import pandas as pd
pd_version = tuple([int(v) for v in pd.__version__.split(".")])
if pd_version < PANDAS_REQ_VERSION:
setup_err("pandas %s.%s.%s or later is required." % PANDAS_REQ_VERSION)
need_update = True
except ImportError:
display(HTML(MISSING_PKG_ERR.format(package="pandas")))
need_update = True
else:
setup_ok(f"Pandas version {pd.__version__}")
if need_update:
resp = input("Install the package now? (y/n)")
if resp.casefold().startswith("y"):
!pip install --user --upgrade pandas
if "pandas" in sys.modules:
importlib.reload(pd)
else:
import pandas as pd
print(f"pandas installed - version {pandas.__version__}")
else:
setup_warn("pandas missing or out-of-date.")
display(Markdown("Please run `pip install --user --upgrade pandas` to upgrade/install pandas"))
This section checks for presence of configuration files config.json
and msticpyconfig.yaml
The msticpyconfig.yaml
can store the workspace and tenant information
for your Azure Sentinel workspace. It can also store values for multiple
workspaces. If you have the values configured in this file you do not
need to worry about the values in config.json
.
You can specify the location of your msticpyconfig.yaml
in the
environment variable MSTICPYCONFIG
. This will make the file
accessible to all notebooks running on the system. For
more information on configuring msticpyconfig.yaml
see the next
cell mstipcy Configuration
If you want to transfer your workspace settings to msticpyconfig.yaml
from config.json
, simply copy the value of the tenant_id
and
workspace_id
settings to the relevant section.
Note the value names in msticpyconfig.yaml use slightly different naming conventions:
WorkspaceId: 0cd830ff-60dc-40d1-8045-11d2b7b277e1
TenantId: aff2102d-1d6c-4501-9efb-6053ab7efb19
Creating an Azure Notebooks project from Azure Sentinel
will automatically create a config.json
file in the root of
your Azure Notebooks project and populate values
for your Azure Sentinel workspace.
If you have copied the notebooks elsewhere (e.g. to run them locally, or you are running them on a Data Science Virtual machine) you should copy this original config.json to the folder from which you are running notebooks.
Note if you are using a
msticpyconfig.yaml
to store your workspace settings, most notebooks will take values from that. As withconfig.json
- you must have a locally accessible copy of this file, so you will need to copy it to other systems if you are running notebooks from there.
If you are using the config.json (default config for Azure Sentinel with Azure Notebooks), your config.json should look something like this
{
"tenant_id": "aff2102d-1d6c-4501-9efb-6053ab7efb19",
"subscription_id": "9ce7caeb-1f42-4141-b076-7f448a00aceb",
"resource_group": "MyResourceGroup",
"workspace_id": "0cd830ff-60dc-40d1-8045-11d2b7b277e1",
"workspace_name": "MyResourceSubscription"
}
The tenant_id and workspace_id values must be configured, other values are optional but recommended.
import os
import json
from pathlib import Path
import uuid
import yaml
def valid_uuid(uuid_str):
try:
uuid.UUID(uuid_str)
except (ValueError, TypeError):
return False
return True
def check_mp_config_ws(mp_path):
with open(mp_path, "r") as mp_yml:
mp_config = yaml.safe_load(mp_yml)
mp_errors = []
as_settings = mp_config.get("AzureSentinel", {})
if not as_settings:
mp_errors.append(f"Missing or empty 'AzureSentinel' section in {mp_path}")
ws_settings = as_settings.get("Workspaces", {})
if not ws_settings:
mp_errors.append(f"Missing or empty 'Workspaces' section in {mp_path}")
no_default = True
for ws, ws_settings in ws_settings.items():
if ws == "Default":
no_default = False
ws_id = ws_settings.get("WorkspaceId")
if not ws_id and not valid_uuid(ws_id):
mp_errors.append(f"Invalid GUID for WorkspaceId in {ws} section")
ten_id = ws_settings.get("TenantId")
if not ten_id and not valid_uuid(ten_id):
mp_errors.append(f"Invalid GUID for TenantId in {ws} section")
warnings = ["No default workspace set"] if no_default else []
return mp_errors, warnings
def check_json_config(json_path):
j_conf_errs = []
with open(json_path, "r") as json_file:
conf_json = json.load(json_file)
conf_tenant = conf_json.get("tenant_id")
if conf_tenant == "{{cookiecutter.tenant_id}}":
j_conf_errs.append("Tenant Id is set to default value")
elif not valid_uuid(conf_tenant):
j_conf_errs.append("Tenant ID is not a valid GUID.")
conf_ws = conf_json.get("workspace_id")
if conf_ws == "{{cookiecutter.workspace_id}}":
j_conf_errs.append("Workspace Id is set to default value")
elif not valid_uuid(conf_ws):
j_conf_errs.append("Workspace ID is not a valid GUID.")
return j_conf_errs
mp_warnings = []
display(Markdown("#### Checking Azure Sentinel Workspace config..."))
mp_path = os.environ.get("MSTICPYCONFIG", "./msticpyconfig.yaml")
if Path(mp_path).exists():
mp_errs, mp_warnings = check_mp_config_ws(mp_path)
else:
mp_errs = [f"{mp_path} not found"]
DEF_CONF_JSON = "./config.json"
if Path(DEF_CONF_JSON).exists():
jc_errs = check_json_config(DEF_CONF_JSON)
if jc_errs and mp_errs:
setup_err("No valid workspace configuration found.")
if jc_errs:
print(jc_errs)
if mp_errs:
print(mp_errs)
else:
if not jc_errs:
setup_ok(f"Workspace configuration found in '{DEF_CONF_JSON}'")
if not mp_errs:
setup_ok(f"Workspace configuration found in '{mp_path}'")
else:
setup_warn(f"Workspace configuration: Cannot find msticpy config {mp_path}")
if mp_warnings:
display(Markdown(f"<h5><font color='orange'>{', '.join(mp_warnings)}</font></h5>"))
The msticpy configuration file msticpyconfig.yaml
holds setting
required by TI Providers and other data providers such as GeoIP.
These features will not work unless you valid API Keys from your
accounts with these providers stored in msticpyconfig.yaml
.
Note you can store the actual values of the keys in environment variables and use the settings in
msticpyconfig.yaml
to reference these.
For more information on msticpy
configuration file settings,
please refer to the following items:
import os
import json
from pathlib import Path
import uuid
import yaml
def check_mp_config_providers(mp_path):
mp_errors = []
if not Path(mp_path).exists():
mp_errors.append(f"No msticpyconfig.yaml found.")
return mp_errors
with open(mp_path, "r") as mp_yml:
mp_config = yaml.safe_load(mp_yml)
for conf_section in ["TIProviders", "OtherProviders"]:
display(Markdown(f"Checking {conf_section}..."))
settings = mp_config.get(conf_section, {})
mp_errors += check_provider_settings(settings, conf_section)
def check_provider_settings(settings, section):
mp_errors = []
if not settings:
mp_errors.append(f"Missing or empty '{section}' section in {mp_path}")
for p_name, p_setting in settings.items():
print(p_name, end=" ")
ti_args = p_setting.get("Args")
if p_name in ["OTX", "VirusTotal", "XForce", "OpenPageRank"]:
if (
"AuthKey" not in ti_args
or not ti_args["AuthKey"]
):
mp_errors.append(f"{section}: Missing or invalid AuthKey for {p_name} section")
if p_name == "XForce":
if (
"ApiKey" not in ti_args
or not ti_args["ApiKey"]
):
mp_errors.append(f"{section}: Missing or invalid ApiKey for {p_name} section")
if p_name == "AzureSentinel":
ws_id = p_setting.get("WorkspaceID")
if not ws_id or not valid_uuid(ws_id):
mp_errors.append(f"{section}: Invalid GUID for WorkspaceID in {p_name} section")
ten_id = p_setting.get("TenantID")
if not ten_id or not valid_uuid(ten_id):
mp_errors.append(f"{section}: Invalid GUID for TenantID in {p_name} section")
print()
return mp_errors
display(Markdown("#### Checking msticpy config..."))
mp_path = os.environ.get("MSTICPYCONFIG", "./msticpyconfig.yaml")
if not Path(mp_path).exists():
setup_err(f"Cannot find msticpy config {mp_path}")
if "MSTICPYCONFIG" in os.environ:
setup_warn(f"'MSTICPYCONFIG' points to non-existent file")
else:
mp_errs = check_mp_config_providers(mp_path)
if mp_errs:
setup_err("Invalid msticpy configuration found.")
print(mp_errs)
else:
setup_ok(f"msticpy configuration in '{mp_path}'")
if errors:
display(Markdown(f"<h3><font color='red'><u>{len(errors)} errors:</u></font>"))
for mssg in errors:
display(Markdown(f"<font color='red'>{mssg}</font>"))
if warns:
display(Markdown(f"<h3><font color='orange'><u>{len(warns)} warnings:</u></font>"))
for mssg in warns:
display(Markdown(f"<font color='orange'>{mssg}</font>"))
display(Markdown(f"<h3><font color='blue'><u>Info/Success:</u></font>"))
for mssg in info:
display(Markdown(f"<font color='blue'>{mssg}</font>"))