Handy way to process the run unit tests (via doctest) and integration tests (via behave) in jupyter notebooks (.ipynb) containing Python functions. The script will convert an .ipynb to a string format (basically a .py file), loads them as modules, and runs the tests on them. To run it in the console, do:
python -m pytest --verbose --disable-warnings --nbval test_ipynb.ipynb
The script should tell you which ipynb file's doctests has failed (e.g. srgan_train.ipynb). You can then open up this very jupyter notebook to debug and inspect the situation further.
from features.environment import _load_ipynb_modules
import behave.__main__
import doctest
import os
import sys
def _unit_test_ipynb(path: str):
"""
Unit tests on loaded modules from a .ipynb file.
Uses doctest.
"""
assert path.endswith(".ipynb")
module = _load_ipynb_modules(ipynb_path=path)
num_failures, num_attempted = doctest.testmod(m=module, verbose=True)
if num_failures > 0:
sys.exit(num_failures)
def _integration_test_ipynb(path: str, summary: bool = False):
"""
Integration tests on various feature behaviours inside a .feature file.
Uses behave.
"""
assert os.path.exists(path=path)
assert path.endswith(".feature")
if summary == False:
args = f"--tags ~@skip --no-summary {path}"
elif summary == True:
args = f"--tags ~@skip {path}"
num_failures = behave.__main__.main(args=args)
if num_failures > 0:
sys.exit(num_failures)
_unit_test_ipynb(path="data_prep.ipynb")
Trying:
os.makedirs(name="/tmp/highres", exist_ok=True)
Expecting nothing
ok
Trying:
download_to_path(path="/tmp/highres/2011_Antarctica_TO.csv",
url="https://data.cresis.ku.edu/data/rds/2011_Antarctica_TO/csv_good/2011_Antarctica_TO.csv")
Expecting:
<Response [200]>
ok
Trying:
_ = shutil.copy(src="highres/20xx_Antarctica_TO.json", dst="/tmp/highres")
Expecting nothing
ok
Trying:
df = ascii_to_xyz(pipeline_file="/tmp/highres/20xx_Antarctica_TO.json")
Expecting nothing
ok
Trying:
df.head(2)
Expecting:
x y z
0 345580.826265 -1.156471e+06 -377.2340
1 345593.322948 -1.156460e+06 -376.6332
ok
Trying:
shutil.rmtree(path="/tmp/highres")
Expecting nothing
ok
Trying:
download_to_path(path="highres/Data_20171204_02.csv",
url="https://data.cresis.ku.edu/data/rds/2017_Antarctica_Basler/csv_good/Data_20171204_02.csv")
Expecting:
<Response [200]>
ok
Trying:
check_sha256("highres/Data_20171204_02.csv")
Expecting:
'53cef7a0d28ff92b30367514f27e888efbc32b1bda929981b371d2e00d4c671b'
ok
Trying:
os.remove(path="highres/Data_20171204_02.csv")
Expecting nothing
ok
Trying:
download_to_path(path="highres/Data_20171204_02.csv",
url="https://data.cresis.ku.edu/data/rds/2017_Antarctica_Basler/csv_good/Data_20171204_02.csv")
Expecting:
<Response [200]>
ok
Trying:
open("highres/Data_20171204_02.csv").readlines()
Expecting:
['LAT,LON,UTCTIMESOD,THICK,ELEVATION,FRAME,SURFACE,BOTTOM,QUALITY\n']
ok
Trying:
os.remove(path="highres/Data_20171204_02.csv")
Expecting nothing
ok
Trying:
xyz_data = pd.DataFrame(np.random.RandomState(seed=42).rand(30).reshape(10, 3))
Expecting nothing
ok
Trying:
get_region(xyz_data=xyz_data)
Expecting:
'0.05808/0.83244/0.02058/0.95071'
ok
Trying:
xr.DataArray(
data=np.zeros(shape=(36, 32)),
coords={"x": np.arange(1, 37), "y": np.arange(1, 33)},
dims=["x", "y"],
).to_netcdf(path="/tmp/tmp_wb.nc")
Expecting nothing
ok
Trying:
get_window_bounds(filepath="/tmp/tmp_wb.nc")
Expecting:
Tiling: /tmp/tmp_wb.nc ... 2
[(0.5, 4.5, 32.5, 36.5), (0.5, 0.5, 32.5, 32.5)]
ok
Trying:
os.remove("/tmp/tmp_wb.nc")
Expecting nothing
ok
Trying:
xr.DataArray(
data=np.random.RandomState(seed=42).rand(64).reshape(8, 8),
coords={"x": np.arange(8), "y": np.arange(8)},
dims=["x", "y"],
).to_netcdf(path="/tmp/tmp_st.nc", mode="w")
Expecting nothing
ok
Trying:
selective_tile(
filepath="/tmp/tmp_st.nc",
window_bounds=[(1.0, 4.0, 3.0, 6.0), (2.0, 5.0, 4.0, 7.0)],
)
Expecting:
Tiling: /tmp/tmp_st.nc
array([[[[0.18485446, 0.96958464],
[0.4951769 , 0.03438852]]],
<BLANKLINE>
<BLANKLINE>
[[[0.04522729, 0.32533032],
[0.96958464, 0.77513283]]]], dtype=float32)
ok
Trying:
os.remove("/tmp/tmp_st.nc")
Expecting nothing
ok
Trying:
xyz_data = 1000*pd.DataFrame(np.random.RandomState(seed=42).rand(60).reshape(20, 3))
Expecting nothing
ok
Trying:
region = get_region(xyz_data=xyz_data)
Expecting nothing
ok
Trying:
grid = xyz_to_grid(xyz_data=xyz_data, region=region, spacing=250)
Expecting nothing
ok
Trying:
grid.to_array().shape
Expecting:
(1, 5, 5)
ok
Trying:
grid.to_array().values
Expecting:
array([[[403.17618 , 544.92535 , 670.7824 , 980.75055 , 961.47723 ],
[379.0757 , 459.26407 , 314.38297 , 377.78555 , 546.0469 ],
[450.67664 , 343.26 , 88.391594, 260.10492 , 452.3337 ],
[586.09906 , 469.74008 , 216.8168 , 486.9802 , 642.2116 ],
[451.4794 , 652.7244 , 325.77896 , 879.8973 , 916.7921 ]]],
dtype=float32)
ok
2 items had no tests:
data_prep
data_prep.parse_datalist
7 items passed all tests:
6 tests in data_prep.ascii_to_xyz
3 tests in data_prep.check_sha256
3 tests in data_prep.download_to_path
2 tests in data_prep.get_region
3 tests in data_prep.get_window_bounds
3 tests in data_prep.selective_tile
5 tests in data_prep.xyz_to_grid
25 tests in 9 items.
25 passed and 0 failed.
Test passed.
_unit_test_ipynb(path="srgan_train.ipynb")
Trying:
discriminator_model = DiscriminatorModel()
Expecting nothing
ok
Trying:
y_pred = discriminator_model.forward(
x=np.random.rand(2, 1, 32, 32).astype("float32")
)
Expecting nothing
ok
Trying:
y_pred.shape
Expecting:
(2, 1)
ok
Trying:
discriminator_model.count_params()
Expecting:
10205129
ok
Trying:
generator_model = GeneratorModel()
Expecting nothing
ok
Trying:
y_pred = generator_model.forward(
x=np.random.rand(1, 1, 10, 10).astype("float32"),
w1=np.random.rand(1, 1, 100, 100).astype("float32"),
w2=np.random.rand(1, 1, 20, 20).astype("float32"),
)
Expecting nothing
ok
Trying:
y_pred.shape
Expecting:
(1, 1, 32, 32)
ok
Trying:
generator_model.count_params()
Expecting:
9088641
ok
Trying:
calculate_discriminator_loss(
real_labels_pred=chainer.variable.Variable(data=np.array([[1.1], [-0.5]])),
fake_labels_pred=chainer.variable.Variable(data=np.array([[-0.3], [1.0]])),
real_minus_fake_target=np.array([[1], [1]]),
fake_minus_real_target=np.array([[0], [0]]),
)
Expecting:
variable(1.56670504)
ok
Trying:
calculate_generator_loss(
y_pred=chainer.variable.Variable(data=np.ones(shape=(2, 1, 3, 3))),
y_true=np.full(shape=(2, 1, 3, 3), fill_value=10.0),
fake_labels=np.array([[-1.2], [0.5]]),
real_labels=np.array([[0.5], [-0.8]]),
fake_minus_real_target=np.array([[1], [1]]).astype(np.int32),
real_minus_fake_target=np.array([[0], [0]]).astype(np.int32),
)
Expecting:
variable(0.09867307)
ok
Trying:
psnr(
y_true=np.ones(shape=(2, 1, 3, 3)),
y_pred=np.full(shape=(2, 1, 3, 3), fill_value=2),
)
Expecting:
192.65919722494797
ok
Trying:
model = GeneratorModel()
Expecting nothing
ok
Trying:
_, _ = save_model_weights_and_architecture(
trained_model=model, save_path="/tmp/weights"
)
Expecting nothing
ok
Trying:
os.path.exists(path="/tmp/weights/srgan_generator_model_architecture.onnx.txt")
Expecting:
True
ok
Trying:
train_arrays = {
"X": np.random.RandomState(seed=42).rand(2, 1, 10, 10).astype(np.float32),
"W1": np.random.RandomState(seed=42).rand(2, 1, 100, 100).astype(np.float32),
"W2": np.random.RandomState(seed=42).rand(2, 1, 20, 20).astype(np.float32),
"Y": np.random.RandomState(seed=42).rand(2, 1, 32, 32).astype(np.float32),
}
Expecting nothing
ok
Trying:
discriminator_model = DiscriminatorModel()
Expecting nothing
ok
Trying:
discriminator_optimizer = chainer.optimizers.Adam(alpha=0.001, eps=1e-7).setup(
link=discriminator_model
)
Expecting nothing
ok
Trying:
generator_model = GeneratorModel()
Expecting nothing
ok
Trying:
d_weight0 = [d for d in discriminator_model.params()][-3][0].array
Expecting nothing
ok
Trying:
d_train_loss, d_train_accu = train_eval_discriminator(
input_arrays=train_arrays,
g_model=generator_model,
d_model=discriminator_model,
d_optimizer=discriminator_optimizer,
)
Expecting nothing
ok
Trying:
d_weight1 = [d for d in discriminator_model.params()][-3][0].array
Expecting nothing
ok
Trying:
d_weight0 != d_weight1 #check that training has occurred (i.e. weights changed)
Expecting:
True
ok
Trying:
train_arrays = {
"X": np.random.RandomState(seed=42).rand(2, 1, 10, 10).astype(np.float32),
"W1": np.random.RandomState(seed=42).rand(2, 1, 100, 100).astype(np.float32),
"W2": np.random.RandomState(seed=42).rand(2, 1, 20, 20).astype(np.float32),
"Y": np.random.RandomState(seed=42).rand(2, 1, 32, 32).astype(np.float32),
}
Expecting nothing
ok
Trying:
generator_model = GeneratorModel()
Expecting nothing
ok
Trying:
generator_optimizer = chainer.optimizers.Adam(alpha=0.001, eps=1e-7).setup(
link=generator_model
)
Expecting nothing
ok
Trying:
discriminator_model = DiscriminatorModel()
Expecting nothing
ok
Trying:
g_weight0 = [g for g in generator_model.params()][8][0, 0, 0, 0].array
Expecting nothing
ok
Trying:
_ = train_eval_generator(
input_arrays=train_arrays,
g_model=generator_model,
d_model=discriminator_model,
g_optimizer=generator_optimizer,
)
Expecting nothing
ok
Trying:
g_weight1 = [g for g in generator_model.params()][8][0, 0, 0, 0].array
Expecting nothing
ok
Trying:
g_weight0 != g_weight1 #check that training has occurred (i.e. weights changed)
Expecting:
True
ok
20 items had no tests:
srgan_train
srgan_train.DeepbedmapInputBlock
srgan_train.DeepbedmapInputBlock.__init__
srgan_train.DeepbedmapInputBlock.forward
srgan_train.DiscriminatorModel.__init__
srgan_train.DiscriminatorModel.forward
srgan_train.GeneratorModel.__init__
srgan_train.GeneratorModel.forward
srgan_train.ResInResDenseBlock
srgan_train.ResInResDenseBlock.__init__
srgan_train.ResInResDenseBlock.forward
srgan_train.ResidualDenseBlock
srgan_train.ResidualDenseBlock.__init__
srgan_train.ResidualDenseBlock.forward
srgan_train.compile_srgan_model
srgan_train.get_deepbedmap_test_result
srgan_train.get_train_dev_iterators
srgan_train.load_data_into_memory
srgan_train.objective
srgan_train.trainer
8 items passed all tests:
4 tests in srgan_train.DiscriminatorModel
4 tests in srgan_train.GeneratorModel
1 tests in srgan_train.calculate_discriminator_loss
1 tests in srgan_train.calculate_generator_loss
1 tests in srgan_train.psnr
3 tests in srgan_train.save_model_weights_and_architecture
8 tests in srgan_train.train_eval_discriminator
8 tests in srgan_train.train_eval_generator
30 tests in 28 items.
30 passed and 0 failed.
Test passed.
_integration_test_ipynb(path="features/data_prep.feature")
@fixture.data_prep
Feature: Data preparation # features/data_prep.feature:3
In order to have reproducible data inputs for everyone
As a data scientist,
We want to share cryptographically secured pieces of the datasets
Scenario Outline: Download and check data -- @1.1 Files to download and check # features/data_prep.feature:15
Given this https://data.cresis.ku.edu/data/rds/2017_Antarctica_Basler/csv_good/Data_20171204_02.csv link to a file hosted on the web # features/steps/test_data_prep.py:8
When we download it to highres/Data_20171204_02.csv # features/steps/test_data_prep.py:13
Then the local file should have this 53cef7a0d28ff92b30367514f27e888efbc32b1bda929981b371d2e00d4c671b checksum # features/steps/test_data_prep.py:19
Scenario Outline: Download and check data -- @1.2 Files to download and check # features/data_prep.feature:16
Given this http://ramadda.nerc-bas.ac.uk/repository/entry/get/Polar%20Data%20Centre/DOI/Rutford%20Ice%20Stream%20bed%20elevation%20DEM%20from%20radar%20data/bed_WGS84_grid.txt?entryid=synth%3A54757cbe-0b13-4385-8b31-4dfaa1dab55e%3AL2JlZF9XR1M4NF9ncmlkLnR4dA%3D%3D link to a file hosted on the web # features/steps/test_data_prep.py:8
When we download it to highres/bed_WGS84_grid.txt # features/steps/test_data_prep.py:13
Then the local file should have this 7396e56cda5adb82cecb01f0b3e01294ed0aa6489a9629f3f7e8858ea6cb91cf checksum # features/steps/test_data_prep.py:19
Scenario Outline: Grid datasets -- @1.1 ASCII text files to grid # features/data_prep.feature:26
Given a collection of raw high resolution datasets bed_WGS84_grid.txt # features/steps/test_data_prep.py:25
When we process the data through bed_WGS84_grid.json # features/steps/test_data_prep.py:38
And interpolate the xyz data table to bed_WGS84_grid.nc # features/steps/test_data_prep.py:45
Then a high resolution raster grid is returned # features/steps/test_data_prep.py:54
Scenario Outline: Tile datasets -- @1.1 Raster grids to tile # features/data_prep.feature:36
Given a big highres raster grid 2010tr.nc # features/steps/test_data_prep.py:60
And a collection of square bounding boxes "model/train/tiles_3031.geojson" # features/steps/test_data_prep.py:70
When we crop the big raster grid using those bounding boxes # features/steps/test_data_prep.py:80
Then a stack of small raster tiles is returned # features/steps/test_data_prep.py:87
_integration_test_ipynb(path="features/srgan_train.feature")
@fixture.srgan_train
Feature: Train Super Resolution Model # features/srgan_train.feature:3
In order to have a well performing super resolution model
As a machine learning engineer,
We want to craft and teach the model to do well on a test area
Background: Load the prepared data # features/srgan_train.feature:8
Scenario Outline: Train Super Resolution Model with fixed hyperparameters -- @1.1 Fixed hyperparameters # features/srgan_train.feature:19
Given a prepared collection of tiled raster data # features/steps/test_srgan_train.py:6
Given some hyperparameter settings 1 0.3 5e-4 # features/steps/test_srgan_train.py:14
And a compiled neural network model # features/steps/test_srgan_train.py:25
When the model is trained for a while # features/steps/test_srgan_train.py:35
Then we know how well the model performs on our test area # features/steps/test_srgan_train.py:60
_integration_test_ipynb(path="features/deepbedmap.feature")
@fixture.deepbedmap
Feature: DeepBedMap # features/deepbedmap.feature:3
In order to create a great map of Antarctica's bed
As a scientist,
We want a model that produces realistic images from many open datasets
Scenario Outline: Determine high resolution bed -- @1.1 Bounding box views of Antarctica # features/deepbedmap.feature:16
Given some view of Antarctica -1593714.328,-164173.7848,-1575464.328,-97923.7848 # features/steps/test_deepbedmap.py:6
When we gather low and high resolution images related to that view # features/steps/test_deepbedmap.py:14
And pass those images into our trained neural network model # features/steps/test_deepbedmap.py:30
Then a four times upsampled super resolution bed elevation map is returned # features/steps/test_deepbedmap.py:38