import ipcoal
import toytree
import ipyrad.analysis as ipa
import ipyparallel as ipp
Here I assume that you already started an ipcluster instance in a terminal using the command below, or by starting engines in the Ipython Clusters tab in Jupyter. Remember that when you pull in new updates and restart your kernel you also need to restart your cluster instance.
ipcluster start --n=4
# connect to a running client
ipyclient = ipp.Client()
# show number of engines
ipyclient.ids
[0, 1, 2, 3]
# make a random tree
tree = toytree.rtree.unittree(ntips=5, treeheight=5e5, seed=1243)
tree.draw(ts='p');
# simulate loci and write to HDF5
model = ipcoal.Model(tree, Ne=1e5, nsamples=4)
model.sim_loci(100, 500)
model.write_loci_to_hdf5(name="test", outdir="/tmp", diploid=True)
wrote 100 loci to /tmp/test.seqs.hdf5
# create an IMAP
IMAP = {
'r' + str(i): [j for j in model.alpha_ordered_names if int(j[1]) == i][:2]
for i in range(5)
}
IMAP
{'r0': ['r0-0', 'r0-1'], 'r1': ['r1-0', 'r1-1'], 'r2': ['r2-0', 'r2-1'], 'r3': ['r3-0', 'r3-1'], 'r4': ['r4-0', 'r4-1']}
# init bpp tool.
bpp1 = ipa.bpp(
data="/tmp/test.seqs.hdf5",
name="test1",
workdir="/tmp",
guidetree=tree,
imap=IMAP,
maxloci=100,
burnin=1000,
nsample=5000,
)
bpp1.kwargs
{'binary': '/tmp/bpp-4.1.4-linux-x86_64/bin/bpp', 'infer_sptree': 0, 'infer_delimit': 0, 'infer_delimit_args': (0, 2), 'speciesmodelprior': 1, 'seed': 12345, 'burnin': 1000, 'nsample': 5000, 'sampfreq': 2, 'thetaprior': (3, 0.002), 'tauprior': (3, 0.002), 'phiprior': (1, 1), 'usedata': 1, 'cleandata': 0, 'finetune': (0.01, 0.02, 0.03, 0.04, 0.05, 0.01, 0.01), 'copied': False}
._run()
)¶# submit 2 jobs to ipyclient
bpp1._run(nreps=2, ipyclient=ipyclient, force=True, block=False, dry_run=False)
[locus filter] full data: 100 [locus filter] post filter: 100 [ipa bpp] bpp v4.1.4 [ipa.bpp] distributed 2 bpp jobs (name=test1, nloci=100)
Here I use the .copy()
function for convenience, but you could just create a new BPP object and call the ._run()
command with the same ipyclient object.
# submit X other jobs to ipyclient (e.g., using diff job name)
bpp2 = bpp1.copy("test2")
bpp2._run(nreps=4, ipyclient=ipyclient, force=True, block=False, dry_run=False)
[locus filter] full data: 100 [locus filter] post filter: 100 [ipa bpp] bpp v4.1.4 [ipa.bpp] distributed 4 bpp jobs (name=test2, nloci=100)
# see the jobs that are submitted
bpp1.asyncs
[<AsyncResult: _call_bpp>, <AsyncResult: _call_bpp>]
bpp2.asyncs
[<AsyncResult: _call_bpp>, <AsyncResult: _call_bpp>, <AsyncResult: _call_bpp>, <AsyncResult: _call_bpp>]
# see outstanding jobs (optional, this does NOT BLOCK)
ipyclient.outstanding
{'ce917031-cd9a6264cd01a65d101b520d_1', 'ce917031-cd9a6264cd01a65d101b520d_2', 'ce917031-cd9a6264cd01a65d101b520d_3', 'ce917031-cd9a6264cd01a65d101b520d_4', 'ce917031-cd9a6264cd01a65d101b520d_5', 'ce917031-cd9a6264cd01a65d101b520d_6'}
# BLOCK until all jobs on ipyclient are finished (returns True when done)
ipyclient.wait()
True
res, mcmc = bpp1.summarize_results("00")
res
[ipa.bpp] found 2 existing result files [ipa.bpp] summarizing algorithm '00' results [ipa.bpp] combining mcmc files
theta_1___r0 | theta_2___r1 | theta_3___r2 | theta_4___r3 | theta_5___r4 | theta_6___r4___r3___r2___r1___r0 | theta_7___r3___r2___r1___r0 | theta_8___r2___r1___r0 | theta_9___r1___r0 | tau_6___r4___r3___r2___r1___r0 | tau_7___r3___r2___r1___r0 | tau_8___r2___r1___r0 | tau_9___r1___r0 | lnL | |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
mean | 0.005618 | 0.005165 | 0.005147 | 0.005015 | 0.005639 | 0.003771 | 0.003454 | 0.003208 | 0.004285 | 0.006047 | 0.004932 | 0.003717 | 0.002147 | -85060.071771 |
median | 0.005465 | 0.005088 | 0.005066 | 0.004959 | 0.005590 | 0.003736 | 0.003276 | 0.002999 | 0.004219 | 0.006041 | 0.004939 | 0.003731 | 0.002151 | -85059.570500 |
S.D | 0.001084 | 0.000905 | 0.000750 | 0.000668 | 0.000705 | 0.000662 | 0.001705 | 0.001433 | 0.001264 | 0.000328 | 0.000332 | 0.000280 | 0.000239 | 20.100881 |
min | 0.003176 | 0.002896 | 0.003237 | 0.003162 | 0.003166 | 0.002102 | 0.000497 | 0.000778 | 0.001362 | 0.005014 | 0.003841 | 0.002844 | 0.001302 | -85152.837000 |
max | 0.012041 | 0.009645 | 0.009551 | 0.008770 | 0.009112 | 0.006463 | 0.010233 | 0.011392 | 0.009579 | 0.007192 | 0.005975 | 0.004538 | 0.002916 | -84984.863000 |
2.5% | 0.003914 | 0.003710 | 0.003892 | 0.003842 | 0.004395 | 0.002578 | 0.000841 | 0.001161 | 0.002080 | 0.005461 | 0.004291 | 0.003164 | 0.001685 | -85099.356000 |
97.5% | 0.008072 | 0.007200 | 0.006797 | 0.006512 | 0.007172 | 0.005149 | 0.007317 | 0.006452 | 0.006955 | 0.006690 | 0.005563 | 0.004225 | 0.002611 | -85021.884000 |
2.5%HPD | 0.003839 | 0.003573 | 0.003788 | 0.003697 | 0.004306 | 0.002547 | 0.000601 | 0.001054 | 0.001920 | 0.005437 | 0.004290 | 0.003193 | 0.001697 | -85100.071000 |
97.5%HPD | 0.007945 | 0.006977 | 0.006630 | 0.006312 | 0.007031 | 0.005061 | 0.006613 | 0.006061 | 0.006703 | 0.006654 | 0.005555 | 0.004247 | 0.002619 | -85022.941000 |
ESS* | 1134.830056 | 921.295891 | 1705.249906 | 1975.046253 | 1918.434542 | 43.496889 | 29.778334 | 43.885352 | 130.541971 | 33.537301 | 33.276858 | 94.383590 | 212.132455 | 643.462922 |
Eff* | 0.226966 | 0.184259 | 0.341050 | 0.395009 | 0.383687 | 0.008699 | 0.005956 | 0.008777 | 0.026108 | 0.006707 | 0.006655 | 0.018877 | 0.042426 | 0.128693 |
res, mcmc = bpp2.summarize_results("00")
res
[ipa.bpp] found 4 existing result files [ipa.bpp] summarizing algorithm '00' results [ipa.bpp] combining mcmc files
theta_1___r0 | theta_2___r1 | theta_3___r2 | theta_4___r3 | theta_5___r4 | theta_6___r4___r3___r2___r1___r0 | theta_7___r3___r2___r1___r0 | theta_8___r2___r1___r0 | theta_9___r1___r0 | tau_6___r4___r3___r2___r1___r0 | tau_7___r3___r2___r1___r0 | tau_8___r2___r1___r0 | tau_9___r1___r0 | lnL | |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
mean | 0.005595 | 0.005268 | 0.005171 | 0.005000 | 0.005598 | 0.003988 | 0.002784 | 0.003788 | 0.004057 | 0.005931 | 0.005048 | 0.003632 | 0.002188 | -85057.458892 |
median | 0.005519 | 0.005158 | 0.005113 | 0.004964 | 0.005555 | 0.003964 | 0.002613 | 0.003579 | 0.004009 | 0.005925 | 0.005057 | 0.003638 | 0.002172 | -85056.968500 |
S.D | 0.001029 | 0.000981 | 0.000757 | 0.000665 | 0.000710 | 0.000597 | 0.001227 | 0.001541 | 0.001474 | 0.000274 | 0.000282 | 0.000310 | 0.000280 | 19.979165 |
min | 0.003266 | 0.002904 | 0.003178 | 0.003119 | 0.003063 | 0.002230 | 0.000527 | 0.000985 | 0.000646 | 0.005103 | 0.004063 | 0.002705 | 0.001193 | -85140.649000 |
max | 0.011813 | 0.010669 | 0.009579 | 0.007917 | 0.009220 | 0.006296 | 0.008654 | 0.012507 | 0.010266 | 0.006933 | 0.005904 | 0.004548 | 0.003183 | -84988.645000 |
2.5% | 0.003878 | 0.003679 | 0.003878 | 0.003836 | 0.004326 | 0.002867 | 0.000987 | 0.001496 | 0.001115 | 0.005424 | 0.004499 | 0.003021 | 0.001684 | -85098.361000 |
97.5% | 0.007902 | 0.007595 | 0.006840 | 0.006448 | 0.007099 | 0.005202 | 0.005678 | 0.007443 | 0.007082 | 0.006486 | 0.005566 | 0.004225 | 0.002799 | -85018.670000 |
2.5%HPD | 0.003734 | 0.003472 | 0.003846 | 0.003757 | 0.004243 | 0.002858 | 0.000866 | 0.001056 | 0.001018 | 0.005394 | 0.004499 | 0.003037 | 0.001688 | -85099.726000 |
97.5%HPD | 0.007697 | 0.007163 | 0.006783 | 0.006288 | 0.006977 | 0.005181 | 0.005341 | 0.006725 | 0.006835 | 0.006435 | 0.005566 | 0.004235 | 0.002799 | -85020.784000 |
ESS* | 933.516203 | 529.779120 | 1462.078574 | 1355.323260 | 1658.165363 | 199.463243 | 86.210668 | 63.843695 | 80.629951 | 132.112942 | 106.775107 | 67.380380 | 103.608294 | 1524.229702 |
Eff* | 0.186703 | 0.105956 | 0.292416 | 0.271065 | 0.331633 | 0.039893 | 0.017242 | 0.012769 | 0.016126 | 0.026423 | 0.021355 | 0.013476 | 0.020722 | 0.304846 |