docker run - p 8888:8888 -v /Users/sam/data/:/data -v /Users/sam/owl_home/:/owl_home -v /Users/sam/owl_web/:owl_web -v /Users/sam/gitrepos/LabDocs/jupyter_nbs/sam/:/jupyter_nbs -it 0ba43904567e
The command allows access to Jupyter Notebook over port 8888 and makes my Jupyter Notebook GitHub repo and my data files on Owl/home
and Owl/web
accessible to the Docker container.
Once the container was started, started Jupyter Notebook with the following command inside the Docker container:
jupyter notebook
This is configured in the Docker container to launch a Jupyter Notebook without a browser on port 8888.
%%bash
date
Thu Nov 17 18:38:26 UTC 2016
%%bash
hostname
65e23f54bcc9
%%bash
lscpu
Architecture: x86_64 CPU op-mode(s): 32-bit, 64-bit Byte Order: Little Endian CPU(s): 8 On-line CPU(s) list: 0-7 Thread(s) per core: 1 Core(s) per socket: 8 Socket(s): 1 Vendor ID: GenuineIntel CPU family: 6 Model: 26 Model name: Intel(R) Xeon(R) CPU E5520 @ 2.27GHz Stepping: 5 CPU MHz: 2260.998 BogoMIPS: 4521.99 Hypervisor vendor: KVM Virtualization type: full L1d cache: 32K L1i cache: 32K L2 cache: 256K L3 cache: 8192K
%%bash
ls -l /root/owl_home/O_lurida_BGI_F15FTSUSAT0327_OSTibkD/
total 62027628 -rw-rw-rw- 1 srlab staff 7062253846 Jan 28 2016 151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz -rw-rw-rw- 1 srlab staff 7352262786 Jan 28 2016 151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz -rw-rw-rw- 1 srlab staff 7002262800 Jan 28 2016 151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz -rw-rw-rw- 1 srlab staff 6579290112 Jan 28 2016 151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz -rw-rw-rw- 1 srlab staff 2539393024 Jan 28 2016 151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz -rw-rw-rw- 1 srlab staff 4269542816 Jan 28 2016 151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz -rw-rw-rw- 1 srlab staff 2570830561 Jan 28 2016 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz -rw-rw-rw- 1 srlab staff 2900561782 Jan 28 2016 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz -rw-rw-rw- 1 srlab staff 1370906266 Jan 28 2016 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz -rw-rw-rw- 1 srlab staff 1531179372 Jan 28 2016 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz -rw-rw-rw- 1 srlab staff 2563497188 Jan 28 2016 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz -rw-rw-rw- 1 srlab staff 2932758209 Jan 28 2016 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz -rw-rw-rw- 1 srlab staff 1350152634 Jan 28 2016 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz -rw-rw-rw- 1 srlab staff 1527682296 Jan 28 2016 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz -rw-rw-rw- 1 srlab staff 2851433324 Jan 28 2016 160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz -rw-rw-rw- 1 srlab staff 3249975601 Jan 28 2016 160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz -rw-rw-rw- 1 srlab staff 2734377366 Jan 28 2016 160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz -rw-rw-rw- 1 srlab staff 3127877914 Jan 28 2016 160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz -rw-rw-rw- 1 srlab staff 2712 Jan 27 2016 checksums.md5 -rw-r--r-- 1 srlab staff 1614 Jan 25 2016 md5.txt
%%bash
ls -l /root/owl_web/nightingales/O_lurida/1[56]*.gz
-rw-rw-rw- 1 srlab staff 7062253846 Jan 25 2016 /root/owl_web/nightingales/O_lurida/151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz -rw-rw-rw- 1 srlab staff 7352262786 Jan 25 2016 /root/owl_web/nightingales/O_lurida/151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz -rw-rw-rw- 1 srlab staff 7002262800 Jan 25 2016 /root/owl_web/nightingales/O_lurida/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz -rw-rw-rw- 1 srlab staff 6579290112 Jan 27 2016 /root/owl_web/nightingales/O_lurida/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz -rw-rw-rw- 1 srlab staff 2539393024 Jan 27 2016 /root/owl_web/nightingales/O_lurida/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz -rw-rw-rw- 1 srlab staff 4269542816 Jan 25 2016 /root/owl_web/nightingales/O_lurida/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz -rw-rw-rw- 1 srlab staff 2570830561 Jan 25 2016 /root/owl_web/nightingales/O_lurida/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz -rw-rw-rw- 1 srlab staff 2900561782 Jan 25 2016 /root/owl_web/nightingales/O_lurida/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz -rw-rw-rw- 1 srlab staff 1370906266 Jan 25 2016 /root/owl_web/nightingales/O_lurida/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz -rw-rw-rw- 1 srlab staff 1531179372 Jan 25 2016 /root/owl_web/nightingales/O_lurida/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz -rw-rw-rw- 1 srlab staff 2563497188 Jan 25 2016 /root/owl_web/nightingales/O_lurida/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz -rw-rw-rw- 1 srlab staff 2932758209 Jan 25 2016 /root/owl_web/nightingales/O_lurida/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz -rw-rw-rw- 1 srlab staff 1350152634 Jan 25 2016 /root/owl_web/nightingales/O_lurida/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz -rw-rw-rw- 1 srlab staff 1527682296 Jan 25 2016 /root/owl_web/nightingales/O_lurida/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz -rw-rw-rw- 1 srlab staff 2851433324 Jan 25 2016 /root/owl_web/nightingales/O_lurida/160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz -rw-rw-rw- 1 srlab staff 3249975601 Jan 25 2016 /root/owl_web/nightingales/O_lurida/160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz -rw-rw-rw- 1 srlab staff 2734377366 Jan 25 2016 /root/owl_web/nightingales/O_lurida/160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz -rw-rw-rw- 1 srlab staff 3127877914 Jan 25 2016 /root/owl_web/nightingales/O_lurida/160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz
%%bash
grep -e 151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz -e 151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz /root/owl_web/nightingales/O_lurida/checksums.md5
MD5 (151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz) = 8bc0d7c7a7af3954baca31a4a7fe9f2b MD5 (151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz) = 08cfdc6fdc5a6190cb05cdcb81fa5b9c
%%bash
md5sum /root/owl_web/nightingales/O_lurida/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz
861c8161c1e0171532a058582db3ae6d /root/owl_web/nightingales/O_lurida/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz
%%bash
md5sum /root/owl_web/nightingales/O_lurida/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz
f10c1f143d153f364cf6be13452eea4a /root/owl_web/nightingales/O_lurida/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz
%%bash
grep -e 151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz -e 151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz \
/root/owl_home/O_lurida_BGI_F15FTSUSAT0327_OSTibkD/checksums.md5
MD5 (/Volumes/owl_home/O_lurida_BGI_F15FTSUSAT0327_OSTibkD/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz) = 8bc0d7c7a7af3954baca31a4a7fe9f2b MD5 (/Volumes/owl_home/O_lurida_BGI_F15FTSUSAT0327_OSTibkD/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz) = 08cfdc6fdc5a6190cb05cdcb81fa5b9c
%%bash
grep -e 151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz -e 151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz \
/root/owl_home/O_lurida_BGI_F15FTSUSAT0327_OSTibkD/md5.txt
8bc0d7c7a7af3954baca31a4a7fe9f2b 151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz 08cfdc6fdc5a6190cb05cdcb81fa5b9c 151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz
%%bash
md5sum /root/owl_home/O_lurida_BGI_F15FTSUSAT0327_OSTibkD/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz
861c8161c1e0171532a058582db3ae6d /root/owl_home/O_lurida_BGI_F15FTSUSAT0327_OSTibkD/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz
%%bash
md5sum /root/owl_home/O_lurida_BGI_F15FTSUSAT0327_OSTibkD/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz
f10c1f143d153f364cf6be13452eea4a /root/owl_home/O_lurida_BGI_F15FTSUSAT0327_OSTibkD/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz
%%bash
ls -l /root/owl_home/O_lurida_BGI_F15FTSUSAT0327_OSTibkD/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz
-rw-rw-rw- 1 srlab staff 6579290112 Jan 28 2016 /root/owl_home/O_lurida_BGI_F15FTSUSAT0327_OSTibkD/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz
%%bash
ls -l /root/owl_web/nightingales/O_lurida/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz
-rw-rw-rw- 1 srlab staff 6579290112 Jan 27 2016 /root/owl_web/nightingales/O_lurida/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz
%%bash
ls -l /root/owl_home/O_lurida_BGI_F15FTSUSAT0327_OSTibkD/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz
-rw-rw-rw- 1 srlab staff 2539393024 Jan 28 2016 /root/owl_home/O_lurida_BGI_F15FTSUSAT0327_OSTibkD/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz
%%bash
ls -l /root/owl_web/nightingales/O_lurida/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz
-rw-rw-rw- 1 srlab staff 2539393024 Jan 27 2016 /root/owl_web/nightingales/O_lurida/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz
Owl/home
directory were re-downloaded at some point. However, I can't explain why the files in both locations have the same MD5 checksum value AND those values differ from the checksum values recorded in the checksum files (mine and that provided by BGI)...¶The command below works as follows: time - Tracks how long process takes to run
Uses for loop to look at all FASTQ files from BGI in the owl/web/nightingales/O_lurida
directory. For each of those files:
Runs the program md5sum
on each file to generate a checksum and pipes the output to awk. Awk prints the first field ($1) from the output of md5sum
. This field conatins the checksum value. This value is stored in the variable called curr_md5
.
Runs the program grep
to search for the filename in the checksums.md5 file. The filename is extracted from the $fastq
variable (this variable contains the full path to each file, which will screw up the search by grep
) using parameter expansion/substring removal. The command looks for the longest pattern matching what we provide (the ##
tells it to look for the longest pattern). We told bash to look for anything matching a forward slash (the */
in the command). The output from grep
is piped to awk
. Awk prints the fourth field ($4) from the output, which corresponds to the checksum value in the checksums.md5
file. This value is stored in the variable called orig_md5
.
Prints (echo "$curr_md5"
) the value currently stored in the curr_md5
variable, then prints (echo "$orig_md5"
) the value currently stored in orig_md5
, followed by printing an empty line (echo ""
) to improve readibility.
%%bash
time for fastq in /root/owl_web/nightingales/O_lurida/1[56]*.gz; do \
curr_md5=$( md5sum "$fastq" | awk '{ print $1 }')
orig_md5=$( grep ${fastq##*/} /root/owl_web/nightingales/O_lurida/checksums.md5 | awk '{ print $4}')
echo "$curr_md5"
echo "$orig_md5"
echo ""
done
810d188468dbd8bb36b2af3bf3b9fee6 810d188468dbd8bb36b2af3bf3b9fee6 cf92b18e0815dc0471d61f9107142257 cf92b18e0815dc0471d61f9107142257 3dc2137d7df0af8d6a007516908361a3 3dc2137d7df0af8d6a007516908361a3 861c8161c1e0171532a058582db3ae6d 8bc0d7c7a7af3954baca31a4a7fe9f2b f10c1f143d153f364cf6be13452eea4a 08cfdc6fdc5a6190cb05cdcb81fa5b9c a503043167457337a65d51151ceb5dd0 a503043167457337a65d51151ceb5dd0 a0c0177a7a4a4ca28c37bd3802361564 a0c0177a7a4a4ca28c37bd3802361564 74e51d608de7a409a29545a95ac3ec14 74e51d608de7a409a29545a95ac3ec14 e829da6f7e7a047586802a05d3b96d18 e829da6f7e7a047586802a05d3b96d18 74953d0143c392f111eec178667d3fcd 74953d0143c392f111eec178667d3fcd 9b5f7c1593f216f710814c299738493f 9b5f7c1593f216f710814c299738493f 5a6392d9c23aa85d170d7f50de4f0b54 5a6392d9c23aa85d170d7f50de4f0b54 2cf349b55558e7424fa0ed029ae0bfca 2cf349b55558e7424fa0ed029ae0bfca 5cb745671596d5d5d307986d35ea9792 5cb745671596d5d5d307986d35ea9792 b371a3b3588060bb2200f5caaf9a9d5c b371a3b3588060bb2200f5caaf9a9d5c dd8b0fef21fc5e330d08ae4a48c8d67b dd8b0fef21fc5e330d08ae4a48c8d67b 50b5dbff9426738005b81efd49d66329 50b5dbff9426738005b81efd49d66329 29be6e734ad65c180aee23d1514c5c35 29be6e734ad65c180aee23d1514c5c35
real 37m34.177s user 0m4.530s sys 6m46.800s