run -p 8888:8888 -v /Users/sam/data/:/data -v /Users/sam/owl_home/:/owl_home -v /Users/sam/owl_web/:/owl_web -v /Users/sam/gitrepos:/gitrepos -it 2f0f50dc230c
The command allows access to Jupyter Notebook over port 8888 and makes my Jupyter Notebook GitHub repo and my data files on Owl/home and Owl/web accessible to the Docker container.
Once the container was started, started Jupyter Notebook with the following command inside the Docker container:
jupyter notebook
This is configured in the Docker container to launch a Jupyter Notebook without a browser on port 8888.
The Docker container is running on an image created from this Dockerfile (Git commit ac060a2)
%%bash
date
Wed Dec 14 20:55:08 UTC 2016
%%bash
hostname
0f2bca9c664b
%%bash
lscpu
Architecture: x86_64 CPU op-mode(s): 32-bit, 64-bit Byte Order: Little Endian CPU(s): 8 On-line CPU(s) list: 0-7 Thread(s) per core: 1 Core(s) per socket: 8 Socket(s): 1 Vendor ID: GenuineIntel CPU family: 6 Model: 26 Model name: Intel(R) Xeon(R) CPU E5520 @ 2.27GHz Stepping: 5 CPU MHz: 2260.998 BogoMIPS: 4521.99 Hypervisor vendor: KVM Virtualization type: full L1d cache: 32K L1i cache: 32K L2 cache: 256K L3 cache: 8192K
%%bash
tree /owl_web/O_lurida_genome_assemblies_BGI/20161201/
/owl_web/O_lurida_genome_assemblies_BGI/20161201/
`-- cdts-hk.genomics.cn
`-- Ostrea_lurida
|-- 17mer.freq
|-- 17mer.log
|-- N50.xls
|-- Ostrea_lurida.fa
|-- clean_data
| |-- 151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz.clean.dup.clean.gz
| |-- 151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz.clean.dup.clean.gz
| |-- 151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz.clean.dup.clean.gz
| |-- 151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz.clean.dup.clean.gz
| |-- 151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz.clean.dup.clean.gz
| |-- 151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz
| `-- lane.lst.stat.xls
|-- md5.check
`-- md5.txt
3 directories, 25 files
%%bash
tree /owl_web/P_generosa_genome_assemblies_BGI/20161201/
/owl_web/P_generosa_genome_assemblies_BGI/20161201/
`-- cdts-hk.genomics.cn
`-- Panopea_generosa
|-- 17mer.freq
|-- 17mer.log
|-- N50.xls
|-- Panopea_generosa.fa
|-- clean_data
| |-- 151114_I191_FCH3Y35BCXX_L1_wHAIPI023989-79_1.fq.gz.clean.dup.clean.gz
| |-- 151114_I191_FCH3Y35BCXX_L1_wHAIPI023989-79_2.fq.gz.clean.dup.clean.gz
| |-- 151114_I191_FCH3Y35BCXX_L2_wHAMPI023988-81_1.fq.gz.clean.dup.clean.gz
| |-- 151114_I191_FCH3Y35BCXX_L2_wHAMPI023988-81_2.fq.gz.clean.dup.clean.gz
| |-- 151122_I136_FCH3L2FBBXX_L7_wHAXPI023990-97_1.fq.gz.clean.dup.clean.gz
| |-- 151122_I136_FCH3L2FBBXX_L7_wHAXPI023990-97_2.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L3_WHPANwalDDAADWAAPEI-101_1.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L3_WHPANwalDDAADWAAPEI-101_2.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L4_WHPANwalDDAADWAAPEI-101_1.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L4_WHPANwalDDAADWAAPEI-101_2.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L5_WHPANwalDDABDLAAPEI-100_1.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L5_WHPANwalDDABDLAAPEI-100_2.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L5_WHPANwalDDACDTAAPEI-102_1.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L5_WHPANwalDDACDTAAPEI-102_2.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L6_WHPANwalDDABDLAAPEI-100_1.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L6_WHPANwalDDABDLAAPEI-100_2.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L6_WHPANwalDDACDTAAPEI-102_1.fq.gz.clean.dup.clean.gz
| |-- 160103_I137_FCH3V5YBBXX_L6_WHPANwalDDACDTAAPEI-102_2.fq.gz.clean.dup.clean.gz
| `-- lane.lst.stat.xls
|-- md5.check
`-- md5.txt
3 directories, 25 files
%%bash
cat /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/md5.check
clean_data/151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz.clean.dup.clean.gz: OK clean_data/151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz.clean.dup.clean.gz: OK clean_data/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz.clean.dup.clean.gz: OK clean_data/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz.clean.dup.clean.gz: OK clean_data/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz.clean.dup.clean.gz: OK clean_data/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz.clean.dup.clean.gz: OK clean_data/160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz: OK clean_data/160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz: OK clean_data/160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz: OK clean_data/160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz: OK clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz: OK clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz: OK clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz: OK clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz: OK clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz: OK clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz: OK clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz: OK clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz: OK 17mer.log: OK 17mer.freq: OK N50.xls: OK Ostrea_lurida.fa: OK
%%bash
cat /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/md5.txt
810d188468dbd8bb36b2af3bf3b9fee6 clean_data/151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz.clean.dup.clean.gz cf92b18e0815dc0471d61f9107142257 clean_data/151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz.clean.dup.clean.gz 3dc2137d7df0af8d6a007516908361a3 clean_data/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz.clean.dup.clean.gz 8bc0d7c7a7af3954baca31a4a7fe9f2b clean_data/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz.clean.dup.clean.gz 08cfdc6fdc5a6190cb05cdcb81fa5b9c clean_data/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz.clean.dup.clean.gz a503043167457337a65d51151ceb5dd0 clean_data/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz.clean.dup.clean.gz b371a3b3588060bb2200f5caaf9a9d5c clean_data/160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz dd8b0fef21fc5e330d08ae4a48c8d67b clean_data/160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz 50b5dbff9426738005b81efd49d66329 clean_data/160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz 29be6e734ad65c180aee23d1514c5c35 clean_data/160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz a0c0177a7a4a4ca28c37bd3802361564 clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz 74e51d608de7a409a29545a95ac3ec14 clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz 9b5f7c1593f216f710814c299738493f clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz 5a6392d9c23aa85d170d7f50de4f0b54 clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz 7438fcf14797976de9288368236ed75d clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz 26dd6bc17c1596e2881924bcc167125a clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz 4221fa24ccbb1735202fcfaa80d69d95 clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz 0adbb85da2610131b728417669e0922c clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz fafc80b25613cff45598f4f37dde9d8e 17mer.log e9ea19f40b0ebb212ab33c78e350a679 17mer.freq a4f92de7bd24d7ebbc02c40696a0f4e8 N50.xls f0a7772d4f1074698b50c913783c6fe2 Ostrea_lurida.fa
%%bash
#For loop generates a md5 checksum has value for each file
#and appends the output to the checksums.md5 file.
time for file in /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/*.gz
do
md5sum "$file" >> /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/checksums.md5
done
real 31m28.393s user 0m7.370s sys 6m27.230s
%%bash
cat /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/checksums.md5
410cfcdf170125f4d8cb1ac4baf0007c /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz.clean.dup.clean.gz cf92b18e0815dc0471d61f9107142257 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz.clean.dup.clean.gz 3dc2137d7df0af8d6a007516908361a3 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz.clean.dup.clean.gz 8bc0d7c7a7af3954baca31a4a7fe9f2b /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz.clean.dup.clean.gz 08cfdc6fdc5a6190cb05cdcb81fa5b9c /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz.clean.dup.clean.gz a503043167457337a65d51151ceb5dd0 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz.clean.dup.clean.gz a0c0177a7a4a4ca28c37bd3802361564 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz 74e51d608de7a409a29545a95ac3ec14 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz 7438fcf14797976de9288368236ed75d /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz 26dd6bc17c1596e2881924bcc167125a /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz 9b5f7c1593f216f710814c299738493f /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz 5a6392d9c23aa85d170d7f50de4f0b54 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz 4221fa24ccbb1735202fcfaa80d69d95 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz 0adbb85da2610131b728417669e0922c /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz b371a3b3588060bb2200f5caaf9a9d5c /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz dd8b0fef21fc5e330d08ae4a48c8d67b /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz 50b5dbff9426738005b81efd49d66329 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz 29be6e734ad65c180aee23d1514c5c35 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz
%%bash
#For loop generates a md5 checksum has value for each file
#and appends the output to the checksums.md5 file.
time for file in /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/*.[lfx]*
do
md5sum "$file" >> /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/checksums.md5
done
real 0m16.567s user 0m0.090s sys 0m4.270s
%%bash
cat /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/checksums.md5
e9ea19f40b0ebb212ab33c78e350a679 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/17mer.freq fafc80b25613cff45598f4f37dde9d8e /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/17mer.log a4f92de7bd24d7ebbc02c40696a0f4e8 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/N50.xls 1c8c33470654e3f7993e48b4a6b4989a /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/Ostrea_lurida.fa
bgi_md5=() - This is an empty array called "bgi_md5".
$() - This is an empty command substitution. The stdout of commands within the parenthese are stored.
awk '/gz/{print $1}' md5_file - Awk looks for any lines from the input file (md5_file) with "gz" in them. If a line contains "gz", awk prints the first field (i.e. the first column).
Summary - The output from each result printed by awk is saved in an auto-incrementing fashion in the array called "bgi_md5".
count=$(()) - A variable called "count". This is a combination of empty command substitution and bash arithmeetic. Double parentheses are required for bash arithmetic.
${#bgi_md5[@]} - 1 - This prints the number of indeces in the array called "bgi_md5" and subtracts 1 from that number. Subtraction of one is necessary because bash is a zero-based language (e.g. the array starts at index 0).
Summary - The length of the array minus one is saved the the variable called "count".
((i=0;i<=$count;++i)) - Sets variable "i" to 0. Then, the loop evaluates whether or not the value of "i" is than/equal to the value in the variable "count". If that condition is met, the loop increases the value stored in "i" by 1 and continues through the loop.
printf "%s\n" "${bgi_md5[$i]}" - Prints the value at the array index designated by the value currently stored in "i" (the printing is specified by the "%s", which means string). This is followed by printing a new line (\n).
Summary - This prints the value at each position within the array and uses printf to improve legibility of output.
%%bash
bgi_md5=($(awk '/gz/{print $1}' /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/md5.txt))
my_md5=($(awk '/gz/{print $1}' /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/checksums.md5))
count=$(( ${#bgi_md5[@]} - 1 ))
for ((i=0;i<=$count;++i))
do
printf "%s\n" "${bgi_md5[$i]}"
printf "%s\n\n" "${my_md5[$i]}"
done
810d188468dbd8bb36b2af3bf3b9fee6 410cfcdf170125f4d8cb1ac4baf0007c cf92b18e0815dc0471d61f9107142257 cf92b18e0815dc0471d61f9107142257 3dc2137d7df0af8d6a007516908361a3 3dc2137d7df0af8d6a007516908361a3 8bc0d7c7a7af3954baca31a4a7fe9f2b 8bc0d7c7a7af3954baca31a4a7fe9f2b 08cfdc6fdc5a6190cb05cdcb81fa5b9c 08cfdc6fdc5a6190cb05cdcb81fa5b9c a503043167457337a65d51151ceb5dd0 a503043167457337a65d51151ceb5dd0 b371a3b3588060bb2200f5caaf9a9d5c a0c0177a7a4a4ca28c37bd3802361564 dd8b0fef21fc5e330d08ae4a48c8d67b 74e51d608de7a409a29545a95ac3ec14 50b5dbff9426738005b81efd49d66329 7438fcf14797976de9288368236ed75d 29be6e734ad65c180aee23d1514c5c35 26dd6bc17c1596e2881924bcc167125a a0c0177a7a4a4ca28c37bd3802361564 9b5f7c1593f216f710814c299738493f 74e51d608de7a409a29545a95ac3ec14 5a6392d9c23aa85d170d7f50de4f0b54 9b5f7c1593f216f710814c299738493f 4221fa24ccbb1735202fcfaa80d69d95 5a6392d9c23aa85d170d7f50de4f0b54 0adbb85da2610131b728417669e0922c 7438fcf14797976de9288368236ed75d b371a3b3588060bb2200f5caaf9a9d5c 26dd6bc17c1596e2881924bcc167125a dd8b0fef21fc5e330d08ae4a48c8d67b 4221fa24ccbb1735202fcfaa80d69d95 50b5dbff9426738005b81efd49d66329 0adbb85da2610131b728417669e0922c 29be6e734ad65c180aee23d1514c5c35
${bgi_filename[$i]##*/} - Like before, this prints the value of the array at each index specified by the value stored in "i". It also uses parameter substitution for substring removal. The ##*/ matches the longest pattern before, and including the last slash, and deletes that pattern. This effectively removes the full path details and leaves us with just the filename.
%%bash
bgi_md5=($(awk '/gz/{print $1}' /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/md5.txt))
bgi_filename=($(awk '/gz/{print $2}' /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/md5.txt))
my_md5=($(awk '/gz/{print $1}' /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/checksums.md5))
my_filename=($(awk '/gz/{print $2}' /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/checksums.md5))
count=$(( ${#bgi_md5[@]} - 1 ))
for ((i=0;i<=$count;++i))
do
printf "%s %s\n" "${bgi_filename[$i]##*/}" "${bgi_md5[$i]}"
printf "%s %s\n\n" "${my_filename[$i]##*/}" "${my_md5[$i]}"
done
151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz.clean.dup.clean.gz 810d188468dbd8bb36b2af3bf3b9fee6 151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz.clean.dup.clean.gz 410cfcdf170125f4d8cb1ac4baf0007c 151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz.clean.dup.clean.gz cf92b18e0815dc0471d61f9107142257 151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz.clean.dup.clean.gz cf92b18e0815dc0471d61f9107142257 151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz.clean.dup.clean.gz 3dc2137d7df0af8d6a007516908361a3 151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz.clean.dup.clean.gz 3dc2137d7df0af8d6a007516908361a3 151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz.clean.dup.clean.gz 8bc0d7c7a7af3954baca31a4a7fe9f2b 151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz.clean.dup.clean.gz 8bc0d7c7a7af3954baca31a4a7fe9f2b 151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz.clean.dup.clean.gz 08cfdc6fdc5a6190cb05cdcb81fa5b9c 151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz.clean.dup.clean.gz 08cfdc6fdc5a6190cb05cdcb81fa5b9c 151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz.clean.dup.clean.gz a503043167457337a65d51151ceb5dd0 151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz.clean.dup.clean.gz a503043167457337a65d51151ceb5dd0 160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz b371a3b3588060bb2200f5caaf9a9d5c 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz a0c0177a7a4a4ca28c37bd3802361564 160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz dd8b0fef21fc5e330d08ae4a48c8d67b 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz 74e51d608de7a409a29545a95ac3ec14 160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz 50b5dbff9426738005b81efd49d66329 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz 7438fcf14797976de9288368236ed75d 160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz 29be6e734ad65c180aee23d1514c5c35 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz 26dd6bc17c1596e2881924bcc167125a 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz a0c0177a7a4a4ca28c37bd3802361564 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz 9b5f7c1593f216f710814c299738493f 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz 74e51d608de7a409a29545a95ac3ec14 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz 5a6392d9c23aa85d170d7f50de4f0b54 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz 9b5f7c1593f216f710814c299738493f 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz 4221fa24ccbb1735202fcfaa80d69d95 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz 5a6392d9c23aa85d170d7f50de4f0b54 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz 0adbb85da2610131b728417669e0922c 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz 7438fcf14797976de9288368236ed75d 160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz b371a3b3588060bb2200f5caaf9a9d5c 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz 26dd6bc17c1596e2881924bcc167125a 160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz dd8b0fef21fc5e330d08ae4a48c8d67b 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz 4221fa24ccbb1735202fcfaa80d69d95 160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz 50b5dbff9426738005b81efd49d66329 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz 0adbb85da2610131b728417669e0922c 160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz 29be6e734ad65c180aee23d1514c5c35
%%bash
awk '/gz/{print $2}' /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/md5.txt
clean_data/151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz.clean.dup.clean.gz clean_data/151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz.clean.dup.clean.gz clean_data/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz.clean.dup.clean.gz clean_data/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz.clean.dup.clean.gz clean_data/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz.clean.dup.clean.gz clean_data/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz
%%bash
awk '/gz/{print $2}' /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/checksums.md5
/owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz.clean.dup.clean.gz /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz.clean.dup.clean.gz /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz.clean.dup.clean.gz /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz.clean.dup.clean.gz /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz.clean.dup.clean.gz /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz.clean.dup.clean.gz /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz
%%bash
sort -k2 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/md5.txt | awk '/gz/{print $2}'
clean_data/151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz.clean.dup.clean.gz clean_data/151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz.clean.dup.clean.gz clean_data/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz.clean.dup.clean.gz clean_data/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz.clean.dup.clean.gz clean_data/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz.clean.dup.clean.gz clean_data/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz clean_data/160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz
%%bash
bgi_md5=($(sort -k2 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/md5.txt | awk '/gz/{print $1}'))
bgi_filename=($(sort -k2 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/md5.txt | awk '/gz/{print $2}'))
my_md5=($(awk '/gz/{print $1}' /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/checksums.md5))
my_filename=($(awk '/gz/{print $2}' /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/checksums.md5))
count=$(( ${#bgi_md5[@]} - 1 ))
for ((i=0;i<=$count;++i))
do
printf "%s %s\n" "${bgi_filename[$i]##*/}" "${bgi_md5[$i]}"
printf "%s %s\n\n" "${my_filename[$i]##*/}" "${my_md5[$i]}"
done
151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz.clean.dup.clean.gz 810d188468dbd8bb36b2af3bf3b9fee6 151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz.clean.dup.clean.gz 410cfcdf170125f4d8cb1ac4baf0007c 151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz.clean.dup.clean.gz cf92b18e0815dc0471d61f9107142257 151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz.clean.dup.clean.gz cf92b18e0815dc0471d61f9107142257 151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz.clean.dup.clean.gz 3dc2137d7df0af8d6a007516908361a3 151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz.clean.dup.clean.gz 3dc2137d7df0af8d6a007516908361a3 151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz.clean.dup.clean.gz 8bc0d7c7a7af3954baca31a4a7fe9f2b 151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz.clean.dup.clean.gz 8bc0d7c7a7af3954baca31a4a7fe9f2b 151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz.clean.dup.clean.gz 08cfdc6fdc5a6190cb05cdcb81fa5b9c 151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz.clean.dup.clean.gz 08cfdc6fdc5a6190cb05cdcb81fa5b9c 151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz.clean.dup.clean.gz a503043167457337a65d51151ceb5dd0 151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz.clean.dup.clean.gz a503043167457337a65d51151ceb5dd0 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz a0c0177a7a4a4ca28c37bd3802361564 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz a0c0177a7a4a4ca28c37bd3802361564 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz 74e51d608de7a409a29545a95ac3ec14 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz 74e51d608de7a409a29545a95ac3ec14 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz 7438fcf14797976de9288368236ed75d 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz 7438fcf14797976de9288368236ed75d 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz 26dd6bc17c1596e2881924bcc167125a 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz 26dd6bc17c1596e2881924bcc167125a 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz 9b5f7c1593f216f710814c299738493f 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz 9b5f7c1593f216f710814c299738493f 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz 5a6392d9c23aa85d170d7f50de4f0b54 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz 5a6392d9c23aa85d170d7f50de4f0b54 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz 4221fa24ccbb1735202fcfaa80d69d95 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz 4221fa24ccbb1735202fcfaa80d69d95 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz 0adbb85da2610131b728417669e0922c 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz 0adbb85da2610131b728417669e0922c 160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz b371a3b3588060bb2200f5caaf9a9d5c 160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz b371a3b3588060bb2200f5caaf9a9d5c 160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz dd8b0fef21fc5e330d08ae4a48c8d67b 160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz dd8b0fef21fc5e330d08ae4a48c8d67b 160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz 50b5dbff9426738005b81efd49d66329 160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz 50b5dbff9426738005b81efd49d66329 160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz 29be6e734ad65c180aee23d1514c5c35 160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz 29be6e734ad65c180aee23d1514c5c35
wget command below utilizes two arguments:¶-q - This is for "quiet" ouput and will not print the thousands of lines of output that screwed up my notebook the first time around.
-O - Specifies the output filename. This is required to overwrite an existing file with the same name.
%%bash
time wget -q ftp://F15FTSUSAT0327:OSTibkD@cdts-hk.genomics.cn/Ostrea_lurida/clean_data/151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz.clean.dup.clean.gz \
-O /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz.clean.dup.clean.gz
real 193m14.075s user 0m0.500s sys 4m1.740s
%%bash
grep -v \
151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz.clean.dup.clean.gz /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/checksums.md5 > \
/owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/checksums.md5_temp; \
mv /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/checksums.md5_temp \
/owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/checksums.md5
%%bash
md5sum /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz.clean.dup.clean.gz \
>> /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/checksums.md5
%%bash
sort -k2 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/checksums.md5 > tmp && mv tmp /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/checksums.md5
cat /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/checksums.md5
810d188468dbd8bb36b2af3bf3b9fee6 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz.clean.dup.clean.gz cf92b18e0815dc0471d61f9107142257 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz.clean.dup.clean.gz 3dc2137d7df0af8d6a007516908361a3 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz.clean.dup.clean.gz 8bc0d7c7a7af3954baca31a4a7fe9f2b /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz.clean.dup.clean.gz 08cfdc6fdc5a6190cb05cdcb81fa5b9c /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz.clean.dup.clean.gz a503043167457337a65d51151ceb5dd0 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz.clean.dup.clean.gz a0c0177a7a4a4ca28c37bd3802361564 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz 74e51d608de7a409a29545a95ac3ec14 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz 7438fcf14797976de9288368236ed75d /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz 26dd6bc17c1596e2881924bcc167125a /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz 9b5f7c1593f216f710814c299738493f /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz 5a6392d9c23aa85d170d7f50de4f0b54 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz 4221fa24ccbb1735202fcfaa80d69d95 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz 0adbb85da2610131b728417669e0922c /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz b371a3b3588060bb2200f5caaf9a9d5c /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz dd8b0fef21fc5e330d08ae4a48c8d67b /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz 50b5dbff9426738005b81efd49d66329 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz 29be6e734ad65c180aee23d1514c5c35 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz
sort -k2 command to deal with the improperly sorted md5.txt file provided by BGI. This command sorts on the second column (-k2), which is the column that contains the filenames.¶%%bash
bgi_md5=($(sort -k2 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/md5.txt | awk '/gz/{print $1}'))
bgi_filename=($(sort -k2 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/md5.txt | awk '/gz/{print $2}'))
my_md5=($(awk '/gz/{print $1}' /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/checksums.md5))
my_filename=($(awk '/gz/{print $2}' /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/clean_data/checksums.md5))
count=$(( ${#bgi_md5[@]} - 1 ))
for ((i=0;i<=$count;++i))
do
printf "%s %s\n" "${bgi_filename[$i]##*/}" "${bgi_md5[$i]}"
printf "%s %s\n\n" "${my_filename[$i]##*/}" "${my_md5[$i]}"
done
151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz.clean.dup.clean.gz 810d188468dbd8bb36b2af3bf3b9fee6 151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_1.fq.gz.clean.dup.clean.gz 810d188468dbd8bb36b2af3bf3b9fee6 151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz.clean.dup.clean.gz cf92b18e0815dc0471d61f9107142257 151114_I191_FCH3Y35BCXX_L1_wHAIPI023992-37_2.fq.gz.clean.dup.clean.gz cf92b18e0815dc0471d61f9107142257 151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz.clean.dup.clean.gz 3dc2137d7df0af8d6a007516908361a3 151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_1.fq.gz.clean.dup.clean.gz 3dc2137d7df0af8d6a007516908361a3 151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz.clean.dup.clean.gz 8bc0d7c7a7af3954baca31a4a7fe9f2b 151114_I191_FCH3Y35BCXX_L2_wHAMPI023991-66_2.fq.gz.clean.dup.clean.gz 8bc0d7c7a7af3954baca31a4a7fe9f2b 151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz.clean.dup.clean.gz 08cfdc6fdc5a6190cb05cdcb81fa5b9c 151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_1.fq.gz.clean.dup.clean.gz 08cfdc6fdc5a6190cb05cdcb81fa5b9c 151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz.clean.dup.clean.gz a503043167457337a65d51151ceb5dd0 151118_I137_FCH3KNJBBXX_L5_wHAXPI023905-96_2.fq.gz.clean.dup.clean.gz a503043167457337a65d51151ceb5dd0 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz a0c0177a7a4a4ca28c37bd3802361564 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz a0c0177a7a4a4ca28c37bd3802361564 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz 74e51d608de7a409a29545a95ac3ec14 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz 74e51d608de7a409a29545a95ac3ec14 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz 7438fcf14797976de9288368236ed75d 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz 7438fcf14797976de9288368236ed75d 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz 26dd6bc17c1596e2881924bcc167125a 160103_I137_FCH3V5YBBXX_L3_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz 26dd6bc17c1596e2881924bcc167125a 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz 9b5f7c1593f216f710814c299738493f 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_1.fq.gz.clean.dup.clean.gz 9b5f7c1593f216f710814c299738493f 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz 5a6392d9c23aa85d170d7f50de4f0b54 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCABDLAAPEI-62_2.fq.gz.clean.dup.clean.gz 5a6392d9c23aa85d170d7f50de4f0b54 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz 4221fa24ccbb1735202fcfaa80d69d95 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_1.fq.gz.clean.dup.clean.gz 4221fa24ccbb1735202fcfaa80d69d95 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz 0adbb85da2610131b728417669e0922c 160103_I137_FCH3V5YBBXX_L4_WHOSTibkDCACDTAAPEI-75_2.fq.gz.clean.dup.clean.gz 0adbb85da2610131b728417669e0922c 160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz b371a3b3588060bb2200f5caaf9a9d5c 160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz b371a3b3588060bb2200f5caaf9a9d5c 160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz dd8b0fef21fc5e330d08ae4a48c8d67b 160103_I137_FCH3V5YBBXX_L5_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz dd8b0fef21fc5e330d08ae4a48c8d67b 160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz 50b5dbff9426738005b81efd49d66329 160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_1.fq.gz.clean.dup.clean.gz 50b5dbff9426738005b81efd49d66329 160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz 29be6e734ad65c180aee23d1514c5c35 160103_I137_FCH3V5YBBXX_L6_WHOSTibkDCAADWAAPEI-74_2.fq.gz.clean.dup.clean.gz 29be6e734ad65c180aee23d1514c5c35
%%bash
awk '/\.fa/{print $0}' /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/md5.txt
awk '/\.fa/{print $0}' /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/checksums.md5
f0a7772d4f1074698b50c913783c6fe2 Ostrea_lurida.fa 1c8c33470654e3f7993e48b4a6b4989a /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/Ostrea_lurida.fa
%%bash
time wget -q ftp://F15FTSUSAT0327:OSTibkD@cdts-hk.genomics.cn/Ostrea_lurida/Ostrea_lurida.fa \
-O /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/Ostrea_lurida.fa
real 22m20.294s user 0m0.050s sys 0m26.100s
%%bash
grep -v \
Ostrea_lurida.fa /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/checksums.md5 > \
tmp; \
mv tmp /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/checksums.md5
%%bash
md5sum /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/Ostrea_lurida.fa \
>> /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/checksums.md5
%%bash
awk '/\.fa/{print $0}' /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/md5.txt
awk '/\.fa/{print $0}' /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/checksums.md5
f0a7772d4f1074698b50c913783c6fe2 Ostrea_lurida.fa f0a7772d4f1074698b50c913783c6fe2 /owl_web/O_lurida_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Ostrea_lurida/Ostrea_lurida.fa
%%bash
cat /owl_web/P_generosa_genome_assemblies_BGI/rosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/md5.txt
4e95a487fd10e60ea5af7b62287f88a0 clean_data/151114_I191_FCH3Y35BCXX_L1_wHAIPI023989-79_1.fq.gz.clean.dup.clean.gz ce78b1856aa612473de14a9599d0c4ff clean_data/151114_I191_FCH3Y35BCXX_L1_wHAIPI023989-79_2.fq.gz.clean.dup.clean.gz a21f45349c1ebae6f8723d07613189cb clean_data/151114_I191_FCH3Y35BCXX_L2_wHAMPI023988-81_1.fq.gz.clean.dup.clean.gz 23fcb4cda2aba74e97644a8cfac46a59 clean_data/151114_I191_FCH3Y35BCXX_L2_wHAMPI023988-81_2.fq.gz.clean.dup.clean.gz 4b6225d3d0280fc9bf4161c6f86b5586 clean_data/151122_I136_FCH3L2FBBXX_L7_wHAXPI023990-97_1.fq.gz.clean.dup.clean.gz b9f80f96fda43362da9ee08aced56bef clean_data/151122_I136_FCH3L2FBBXX_L7_wHAXPI023990-97_2.fq.gz.clean.dup.clean.gz 109507129ac270d54159bce197ce0b13 clean_data/160103_I137_FCH3V5YBBXX_L3_WHPANwalDDAADWAAPEI-101_1.fq.gz.clean.dup.clean.gz 10acd92a3b2170367b09f45be30c7b4c clean_data/160103_I137_FCH3V5YBBXX_L3_WHPANwalDDAADWAAPEI-101_2.fq.gz.clean.dup.clean.gz 203c8428f6a787db0d49d43801fef0e3 clean_data/160103_I137_FCH3V5YBBXX_L4_WHPANwalDDAADWAAPEI-101_1.fq.gz.clean.dup.clean.gz b9edf4fe8ea20d2b2f044837c18c2d2f clean_data/160103_I137_FCH3V5YBBXX_L4_WHPANwalDDAADWAAPEI-101_2.fq.gz.clean.dup.clean.gz efcefa598dfc9230aa2f5b4d787388ee clean_data/160103_I137_FCH3V5YBBXX_L5_WHPANwalDDABDLAAPEI-100_1.fq.gz.clean.dup.clean.gz 0d3fabf04fc56e6249378fcddc286f06 clean_data/160103_I137_FCH3V5YBBXX_L5_WHPANwalDDABDLAAPEI-100_2.fq.gz.clean.dup.clean.gz 666f18e9cf0ba248e41bc089736932bb clean_data/160103_I137_FCH3V5YBBXX_L6_WHPANwalDDABDLAAPEI-100_1.fq.gz.clean.dup.clean.gz e73619c9bff1dbc816b6b17ca16518b1 clean_data/160103_I137_FCH3V5YBBXX_L6_WHPANwalDDABDLAAPEI-100_2.fq.gz.clean.dup.clean.gz ea41c8677502dc096a67dfabf279d556 clean_data/160103_I137_FCH3V5YBBXX_L5_WHPANwalDDACDTAAPEI-102_1.fq.gz.clean.dup.clean.gz 8cd443fbd790d71378cab4df87cac247 clean_data/160103_I137_FCH3V5YBBXX_L5_WHPANwalDDACDTAAPEI-102_2.fq.gz.clean.dup.clean.gz 712a5c2c6d7a5851101face85b234bb6 clean_data/160103_I137_FCH3V5YBBXX_L6_WHPANwalDDACDTAAPEI-102_1.fq.gz.clean.dup.clean.gz 97089726a31e080920b66caec4a7aee9 clean_data/160103_I137_FCH3V5YBBXX_L6_WHPANwalDDACDTAAPEI-102_2.fq.gz.clean.dup.clean.gz 873963ffcaffe6bf1ecc65bd34fedf2a 17mer.log 4df56d94c5357baf764658c5f53f7609 17mer.freq c01069a6d2a6a0e6bcce7daa1e339253 N50.xls 0348d8a1c5aea2c936ce47b5addcd857 Panopea_generosa.fa
%%bash
#For loop generates a md5 checksum has value for each file
#and appends the output to the checksums.md5 file.
time for file in /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/*.gz
do
md5sum "$file" >> /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/checksums.md5
done
real 41m43.942s user 0m9.400s sys 8m4.230s
%%bash
cat /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/checksums.md5
4e95a487fd10e60ea5af7b62287f88a0 /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/151114_I191_FCH3Y35BCXX_L1_wHAIPI023989-79_1.fq.gz.clean.dup.clean.gz ce78b1856aa612473de14a9599d0c4ff /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/151114_I191_FCH3Y35BCXX_L1_wHAIPI023989-79_2.fq.gz.clean.dup.clean.gz a21f45349c1ebae6f8723d07613189cb /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/151114_I191_FCH3Y35BCXX_L2_wHAMPI023988-81_1.fq.gz.clean.dup.clean.gz 23fcb4cda2aba74e97644a8cfac46a59 /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/151114_I191_FCH3Y35BCXX_L2_wHAMPI023988-81_2.fq.gz.clean.dup.clean.gz 4b6225d3d0280fc9bf4161c6f86b5586 /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/151122_I136_FCH3L2FBBXX_L7_wHAXPI023990-97_1.fq.gz.clean.dup.clean.gz b9f80f96fda43362da9ee08aced56bef /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/151122_I136_FCH3L2FBBXX_L7_wHAXPI023990-97_2.fq.gz.clean.dup.clean.gz 109507129ac270d54159bce197ce0b13 /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/160103_I137_FCH3V5YBBXX_L3_WHPANwalDDAADWAAPEI-101_1.fq.gz.clean.dup.clean.gz 10acd92a3b2170367b09f45be30c7b4c /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/160103_I137_FCH3V5YBBXX_L3_WHPANwalDDAADWAAPEI-101_2.fq.gz.clean.dup.clean.gz 203c8428f6a787db0d49d43801fef0e3 /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/160103_I137_FCH3V5YBBXX_L4_WHPANwalDDAADWAAPEI-101_1.fq.gz.clean.dup.clean.gz b9edf4fe8ea20d2b2f044837c18c2d2f /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/160103_I137_FCH3V5YBBXX_L4_WHPANwalDDAADWAAPEI-101_2.fq.gz.clean.dup.clean.gz efcefa598dfc9230aa2f5b4d787388ee /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/160103_I137_FCH3V5YBBXX_L5_WHPANwalDDABDLAAPEI-100_1.fq.gz.clean.dup.clean.gz 0d3fabf04fc56e6249378fcddc286f06 /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/160103_I137_FCH3V5YBBXX_L5_WHPANwalDDABDLAAPEI-100_2.fq.gz.clean.dup.clean.gz ea41c8677502dc096a67dfabf279d556 /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/160103_I137_FCH3V5YBBXX_L5_WHPANwalDDACDTAAPEI-102_1.fq.gz.clean.dup.clean.gz 8cd443fbd790d71378cab4df87cac247 /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/160103_I137_FCH3V5YBBXX_L5_WHPANwalDDACDTAAPEI-102_2.fq.gz.clean.dup.clean.gz 666f18e9cf0ba248e41bc089736932bb /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/160103_I137_FCH3V5YBBXX_L6_WHPANwalDDABDLAAPEI-100_1.fq.gz.clean.dup.clean.gz e73619c9bff1dbc816b6b17ca16518b1 /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/160103_I137_FCH3V5YBBXX_L6_WHPANwalDDABDLAAPEI-100_2.fq.gz.clean.dup.clean.gz 712a5c2c6d7a5851101face85b234bb6 /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/160103_I137_FCH3V5YBBXX_L6_WHPANwalDDACDTAAPEI-102_1.fq.gz.clean.dup.clean.gz 97089726a31e080920b66caec4a7aee9 /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/160103_I137_FCH3V5YBBXX_L6_WHPANwalDDACDTAAPEI-102_2.fq.gz.clean.dup.clean.gz
%%bash
#For loop generates a md5 checksum has value for each file
#and appends the output to the checksums.md5 file.
time for file in /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/*.[lfx]*
do
md5sum "$file" >> /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/checksums.md5
done
real 0m45.120s user 0m0.120s sys 0m8.630s
%%bash
cat /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/checksums.md5
4df56d94c5357baf764658c5f53f7609 /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/17mer.freq 873963ffcaffe6bf1ecc65bd34fedf2a /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/17mer.log c01069a6d2a6a0e6bcce7daa1e339253 /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/N50.xls 0348d8a1c5aea2c936ce47b5addcd857 /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/Panopea_generosa.fa
%%bash
bgi_md5=($(sort -k2 /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/md5.txt | awk '/gz/{print $1}'))
bgi_filename=($(sort -k2 /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/md5.txt | awk '/gz/{print $2}'))
my_md5=($(awk '/gz/{print $1}' /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/checksums.md5))
my_filename=($(awk '/gz/{print $2}' /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/clean_data/checksums.md5))
count=$(( ${#bgi_md5[@]} - 1 ))
for ((i=0;i<=$count;++i))
do
printf "%s %s\n" "${bgi_filename[$i]##*/}" "${bgi_md5[$i]}"
printf "%s %s\n\n" "${my_filename[$i]##*/}" "${my_md5[$i]}"
done
151114_I191_FCH3Y35BCXX_L1_wHAIPI023989-79_1.fq.gz.clean.dup.clean.gz 4e95a487fd10e60ea5af7b62287f88a0 151114_I191_FCH3Y35BCXX_L1_wHAIPI023989-79_1.fq.gz.clean.dup.clean.gz 4e95a487fd10e60ea5af7b62287f88a0 151114_I191_FCH3Y35BCXX_L1_wHAIPI023989-79_2.fq.gz.clean.dup.clean.gz ce78b1856aa612473de14a9599d0c4ff 151114_I191_FCH3Y35BCXX_L1_wHAIPI023989-79_2.fq.gz.clean.dup.clean.gz ce78b1856aa612473de14a9599d0c4ff 151114_I191_FCH3Y35BCXX_L2_wHAMPI023988-81_1.fq.gz.clean.dup.clean.gz a21f45349c1ebae6f8723d07613189cb 151114_I191_FCH3Y35BCXX_L2_wHAMPI023988-81_1.fq.gz.clean.dup.clean.gz a21f45349c1ebae6f8723d07613189cb 151114_I191_FCH3Y35BCXX_L2_wHAMPI023988-81_2.fq.gz.clean.dup.clean.gz 23fcb4cda2aba74e97644a8cfac46a59 151114_I191_FCH3Y35BCXX_L2_wHAMPI023988-81_2.fq.gz.clean.dup.clean.gz 23fcb4cda2aba74e97644a8cfac46a59 151122_I136_FCH3L2FBBXX_L7_wHAXPI023990-97_1.fq.gz.clean.dup.clean.gz 4b6225d3d0280fc9bf4161c6f86b5586 151122_I136_FCH3L2FBBXX_L7_wHAXPI023990-97_1.fq.gz.clean.dup.clean.gz 4b6225d3d0280fc9bf4161c6f86b5586 151122_I136_FCH3L2FBBXX_L7_wHAXPI023990-97_2.fq.gz.clean.dup.clean.gz b9f80f96fda43362da9ee08aced56bef 151122_I136_FCH3L2FBBXX_L7_wHAXPI023990-97_2.fq.gz.clean.dup.clean.gz b9f80f96fda43362da9ee08aced56bef 160103_I137_FCH3V5YBBXX_L3_WHPANwalDDAADWAAPEI-101_1.fq.gz.clean.dup.clean.gz 109507129ac270d54159bce197ce0b13 160103_I137_FCH3V5YBBXX_L3_WHPANwalDDAADWAAPEI-101_1.fq.gz.clean.dup.clean.gz 109507129ac270d54159bce197ce0b13 160103_I137_FCH3V5YBBXX_L3_WHPANwalDDAADWAAPEI-101_2.fq.gz.clean.dup.clean.gz 10acd92a3b2170367b09f45be30c7b4c 160103_I137_FCH3V5YBBXX_L3_WHPANwalDDAADWAAPEI-101_2.fq.gz.clean.dup.clean.gz 10acd92a3b2170367b09f45be30c7b4c 160103_I137_FCH3V5YBBXX_L4_WHPANwalDDAADWAAPEI-101_1.fq.gz.clean.dup.clean.gz 203c8428f6a787db0d49d43801fef0e3 160103_I137_FCH3V5YBBXX_L4_WHPANwalDDAADWAAPEI-101_1.fq.gz.clean.dup.clean.gz 203c8428f6a787db0d49d43801fef0e3 160103_I137_FCH3V5YBBXX_L4_WHPANwalDDAADWAAPEI-101_2.fq.gz.clean.dup.clean.gz b9edf4fe8ea20d2b2f044837c18c2d2f 160103_I137_FCH3V5YBBXX_L4_WHPANwalDDAADWAAPEI-101_2.fq.gz.clean.dup.clean.gz b9edf4fe8ea20d2b2f044837c18c2d2f 160103_I137_FCH3V5YBBXX_L5_WHPANwalDDABDLAAPEI-100_1.fq.gz.clean.dup.clean.gz efcefa598dfc9230aa2f5b4d787388ee 160103_I137_FCH3V5YBBXX_L5_WHPANwalDDABDLAAPEI-100_1.fq.gz.clean.dup.clean.gz efcefa598dfc9230aa2f5b4d787388ee 160103_I137_FCH3V5YBBXX_L5_WHPANwalDDABDLAAPEI-100_2.fq.gz.clean.dup.clean.gz 0d3fabf04fc56e6249378fcddc286f06 160103_I137_FCH3V5YBBXX_L5_WHPANwalDDABDLAAPEI-100_2.fq.gz.clean.dup.clean.gz 0d3fabf04fc56e6249378fcddc286f06 160103_I137_FCH3V5YBBXX_L5_WHPANwalDDACDTAAPEI-102_1.fq.gz.clean.dup.clean.gz ea41c8677502dc096a67dfabf279d556 160103_I137_FCH3V5YBBXX_L5_WHPANwalDDACDTAAPEI-102_1.fq.gz.clean.dup.clean.gz ea41c8677502dc096a67dfabf279d556 160103_I137_FCH3V5YBBXX_L5_WHPANwalDDACDTAAPEI-102_2.fq.gz.clean.dup.clean.gz 8cd443fbd790d71378cab4df87cac247 160103_I137_FCH3V5YBBXX_L5_WHPANwalDDACDTAAPEI-102_2.fq.gz.clean.dup.clean.gz 8cd443fbd790d71378cab4df87cac247 160103_I137_FCH3V5YBBXX_L6_WHPANwalDDABDLAAPEI-100_1.fq.gz.clean.dup.clean.gz 666f18e9cf0ba248e41bc089736932bb 160103_I137_FCH3V5YBBXX_L6_WHPANwalDDABDLAAPEI-100_1.fq.gz.clean.dup.clean.gz 666f18e9cf0ba248e41bc089736932bb 160103_I137_FCH3V5YBBXX_L6_WHPANwalDDABDLAAPEI-100_2.fq.gz.clean.dup.clean.gz e73619c9bff1dbc816b6b17ca16518b1 160103_I137_FCH3V5YBBXX_L6_WHPANwalDDABDLAAPEI-100_2.fq.gz.clean.dup.clean.gz e73619c9bff1dbc816b6b17ca16518b1 160103_I137_FCH3V5YBBXX_L6_WHPANwalDDACDTAAPEI-102_1.fq.gz.clean.dup.clean.gz 712a5c2c6d7a5851101face85b234bb6 160103_I137_FCH3V5YBBXX_L6_WHPANwalDDACDTAAPEI-102_1.fq.gz.clean.dup.clean.gz 712a5c2c6d7a5851101face85b234bb6 160103_I137_FCH3V5YBBXX_L6_WHPANwalDDACDTAAPEI-102_2.fq.gz.clean.dup.clean.gz 97089726a31e080920b66caec4a7aee9 160103_I137_FCH3V5YBBXX_L6_WHPANwalDDACDTAAPEI-102_2.fq.gz.clean.dup.clean.gz 97089726a31e080920b66caec4a7aee9
%%bash
awk '/\.fa/{print $0}' /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/md5.txt
awk '/\.fa/{print $0}' /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/checksums.md5
0348d8a1c5aea2c936ce47b5addcd857 Panopea_generosa.fa 0348d8a1c5aea2c936ce47b5addcd857 /owl_web/P_generosa_genome_assemblies_BGI/20161201/cdts-hk.genomics.cn/Panopea_generosa/Panopea_generosa.fa