%%bash
echo "TODAY'S DATE:"
date
echo "------------"
echo ""
#Display operating system info
lsb_release -a
echo ""
echo "------------"
echo "HOSTNAME: "; hostname
echo ""
echo "------------"
echo "Computer Specs:"
echo ""
lscpu
echo ""
echo "------------"
echo ""
echo "Memory Specs"
echo ""
free -mh
TODAY'S DATE:
Thu Oct 24 15:42:18 PDT 2019
------------
Distributor ID: Ubuntu
Description: Ubuntu 16.04.6 LTS
Release: 16.04
Codename: xenial
------------
HOSTNAME:
swoose
------------
Computer Specs:
Architecture: x86_64
CPU op-mode(s): 32-bit, 64-bit
Byte Order: Little Endian
CPU(s): 24
On-line CPU(s) list: 0-23
Thread(s) per core: 2
Core(s) per socket: 6
Socket(s): 2
NUMA node(s): 1
Vendor ID: GenuineIntel
CPU family: 6
Model: 44
Model name: Intel(R) Xeon(R) CPU X5670 @ 2.93GHz
Stepping: 2
CPU MHz: 2925.798
BogoMIPS: 5851.95
Virtualization: VT-x
L1d cache: 32K
L1i cache: 32K
L2 cache: 256K
L3 cache: 12288K
NUMA node0 CPU(s): 0-23
Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 popcnt aes lahf_lm epb ssbd ibrs ibpb stibp kaiser tpr_shadow vnmi flexpriority ept vpid dtherm ida arat flush_l1d
------------
Memory Specs
total used free shared buff/cache available
Mem: 70G 16G 494M 479M 53G 53G
Swap: 4.7G 1.7G 3.0G
No LSB modules are available.
%env variables are good for passing to bash cells
NOTE: Seems like paths need to be absolute. Had issues trying to use relative path to home directory (~).
wd="/home/sam/analyses/20191024_cbai_fastq_concatenation"
%env wd=/home/sam/analyses/20191024_cbai_fastq_concatenation
%env fastq_100219_dir=/home/sam/Downloads/FASTQ-100219/
%env fastq_102319_dir=/home/sam/Downloads/FASTQ-102319/
env: wd=/home/sam/analyses/20191024_cbai_fastq_concatenation env: fastq_100219_dir=/home/sam/Downloads/FASTQ-100219/ env: fastq_102319_dir=/home/sam/Downloads/FASTQ-102319/
%%bash
[ -d "${wd}" ] || mkdir --parents "${wd}"
cd {wd}
/home/sam/analyses/20191024_cbai_fastq_concatenation
%%bash
# Create array of original FastQ files
for file in "${fastq_100219_dir}"*.gz
do
fastq_100219_array+=($(echo "$file"))
echo "Adding ${file} to fastq_100219_array"
done
echo ""
# Create arrray of second FastQ files
for file in "${fastq_102319_dir}"*.gz
do
fastq_102319_array+=($(echo "$file"))
echo "Adding ${file} to fastq_102319_array"
done
echo ""
# Check line numbers of each FastQ file
for index in "${!fastq_100219_array[@]}"
do
# Strip leading path from filename
fastq_file=$(echo "${fastq_100219_array[index]##*/}")
echo "Now concatenating ${fastq_100219_array[index]} and ${fastq_102319_array[index]} to ${fastq_file}..."
{
cat "${fastq_100219_array[index]}"
cat "${fastq_102319_array[index]}"
} >> "${fastq_file}"
wc -l "${fastq_100219_array[index]}"
wc -l "${fastq_102319_array[index]}"
wc -l "${fastq_file}"
done
Adding /home/sam/Downloads/FASTQ-100219/329774_S1_L001_R1_001.fastq.gz to fastq_100219_array Adding /home/sam/Downloads/FASTQ-100219/329774_S1_L001_R2_001.fastq.gz to fastq_100219_array Adding /home/sam/Downloads/FASTQ-100219/329774_S1_L002_R1_001.fastq.gz to fastq_100219_array Adding /home/sam/Downloads/FASTQ-100219/329774_S1_L002_R2_001.fastq.gz to fastq_100219_array Adding /home/sam/Downloads/FASTQ-100219/329775_S2_L001_R1_001.fastq.gz to fastq_100219_array Adding /home/sam/Downloads/FASTQ-100219/329775_S2_L001_R2_001.fastq.gz to fastq_100219_array Adding /home/sam/Downloads/FASTQ-100219/329775_S2_L002_R1_001.fastq.gz to fastq_100219_array Adding /home/sam/Downloads/FASTQ-100219/329775_S2_L002_R2_001.fastq.gz to fastq_100219_array Adding /home/sam/Downloads/FASTQ-100219/329776_S3_L001_R1_001.fastq.gz to fastq_100219_array Adding /home/sam/Downloads/FASTQ-100219/329776_S3_L001_R2_001.fastq.gz to fastq_100219_array Adding /home/sam/Downloads/FASTQ-100219/329776_S3_L002_R1_001.fastq.gz to fastq_100219_array Adding /home/sam/Downloads/FASTQ-100219/329776_S3_L002_R2_001.fastq.gz to fastq_100219_array Adding /home/sam/Downloads/FASTQ-100219/329777_S4_L001_R1_001.fastq.gz to fastq_100219_array Adding /home/sam/Downloads/FASTQ-100219/329777_S4_L001_R2_001.fastq.gz to fastq_100219_array Adding /home/sam/Downloads/FASTQ-100219/329777_S4_L002_R1_001.fastq.gz to fastq_100219_array Adding /home/sam/Downloads/FASTQ-100219/329777_S4_L002_R2_001.fastq.gz to fastq_100219_array Adding /home/sam/Downloads/FASTQ-102319/329774_S1_L001_R1_001.fastq.gz to fastq_102319_array Adding /home/sam/Downloads/FASTQ-102319/329774_S1_L001_R2_001.fastq.gz to fastq_102319_array Adding /home/sam/Downloads/FASTQ-102319/329774_S1_L002_R1_001.fastq.gz to fastq_102319_array Adding /home/sam/Downloads/FASTQ-102319/329774_S1_L002_R2_001.fastq.gz to fastq_102319_array Adding /home/sam/Downloads/FASTQ-102319/329775_S2_L001_R1_001.fastq.gz to fastq_102319_array Adding /home/sam/Downloads/FASTQ-102319/329775_S2_L001_R2_001.fastq.gz to fastq_102319_array Adding /home/sam/Downloads/FASTQ-102319/329775_S2_L002_R1_001.fastq.gz to fastq_102319_array Adding /home/sam/Downloads/FASTQ-102319/329775_S2_L002_R2_001.fastq.gz to fastq_102319_array Adding /home/sam/Downloads/FASTQ-102319/329776_S3_L001_R1_001.fastq.gz to fastq_102319_array Adding /home/sam/Downloads/FASTQ-102319/329776_S3_L001_R2_001.fastq.gz to fastq_102319_array Adding /home/sam/Downloads/FASTQ-102319/329776_S3_L002_R1_001.fastq.gz to fastq_102319_array Adding /home/sam/Downloads/FASTQ-102319/329776_S3_L002_R2_001.fastq.gz to fastq_102319_array Adding /home/sam/Downloads/FASTQ-102319/329777_S4_L001_R1_001.fastq.gz to fastq_102319_array Adding /home/sam/Downloads/FASTQ-102319/329777_S4_L001_R2_001.fastq.gz to fastq_102319_array Adding /home/sam/Downloads/FASTQ-102319/329777_S4_L002_R1_001.fastq.gz to fastq_102319_array Adding /home/sam/Downloads/FASTQ-102319/329777_S4_L002_R2_001.fastq.gz to fastq_102319_array Now concatenating /home/sam/Downloads/FASTQ-100219/329774_S1_L001_R1_001.fastq.gz and /home/sam/Downloads/FASTQ-102319/329774_S1_L001_R1_001.fastq.gz to 329774_S1_L001_R1_001.fastq.gz... 2300543 /home/sam/Downloads/FASTQ-100219/329774_S1_L001_R1_001.fastq.gz 4282884 /home/sam/Downloads/FASTQ-102319/329774_S1_L001_R1_001.fastq.gz 6583427 329774_S1_L001_R1_001.fastq.gz Now concatenating /home/sam/Downloads/FASTQ-100219/329774_S1_L001_R2_001.fastq.gz and /home/sam/Downloads/FASTQ-102319/329774_S1_L001_R2_001.fastq.gz to 329774_S1_L001_R2_001.fastq.gz... 2257049 /home/sam/Downloads/FASTQ-100219/329774_S1_L001_R2_001.fastq.gz 4209231 /home/sam/Downloads/FASTQ-102319/329774_S1_L001_R2_001.fastq.gz 6466280 329774_S1_L001_R2_001.fastq.gz Now concatenating /home/sam/Downloads/FASTQ-100219/329774_S1_L002_R1_001.fastq.gz and /home/sam/Downloads/FASTQ-102319/329774_S1_L002_R1_001.fastq.gz to 329774_S1_L002_R1_001.fastq.gz... 2265690 /home/sam/Downloads/FASTQ-100219/329774_S1_L002_R1_001.fastq.gz 3483571 /home/sam/Downloads/FASTQ-102319/329774_S1_L002_R1_001.fastq.gz 5749261 329774_S1_L002_R1_001.fastq.gz Now concatenating /home/sam/Downloads/FASTQ-100219/329774_S1_L002_R2_001.fastq.gz and /home/sam/Downloads/FASTQ-102319/329774_S1_L002_R2_001.fastq.gz to 329774_S1_L002_R2_001.fastq.gz... 2227691 /home/sam/Downloads/FASTQ-100219/329774_S1_L002_R2_001.fastq.gz 3427731 /home/sam/Downloads/FASTQ-102319/329774_S1_L002_R2_001.fastq.gz 5655422 329774_S1_L002_R2_001.fastq.gz Now concatenating /home/sam/Downloads/FASTQ-100219/329775_S2_L001_R1_001.fastq.gz and /home/sam/Downloads/FASTQ-102319/329775_S2_L001_R1_001.fastq.gz to 329775_S2_L001_R1_001.fastq.gz... 1750105 /home/sam/Downloads/FASTQ-100219/329775_S2_L001_R1_001.fastq.gz 3270584 /home/sam/Downloads/FASTQ-102319/329775_S2_L001_R1_001.fastq.gz 5020689 329775_S2_L001_R1_001.fastq.gz Now concatenating /home/sam/Downloads/FASTQ-100219/329775_S2_L001_R2_001.fastq.gz and /home/sam/Downloads/FASTQ-102319/329775_S2_L001_R2_001.fastq.gz to 329775_S2_L001_R2_001.fastq.gz... 1773269 /home/sam/Downloads/FASTQ-100219/329775_S2_L001_R2_001.fastq.gz 3285295 /home/sam/Downloads/FASTQ-102319/329775_S2_L001_R2_001.fastq.gz 5058564 329775_S2_L001_R2_001.fastq.gz Now concatenating /home/sam/Downloads/FASTQ-100219/329775_S2_L002_R1_001.fastq.gz and /home/sam/Downloads/FASTQ-102319/329775_S2_L002_R1_001.fastq.gz to 329775_S2_L002_R1_001.fastq.gz... 1725318 /home/sam/Downloads/FASTQ-100219/329775_S2_L002_R1_001.fastq.gz 2672797 /home/sam/Downloads/FASTQ-102319/329775_S2_L002_R1_001.fastq.gz 4398115 329775_S2_L002_R1_001.fastq.gz Now concatenating /home/sam/Downloads/FASTQ-100219/329775_S2_L002_R2_001.fastq.gz and /home/sam/Downloads/FASTQ-102319/329775_S2_L002_R2_001.fastq.gz to 329775_S2_L002_R2_001.fastq.gz... 1757707 /home/sam/Downloads/FASTQ-100219/329775_S2_L002_R2_001.fastq.gz 2712314 /home/sam/Downloads/FASTQ-102319/329775_S2_L002_R2_001.fastq.gz 4470021 329775_S2_L002_R2_001.fastq.gz Now concatenating /home/sam/Downloads/FASTQ-100219/329776_S3_L001_R1_001.fastq.gz and /home/sam/Downloads/FASTQ-102319/329776_S3_L001_R1_001.fastq.gz to 329776_S3_L001_R1_001.fastq.gz... 2135788 /home/sam/Downloads/FASTQ-100219/329776_S3_L001_R1_001.fastq.gz 4277244 /home/sam/Downloads/FASTQ-102319/329776_S3_L001_R1_001.fastq.gz 6413032 329776_S3_L001_R1_001.fastq.gz Now concatenating /home/sam/Downloads/FASTQ-100219/329776_S3_L001_R2_001.fastq.gz and /home/sam/Downloads/FASTQ-102319/329776_S3_L001_R2_001.fastq.gz to 329776_S3_L001_R2_001.fastq.gz... 2103280 /home/sam/Downloads/FASTQ-100219/329776_S3_L001_R2_001.fastq.gz 4238439 /home/sam/Downloads/FASTQ-102319/329776_S3_L001_R2_001.fastq.gz 6341719 329776_S3_L001_R2_001.fastq.gz Now concatenating /home/sam/Downloads/FASTQ-100219/329776_S3_L002_R1_001.fastq.gz and /home/sam/Downloads/FASTQ-102319/329776_S3_L002_R1_001.fastq.gz to 329776_S3_L002_R1_001.fastq.gz... 2108386 /home/sam/Downloads/FASTQ-100219/329776_S3_L002_R1_001.fastq.gz 3455578 /home/sam/Downloads/FASTQ-102319/329776_S3_L002_R1_001.fastq.gz 5563964 329776_S3_L002_R1_001.fastq.gz Now concatenating /home/sam/Downloads/FASTQ-100219/329776_S3_L002_R2_001.fastq.gz and /home/sam/Downloads/FASTQ-102319/329776_S3_L002_R2_001.fastq.gz to 329776_S3_L002_R2_001.fastq.gz... 2081109 /home/sam/Downloads/FASTQ-100219/329776_S3_L002_R2_001.fastq.gz 3427550 /home/sam/Downloads/FASTQ-102319/329776_S3_L002_R2_001.fastq.gz 5508659 329776_S3_L002_R2_001.fastq.gz Now concatenating /home/sam/Downloads/FASTQ-100219/329777_S4_L001_R1_001.fastq.gz and /home/sam/Downloads/FASTQ-102319/329777_S4_L001_R1_001.fastq.gz to 329777_S4_L001_R1_001.fastq.gz... 2300923 /home/sam/Downloads/FASTQ-100219/329777_S4_L001_R1_001.fastq.gz 3170048 /home/sam/Downloads/FASTQ-102319/329777_S4_L001_R1_001.fastq.gz 5470971 329777_S4_L001_R1_001.fastq.gz Now concatenating /home/sam/Downloads/FASTQ-100219/329777_S4_L001_R2_001.fastq.gz and /home/sam/Downloads/FASTQ-102319/329777_S4_L001_R2_001.fastq.gz to 329777_S4_L001_R2_001.fastq.gz... 2298157 /home/sam/Downloads/FASTQ-100219/329777_S4_L001_R2_001.fastq.gz 3136767 /home/sam/Downloads/FASTQ-102319/329777_S4_L001_R2_001.fastq.gz 5434924 329777_S4_L001_R2_001.fastq.gz Now concatenating /home/sam/Downloads/FASTQ-100219/329777_S4_L002_R1_001.fastq.gz and /home/sam/Downloads/FASTQ-102319/329777_S4_L002_R1_001.fastq.gz to 329777_S4_L002_R1_001.fastq.gz... 2262007 /home/sam/Downloads/FASTQ-100219/329777_S4_L002_R1_001.fastq.gz 2621485 /home/sam/Downloads/FASTQ-102319/329777_S4_L002_R1_001.fastq.gz 4883492 329777_S4_L002_R1_001.fastq.gz Now concatenating /home/sam/Downloads/FASTQ-100219/329777_S4_L002_R2_001.fastq.gz and /home/sam/Downloads/FASTQ-102319/329777_S4_L002_R2_001.fastq.gz to 329777_S4_L002_R2_001.fastq.gz... 2274328 /home/sam/Downloads/FASTQ-100219/329777_S4_L002_R2_001.fastq.gz 2619848 /home/sam/Downloads/FASTQ-102319/329777_S4_L002_R2_001.fastq.gz 4894176 329777_S4_L002_R2_001.fastq.gz
%%bash
rsync \
--archive \
--verbose \
*.gz \
owl:/volume1/web/nightingales/C_bairdi/
sending incremental file list 329774_S1_L001_R1_001.fastq.gz 329774_S1_L001_R2_001.fastq.gz 329774_S1_L002_R1_001.fastq.gz 329774_S1_L002_R2_001.fastq.gz 329775_S2_L001_R1_001.fastq.gz 329775_S2_L001_R2_001.fastq.gz 329775_S2_L002_R1_001.fastq.gz 329775_S2_L002_R2_001.fastq.gz 329776_S3_L001_R1_001.fastq.gz 329776_S3_L001_R2_001.fastq.gz 329776_S3_L002_R1_001.fastq.gz 329776_S3_L002_R2_001.fastq.gz 329777_S4_L001_R1_001.fastq.gz 329777_S4_L001_R2_001.fastq.gz 329777_S4_L002_R1_001.fastq.gz 329777_S4_L002_R2_001.fastq.gz sent 19,533,901,406 bytes received 323 bytes 28,412,947.97 bytes/sec total size is 19,529,132,417 speedup is 1.00