%%bash
echo "TODAY'S DATE:"
date
echo "------------"
echo ""
#Display operating system info
lsb_release -a
echo ""
echo "------------"
echo "HOSTNAME: "; hostname
echo ""
echo "------------"
echo "Computer Specs:"
echo ""
lscpu
echo ""
echo "------------"
echo ""
echo "Memory Specs"
echo ""
free -mh
TODAY'S DATE:
Fri Jan 31 08:14:52 PST 2020
------------
Distributor ID: Ubuntu
Description: Ubuntu 16.04.6 LTS
Release: 16.04
Codename: xenial
------------
HOSTNAME:
swoose
------------
Computer Specs:
Architecture: x86_64
CPU op-mode(s): 32-bit, 64-bit
Byte Order: Little Endian
CPU(s): 24
On-line CPU(s) list: 0-23
Thread(s) per core: 2
Core(s) per socket: 6
Socket(s): 2
NUMA node(s): 1
Vendor ID: GenuineIntel
CPU family: 6
Model: 44
Model name: Intel(R) Xeon(R) CPU X5670 @ 2.93GHz
Stepping: 2
CPU MHz: 2925.815
BogoMIPS: 5851.59
Virtualization: VT-x
L1d cache: 32K
L1i cache: 32K
L2 cache: 256K
L3 cache: 12288K
NUMA node0 CPU(s): 0-23
Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 popcnt aes lahf_lm epb ssbd ibrs ibpb stibp kaiser tpr_shadow vnmi flexpriority ept vpid dtherm ida arat flush_l1d
------------
Memory Specs
total used free shared buff/cache available
Mem: 70G 4.6G 60G 515M 5.8G 65G
Swap: 4.7G 0B 4.7G
No LSB modules are available.
# Set data directories
%env crab_data=/home/sam/data/C_bairdi/RNAseq
%env hemat_data=/home/sam/data/Hematodinium/RNAseq
%env wd=/home/sam/analyses
# Programs
%env seqtk=/home/sam/programs/seqtk-1.3/seqtk
env: crab_data=/home/sam/data/C_bairdi/RNAseq env: hemat_data=/home/sam/data/Hematodinium/RNAseq env: wd=/home/sam/analyses env: seqtk=/home/sam/programs/seqtk-1.3/seqtk
FastAs: https://gannet.fish.washington.edu/Atumefaciens/20200114_cbai_MEGAN_read_extractions/
Trimmed-FastQs: https://gannet.fish.washington.edu/Atumefaciens/20191218_cbai_fastp_RNAseq_trimming/
Use FastA IDs from MEGAN6 taxonomic read extraction FastAs to pull out appropriate reads from each phylum (Arthropoda and Alveolata). This is performed because MEGAN6 strips paired read ID after the first space. As such, the resulting read extractions using MEGAN end up with a FastA file containing two reads with identicial headers. Not sure if this will cause any downstream issues (i.e. with Trinity) where paired end data is used, so playing it safe and using the truncated IDs to pull FastQs with complete sequence headers for use in subsequent data wrangling.
%%bash
timestamp=$(date +%Y%m%d)
# Create associative arrays
## Infection status
declare -A inf_status_array=( [329774]=infected [329775]=uninfected [329776]=infected [329777]=uninfected )
## Sampling day
declare -A sample_day_array=( [329774]=D12 [329775]=D12 [329776]=D26 [329777]=D26 )
for directory in ${crab_data} ${hemat_data}
do
# Get species name
species=$(echo "${directory}" | awk -F"/" '{print $5}')
# Make new directory and change to that directory ("$_" means use previous command's argument)
mkdir --parents "${wd}"/"${timestamp}"."${species}"_megan_reads \
&& cd "$_" || exit
# Set seqtk list filename
seqtk_list=${timestamp}.${species}.seqtk.read_id.list
# Set output FastQ filenames
prefix=${timestamp}.${species}
R1_suffix=megan_R1.fq
R2_suffix=megan_R2.fq
######################################################
# Create FastA IDs list to use for sequence extraction
######################################################
for fasta in "${directory}"/*.fasta
do
echo "Pulling FastA IDs from ${fasta}"
echo ""
grep ">" "${fasta}" | awk 'sub(/^>/, "")'
done | sort -u >> "${seqtk_list}"
echo ""
echo "Finished with FastA ID extraction."
echo ""
echo "Moving on to read extractions..."
echo ""
echo ""
######################################################
# Extract corresponding R1 and R2 reads using seqtk FastA ID list
######################################################
for fastq in "${directory}"/*R1*.gz
do
# Strip path from filename
fastq_nopath=${fastq##*/}
# Get sample ID from FastQ filename
sample=$(echo "${fastq_nopath}" | awk -F "_" '{print $1}')
# Ignore sample 304428 - it's a general pool of various sample types
if [ "${sample}" != "304428" ]
then
# Pull infection status and sample day from associative array
inf_status=${inf_status_array[$sample]}
sample_day=${sample_day_array[$sample]}
# Set output filename
R1_out="${prefix}.${sample}.${sample_day}.${inf_status}.${R1_suffix}"
echo "Extracting R1 reads from ${fastq}."
echo ""
echo "Writing R1 reads to ${R1_out}"
echo ""
echo ""
# Use seqtk to pull out desired FastQ reads
${seqtk} subseq "${fastq}" "${seqtk_list}" >> "${R1_out}"
fi
done
echo ""
echo "Done with R1 read extractions"
echo ""
echo "------------"
echo ""
for fastq in "${directory}"/*R2*.gz
do
# Strip path from filename
fastq_nopath=${fastq##*/}
# Get sample ID from FastQ filename
sample=$(echo "${fastq_nopath}" | awk -F "_" '{print $1}')
# Ignore sample 304428 - it's a general pool of various sample types
if [ "${sample}" != "304428" ]
then
# Pull infection status and sample day from associative array
inf_status=${inf_status_array[$sample]}
sample_day=${sample_day_array[$sample]}
# Set output filename
R2_out="${prefix}.${sample}.${sample_day}.${inf_status}.${R2_suffix}"
echo "Extracting R2 reads from ${fastq}."
echo ""
echo "Writing R2 reads to ${R2_out}"
echo ""
echo ""
${seqtk} subseq "${fastq}" "${seqtk_list}" >> "${R2_out}"
fi
done
echo "-------------------------------------"
echo ""
# Print working directory and list files
pwd
ls -ltrh
echo ""
echo "-------------------------------------"
echo ""
done
Finished with FastA ID extraction. Moving on to read extractions... Extracting R1 reads from /home/sam/data/C_bairdi/RNAseq/329774_S1_L001_R1_001.fastp-trim.201912184855.fq.gz. Writing R1 reads to 20200131.C_bairdi.329774.D12.infected.megan_R1.fq Extracting R1 reads from /home/sam/data/C_bairdi/RNAseq/329774_S1_L002_R1_001.fastp-trim.201912185239.fq.gz. Writing R1 reads to 20200131.C_bairdi.329774.D12.infected.megan_R1.fq Extracting R1 reads from /home/sam/data/C_bairdi/RNAseq/329775_S2_L001_R1_001.fastp-trim.201912185554.fq.gz. Writing R1 reads to 20200131.C_bairdi.329775.D12.uninfected.megan_R1.fq Extracting R1 reads from /home/sam/data/C_bairdi/RNAseq/329775_S2_L002_R1_001.fastp-trim.201912185856.fq.gz. Writing R1 reads to 20200131.C_bairdi.329775.D12.uninfected.megan_R1.fq Extracting R1 reads from /home/sam/data/C_bairdi/RNAseq/329776_S3_L001_R1_001.fastp-trim.201912180142.fq.gz. Writing R1 reads to 20200131.C_bairdi.329776.D26.infected.megan_R1.fq Extracting R1 reads from /home/sam/data/C_bairdi/RNAseq/329776_S3_L002_R1_001.fastp-trim.201912180540.fq.gz. Writing R1 reads to 20200131.C_bairdi.329776.D26.infected.megan_R1.fq Extracting R1 reads from /home/sam/data/C_bairdi/RNAseq/329777_S4_L001_R1_001.fastp-trim.201912180900.fq.gz. Writing R1 reads to 20200131.C_bairdi.329777.D26.uninfected.megan_R1.fq Extracting R1 reads from /home/sam/data/C_bairdi/RNAseq/329777_S4_L002_R1_001.fastp-trim.201912181245.fq.gz. Writing R1 reads to 20200131.C_bairdi.329777.D26.uninfected.megan_R1.fq Done with R1 read extractions ------------ Extracting R2 reads from /home/sam/data/C_bairdi/RNAseq/329774_S1_L001_R2_001.fastp-trim.201912184855.fq.gz. Writing R2 reads to 20200131.C_bairdi.329774.D12.infected.megan_R2.fq Extracting R2 reads from /home/sam/data/C_bairdi/RNAseq/329774_S1_L002_R2_001.fastp-trim.201912185239.fq.gz. Writing R2 reads to 20200131.C_bairdi.329774.D12.infected.megan_R2.fq Extracting R2 reads from /home/sam/data/C_bairdi/RNAseq/329775_S2_L001_R2_001.fastp-trim.201912185554.fq.gz. Writing R2 reads to 20200131.C_bairdi.329775.D12.uninfected.megan_R2.fq Extracting R2 reads from /home/sam/data/C_bairdi/RNAseq/329775_S2_L002_R2_001.fastp-trim.201912185856.fq.gz. Writing R2 reads to 20200131.C_bairdi.329775.D12.uninfected.megan_R2.fq Extracting R2 reads from /home/sam/data/C_bairdi/RNAseq/329776_S3_L001_R2_001.fastp-trim.201912180142.fq.gz. Writing R2 reads to 20200131.C_bairdi.329776.D26.infected.megan_R2.fq Extracting R2 reads from /home/sam/data/C_bairdi/RNAseq/329776_S3_L002_R2_001.fastp-trim.201912180540.fq.gz. Writing R2 reads to 20200131.C_bairdi.329776.D26.infected.megan_R2.fq Extracting R2 reads from /home/sam/data/C_bairdi/RNAseq/329777_S4_L001_R2_001.fastp-trim.201912180900.fq.gz. Writing R2 reads to 20200131.C_bairdi.329777.D26.uninfected.megan_R2.fq Extracting R2 reads from /home/sam/data/C_bairdi/RNAseq/329777_S4_L002_R2_001.fastp-trim.201912181245.fq.gz. Writing R2 reads to 20200131.C_bairdi.329777.D26.uninfected.megan_R2.fq ------------------------------------- /home/sam/analyses/20200131.C_bairdi_megan_reads total 15G -rw-rw-r-- 1 sam sam 1.3G Jan 31 11:35 20200131.C_bairdi.seqtk.read_id.list -rw-rw-r-- 1 sam sam 1.7G Jan 31 11:38 20200131.C_bairdi.329774.D12.infected.megan_R1.fq -rw-rw-r-- 1 sam sam 1.7G Jan 31 11:42 20200131.C_bairdi.329775.D12.uninfected.megan_R1.fq -rw-rw-r-- 1 sam sam 1.6G Jan 31 11:45 20200131.C_bairdi.329776.D26.infected.megan_R1.fq -rw-rw-r-- 1 sam sam 1.8G Jan 31 11:49 20200131.C_bairdi.329777.D26.uninfected.megan_R1.fq -rw-rw-r-- 1 sam sam 1.7G Jan 31 11:53 20200131.C_bairdi.329774.D12.infected.megan_R2.fq -rw-rw-r-- 1 sam sam 1.7G Jan 31 11:57 20200131.C_bairdi.329775.D12.uninfected.megan_R2.fq -rw-rw-r-- 1 sam sam 1.6G Jan 31 12:01 20200131.C_bairdi.329776.D26.infected.megan_R2.fq -rw-rw-r-- 1 sam sam 1.8G Jan 31 12:05 20200131.C_bairdi.329777.D26.uninfected.megan_R2.fq ------------------------------------- Finished with FastA ID extraction. Moving on to read extractions... Extracting R1 reads from /home/sam/data/Hematodinium/RNAseq/329774_S1_L001_R1_001.fastp-trim.201912184855.fq.gz. Writing R1 reads to 20200131.Hematodinium.329774.D12.infected.megan_R1.fq Extracting R1 reads from /home/sam/data/Hematodinium/RNAseq/329774_S1_L002_R1_001.fastp-trim.201912185239.fq.gz. Writing R1 reads to 20200131.Hematodinium.329774.D12.infected.megan_R1.fq Extracting R1 reads from /home/sam/data/Hematodinium/RNAseq/329775_S2_L001_R1_001.fastp-trim.201912185554.fq.gz. Writing R1 reads to 20200131.Hematodinium.329775.D12.uninfected.megan_R1.fq Extracting R1 reads from /home/sam/data/Hematodinium/RNAseq/329775_S2_L002_R1_001.fastp-trim.201912185856.fq.gz. Writing R1 reads to 20200131.Hematodinium.329775.D12.uninfected.megan_R1.fq Extracting R1 reads from /home/sam/data/Hematodinium/RNAseq/329776_S3_L001_R1_001.fastp-trim.201912180142.fq.gz. Writing R1 reads to 20200131.Hematodinium.329776.D26.infected.megan_R1.fq Extracting R1 reads from /home/sam/data/Hematodinium/RNAseq/329776_S3_L002_R1_001.fastp-trim.201912180540.fq.gz. Writing R1 reads to 20200131.Hematodinium.329776.D26.infected.megan_R1.fq Extracting R1 reads from /home/sam/data/Hematodinium/RNAseq/329777_S4_L001_R1_001.fastp-trim.201912180900.fq.gz. Writing R1 reads to 20200131.Hematodinium.329777.D26.uninfected.megan_R1.fq Extracting R1 reads from /home/sam/data/Hematodinium/RNAseq/329777_S4_L002_R1_001.fastp-trim.201912181245.fq.gz. Writing R1 reads to 20200131.Hematodinium.329777.D26.uninfected.megan_R1.fq Done with R1 read extractions ------------ Extracting R2 reads from /home/sam/data/Hematodinium/RNAseq/329774_S1_L001_R2_001.fastp-trim.201912184855.fq.gz. Writing R2 reads to 20200131.Hematodinium.329774.D12.infected.megan_R2.fq Extracting R2 reads from /home/sam/data/Hematodinium/RNAseq/329774_S1_L002_R2_001.fastp-trim.201912185239.fq.gz. Writing R2 reads to 20200131.Hematodinium.329774.D12.infected.megan_R2.fq Extracting R2 reads from /home/sam/data/Hematodinium/RNAseq/329775_S2_L001_R2_001.fastp-trim.201912185554.fq.gz. Writing R2 reads to 20200131.Hematodinium.329775.D12.uninfected.megan_R2.fq Extracting R2 reads from /home/sam/data/Hematodinium/RNAseq/329775_S2_L002_R2_001.fastp-trim.201912185856.fq.gz. Writing R2 reads to 20200131.Hematodinium.329775.D12.uninfected.megan_R2.fq Extracting R2 reads from /home/sam/data/Hematodinium/RNAseq/329776_S3_L001_R2_001.fastp-trim.201912180142.fq.gz. Writing R2 reads to 20200131.Hematodinium.329776.D26.infected.megan_R2.fq Extracting R2 reads from /home/sam/data/Hematodinium/RNAseq/329776_S3_L002_R2_001.fastp-trim.201912180540.fq.gz. Writing R2 reads to 20200131.Hematodinium.329776.D26.infected.megan_R2.fq Extracting R2 reads from /home/sam/data/Hematodinium/RNAseq/329777_S4_L001_R2_001.fastp-trim.201912180900.fq.gz. Writing R2 reads to 20200131.Hematodinium.329777.D26.uninfected.megan_R2.fq Extracting R2 reads from /home/sam/data/Hematodinium/RNAseq/329777_S4_L002_R2_001.fastp-trim.201912181245.fq.gz. Writing R2 reads to 20200131.Hematodinium.329777.D26.uninfected.megan_R2.fq ------------------------------------- /home/sam/analyses/20200131.Hematodinium_megan_reads total 777M -rw-rw-r-- 1 sam sam 64M Jan 31 12:06 20200131.Hematodinium.seqtk.read_id.list -rw-rw-r-- 1 sam sam 188M Jan 31 12:07 20200131.Hematodinium.329774.D12.infected.megan_R1.fq -rw-rw-r-- 1 sam sam 0 Jan 31 12:07 20200131.Hematodinium.329775.D12.uninfected.megan_R1.fq -rw-rw-r-- 1 sam sam 170M Jan 31 12:10 20200131.Hematodinium.329776.D26.infected.megan_R1.fq -rw-rw-r-- 1 sam sam 0 Jan 31 12:10 20200131.Hematodinium.329777.D26.uninfected.megan_R1.fq -rw-rw-r-- 1 sam sam 188M Jan 31 12:13 20200131.Hematodinium.329774.D12.infected.megan_R2.fq -rw-rw-r-- 1 sam sam 0 Jan 31 12:13 20200131.Hematodinium.329775.D12.uninfected.megan_R2.fq -rw-rw-r-- 1 sam sam 170M Jan 31 12:16 20200131.Hematodinium.329776.D26.infected.megan_R2.fq -rw-rw-r-- 1 sam sam 0 Jan 31 12:16 20200131.Hematodinium.329777.D26.uninfected.megan_R2.fq -------------------------------------