%%bash
echo "TODAY'S DATE:"
date
echo "------------"
echo ""
#Display operating system info
lsb_release -a
echo ""
echo "------------"
echo "HOSTNAME: "; hostname
echo ""
echo "------------"
echo "Computer Specs:"
echo ""
lscpu
echo ""
echo "------------"
echo ""
echo "Memory Specs"
echo ""
free -mh
TODAY'S DATE:
Wed Sep 4 15:18:48 PDT 2019
------------
Distributor ID: Ubuntu
Description: Ubuntu 16.04.6 LTS
Release: 16.04
Codename: xenial
------------
HOSTNAME:
swoose
------------
Computer Specs:
Architecture: x86_64
CPU op-mode(s): 32-bit, 64-bit
Byte Order: Little Endian
CPU(s): 24
On-line CPU(s) list: 0-23
Thread(s) per core: 2
Core(s) per socket: 6
Socket(s): 2
NUMA node(s): 1
Vendor ID: GenuineIntel
CPU family: 6
Model: 44
Model name: Intel(R) Xeon(R) CPU X5670 @ 2.93GHz
Stepping: 2
CPU MHz: 2925.993
BogoMIPS: 5851.93
Virtualization: VT-x
L1d cache: 32K
L1i cache: 32K
L2 cache: 256K
L3 cache: 12288K
NUMA node0 CPU(s): 0-23
Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 popcnt aes lahf_lm epb ssbd ibrs ibpb stibp pti tpr_shadow vnmi flexpriority ept vpid dtherm ida arat flush_l1d
------------
Memory Specs
total used free shared buff/cache available
Mem: 70G 10G 830M 629M 59G 58G
Swap: 4.7G 290M 4.4G
No LSB modules are available.
%env wd=/home/sam/analyses/20190904_pgen_v074.a3_genome_feature_counts
wd="/home/sam/analyses/20190904_pgen_v074.a3_genome_feature_counts"
%env rysnc_owl=owl:/volume1/web/halfshell/genomic-databank/
%env gffs=Panopea-generosa-vv0.74.a3.[Cegm]*.gff3
%env wget_gffs=--directory-prefix=${wd} --recursive --quiety --no-directories --no-check-certificate --no-parent --accept 'Panopea-generosa-vv0.74.a3.[Cegm]*.gff3' https://owl.fish.washington.edu/halfshell/genomic-databank/
env: wd=/home/sam/analyses/20190904_pgen_v074.a3_genome_feature_counts env: rysnc_owl=owl:/volume1/web/halfshell/genomic-databank/ env: gffs=Panopea-generosa-vv0.74.a3.[Cegm]*.gff3 env: wget_gffs=--directory-prefix=$/home/sam/analyses/20190904_pgen_v074.a3_genome_feature_counts --recursive --quiety --no-directories --no-check-certificate --no-parent --accept 'Panopea-generosa-vv0.74.a3.[Cegm]*.gff3' https://owl.fish.washington.edu/halfshell/genomic-databank/
import fnmatch
import os
import pandas
%%bash
mkdir --parents ${wd}
cd {wd}
/home/sam/analyses/20190904_pgen_v074.a3_genome_feature_counts
Info on GFFs is here: [https://github.com/RobertsLab/resources/wiki/Genomic-Resources#genome-feature-tracks-3)
%%bash
rsync \
--archive \
--verbose \
--progress \
--include="${gffs}" \
--exclude="*" \
"${rysnc_owl}" \
.
echo ""
echo ""
echo "----------------------------------------------------------"
ls -lh
receiving incremental file list
./
Panopea-generosa-vv0.74.a3.CDS.gff3
50,945,464 100% 2.09MB/s 0:00:23 (xfr#1, to-chk=3/5)
Panopea-generosa-vv0.74.a3.exon.gff3
52,378,458 100% 467.22kB/s 0:01:49 (xfr#2, to-chk=2/5)
Panopea-generosa-vv0.74.a3.gene.gff3
10,251,196 100% 429.17kB/s 0:00:23 (xfr#3, to-chk=1/5)
Panopea-generosa-vv0.74.a3.mRNA.gff3
12,732,694 100% 210.54kB/s 0:00:59 (xfr#4, to-chk=0/5)
sent 142 bytes received 126,323,558 bytes 514,556.82 bytes/sec
total size is 126,307,812 speedup is 1.00
----------------------------------------------------------
total 121M
-rw-rw-r-- 1 sam users 49M Sep 3 06:23 Panopea-generosa-vv0.74.a3.CDS.gff3
-rw-rw-r-- 1 sam users 50M Sep 3 06:23 Panopea-generosa-vv0.74.a3.exon.gff3
-rw-rw-r-- 1 sam users 9.8M Sep 3 06:23 Panopea-generosa-vv0.74.a3.gene.gff3
-rw-rw-r-- 1 sam users 13M Sep 3 06:23 Panopea-generosa-vv0.74.a3.mRNA.gff3
# %%bash
# time \
# wget "${wget_gffs}"
# ls -lh ${wd}
%%bash
head Panopea-generosa-vv0.74.a3.CDS.gff3
##gff-version 3 ##Generated using GenSAS, Tuesday 3rd of September 2019 06:14:34 AM ##Project Name : Pgenerosa_v074 PGA_scaffold1__77_contigs__length_89643857 GenSAS_5d6843e21278a-publish CDS 1 125 . + 2 ID=PGEN_.00g000010.m01.CDS01;Name=PGEN_.00g000010.m01.CDS01;Parent=PGEN_.00g000010.m01;original_ID=cds.19849-PGEN_.00g000010.m01;Alias=cds.19849-PGEN_.00g000010.m01 PGA_scaffold1__77_contigs__length_89643857 GenSAS_5d6843e21278a-publish CDS 1995 2095 . + 1 ID=PGEN_.00g000010.m01.CDS02;Name=PGEN_.00g000010.m01.CDS02;Parent=PGEN_.00g000010.m01;original_ID=cds.19849-PGEN_.00g000010.m01;Alias=cds.19849-PGEN_.00g000010.m01 PGA_scaffold1__77_contigs__length_89643857 GenSAS_5d6843e21278a-publish CDS 3325 3495 . + 0 ID=PGEN_.00g000010.m01.CDS03;Name=PGEN_.00g000010.m01.CDS03;Parent=PGEN_.00g000010.m01;original_ID=cds.19849-PGEN_.00g000010.m01;Alias=cds.19849-PGEN_.00g000010.m01 PGA_scaffold1__77_contigs__length_89643857 GenSAS_5d6843e21278a-publish CDS 14288 14485 . + 0 ID=PGEN_.00g000010.m01.CDS04;Name=PGEN_.00g000010.m01.CDS04;Parent=PGEN_.00g000010.m01;original_ID=cds.19849-PGEN_.00g000010.m01;Alias=cds.19849-PGEN_.00g000010.m01 PGA_scaffold1__77_contigs__length_89643857 GenSAS_5d6843e21278a-publish CDS 17116 17292 . + 0 ID=PGEN_.00g000010.m01.CDS05;Name=PGEN_.00g000010.m01.CDS05;Parent=PGEN_.00g000010.m01;original_ID=cds.19849-PGEN_.00g000010.m01;Alias=cds.19849-PGEN_.00g000010.m01 PGA_scaffold1__77_contigs__length_89643857 GenSAS_5d6843e21278a-publish CDS 19808 19943 . - 2 ID=PGEN_.00g000020.m01.CDS01;Name=PGEN_.00g000020.m01.CDS01;Parent=PGEN_.00g000020.m01;original_ID=cds.19849-PGEN_.00g000020.m01;Alias=cds.19849-PGEN_.00g000020.m01 PGA_scaffold1__77_contigs__length_89643857 GenSAS_5d6843e21278a-publish CDS 21133 21362 . - 0 ID=PGEN_.00g000020.m01.CDS02;Name=PGEN_.00g000020.m01.CDS02;Parent=PGEN_.00g000020.m01;original_ID=cds.19849-PGEN_.00g000020.m01;Alias=cds.19849-PGEN_.00g000020.m01
# Set list of column header names
gff_header = ['seqid','source','type','start','end','score','strand','phase','attributes']
for file in os.listdir('.'):
if fnmatch.fnmatch(file, 'Panopea-generosa-vv0.74.a3*.gff3'):
print('\n' * 2)
print(file)
print("-------------------------")
# Import GFF.
# Skip first row (gff header line) and indicate file is tab-separated
gff=pandas.read_csv(file, header=None, skiprows=3, sep="\t")
# Rename columns
gff.columns = gff_header
# Subtract start value from end value.
# Have to add 1 so that sequence length can't equal zero (i.e. adjust for 1-based counting system)
gff['seqlength'] = gff.apply(lambda position: position['end'] - position['start'] + 1, axis=1)
# Apply functions in list to seqlength column
gff_stats = gff['seqlength'].agg(['mean', 'min', 'median', 'max'])
print (gff_stats)
Panopea-generosa-vv0.74.a3.exon.gff3 ------------------------- mean 255.932825 min 3.000000 median 157.000000 max 13359.000000 Name: seqlength, dtype: float64 Panopea-generosa-vv0.74.a3.CDS.gff3 ------------------------- mean 255.932825 min 3.000000 median 157.000000 max 13359.000000 Name: seqlength, dtype: float64 Panopea-generosa-vv0.74.a3.mRNA.gff3 ------------------------- mean 13318.053183 min 201.000000 median 2346.000000 max 345225.000000 Name: seqlength, dtype: float64 Panopea-generosa-vv0.74.a3.gene.gff3 ------------------------- mean 13318.053183 min 201.000000 median 2346.000000 max 345225.000000 Name: seqlength, dtype: float64