mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-21 05:45:37 +08:00
Apply black formatting.
Apply isort.
This commit is contained in:
Dannon Baker
@@ -26,47 +26,120 @@ def __main__():
|
||||
for line in info:
|
||||
fields = line.replace("\n", "").split("=")
|
||||
tmp_dict[fields[0]] = "=".join(fields[1:])
|
||||
if 'genome project id' in tmp_dict.keys():
|
||||
name = tmp_dict['genome project id']
|
||||
if 'build' in tmp_dict.keys():
|
||||
name = tmp_dict['build']
|
||||
if "genome project id" in tmp_dict.keys():
|
||||
name = tmp_dict["genome project id"]
|
||||
if "build" in tmp_dict.keys():
|
||||
name = tmp_dict["build"]
|
||||
if name not in organisms.keys():
|
||||
organisms[name] = {'chrs': {}, 'base_dir': this_base_dir}
|
||||
organisms[name] = {"chrs": {}, "base_dir": this_base_dir}
|
||||
for key in tmp_dict.keys():
|
||||
organisms[name][key] = tmp_dict[key]
|
||||
else:
|
||||
if tmp_dict['organism'] not in organisms.keys():
|
||||
organisms[tmp_dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
|
||||
organisms[tmp_dict['organism']]['chrs'][tmp_dict['chromosome']] = tmp_dict
|
||||
if tmp_dict["organism"] not in organisms.keys():
|
||||
organisms[tmp_dict["organism"]] = {"chrs": {}, "base_dir": this_base_dir}
|
||||
organisms[tmp_dict["organism"]]["chrs"][tmp_dict["chromosome"]] = tmp_dict
|
||||
for org in organisms:
|
||||
org = organisms[org]
|
||||
# if no gpi, then must be a ncbi chr which corresponds to a UCSC org, w/o matching UCSC designation
|
||||
try:
|
||||
build = org['genome project id']
|
||||
build = org["genome project id"]
|
||||
except KeyError:
|
||||
continue
|
||||
if 'build' in org:
|
||||
build = org['build']
|
||||
print("ORG\t{}\t{}\t{}\t{}\t{}\t{}\tUCSC".format(build, org['name'], org['kingdom'], org['group'], org['chromosomes'], org['info url']))
|
||||
if "build" in org:
|
||||
build = org["build"]
|
||||
print(
|
||||
"ORG\t{}\t{}\t{}\t{}\t{}\t{}\tUCSC".format(
|
||||
build, org["name"], org["kingdom"], org["group"], org["chromosomes"], org["info url"]
|
||||
)
|
||||
)
|
||||
else:
|
||||
print("ORG\t{}\t{}\t{}\t{}\t{}\t{}\tNone".format(build, org['name'], org['kingdom'], org['group'], org['chromosomes'], org['info url']))
|
||||
print(
|
||||
"ORG\t{}\t{}\t{}\t{}\t{}\t{}\tNone".format(
|
||||
build, org["name"], org["kingdom"], org["group"], org["chromosomes"], org["info url"]
|
||||
)
|
||||
)
|
||||
|
||||
for chr in org['chrs']:
|
||||
chr = org['chrs'][chr]
|
||||
print("CHR\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format(build, chr['chromosome'], chr['name'], chr['length'], chr['gi'], chr['gb'], "http://www.ncbi.nlm.nih.gov/entrez/viewer.fcgi?db=nucleotide&val=" + chr['refseq']))
|
||||
for feature in ['CDS', 'tRNA', 'rRNA']:
|
||||
print("DATA\t{}_{}_{}\t{}\t{}\t{}\t{}\t{}".format(build, chr['chromosome'], feature, build, chr['chromosome'], feature, "bed", os.path.join(org['base_dir'], "{}.{}.bed".format(chr['chromosome'], feature))))
|
||||
for chr in org["chrs"]:
|
||||
chr = org["chrs"][chr]
|
||||
print(
|
||||
"CHR\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format(
|
||||
build,
|
||||
chr["chromosome"],
|
||||
chr["name"],
|
||||
chr["length"],
|
||||
chr["gi"],
|
||||
chr["gb"],
|
||||
"http://www.ncbi.nlm.nih.gov/entrez/viewer.fcgi?db=nucleotide&val=" + chr["refseq"],
|
||||
)
|
||||
)
|
||||
for feature in ["CDS", "tRNA", "rRNA"]:
|
||||
print(
|
||||
"DATA\t{}_{}_{}\t{}\t{}\t{}\t{}\t{}".format(
|
||||
build,
|
||||
chr["chromosome"],
|
||||
feature,
|
||||
build,
|
||||
chr["chromosome"],
|
||||
feature,
|
||||
"bed",
|
||||
os.path.join(org["base_dir"], "{}.{}.bed".format(chr["chromosome"], feature)),
|
||||
)
|
||||
)
|
||||
# FASTA
|
||||
print("DATA\t{}_{}_{}\t{}\t{}\t{}\t{}\t{}".format(build, chr['chromosome'], "seq", build, chr['chromosome'], "sequence", "fasta", os.path.join(org['base_dir'], "%s.fna" % chr['chromosome'])))
|
||||
print(
|
||||
"DATA\t{}_{}_{}\t{}\t{}\t{}\t{}\t{}".format(
|
||||
build,
|
||||
chr["chromosome"],
|
||||
"seq",
|
||||
build,
|
||||
chr["chromosome"],
|
||||
"sequence",
|
||||
"fasta",
|
||||
os.path.join(org["base_dir"], "%s.fna" % chr["chromosome"]),
|
||||
)
|
||||
)
|
||||
# GeneMark
|
||||
if os.path.exists(os.path.join(org['base_dir'], "%s.GeneMark.bed" % chr['chromosome'])):
|
||||
print("DATA\t{}_{}_{}\t{}\t{}\t{}\t{}\t{}".format(build, chr['chromosome'], "GeneMark", build, chr['chromosome'], "GeneMark", "bed", os.path.join(org['base_dir'], "%s.GeneMark.bed" % chr['chromosome'])))
|
||||
if os.path.exists(os.path.join(org["base_dir"], "%s.GeneMark.bed" % chr["chromosome"])):
|
||||
print(
|
||||
"DATA\t{}_{}_{}\t{}\t{}\t{}\t{}\t{}".format(
|
||||
build,
|
||||
chr["chromosome"],
|
||||
"GeneMark",
|
||||
build,
|
||||
chr["chromosome"],
|
||||
"GeneMark",
|
||||
"bed",
|
||||
os.path.join(org["base_dir"], "%s.GeneMark.bed" % chr["chromosome"]),
|
||||
)
|
||||
)
|
||||
# GenMarkHMM
|
||||
if os.path.exists(os.path.join(org['base_dir'], "%s.GeneMarkHMM.bed" % chr['chromosome'])):
|
||||
print("DATA\t{}_{}_{}\t{}\t{}\t{}\t{}\t{}".format(build, chr['chromosome'], "GeneMarkHMM", build, chr['chromosome'], "GeneMarkHMM", "bed", os.path.join(org['base_dir'], "%s.GeneMarkHMM.bed" % chr['chromosome'])))
|
||||
if os.path.exists(os.path.join(org["base_dir"], "%s.GeneMarkHMM.bed" % chr["chromosome"])):
|
||||
print(
|
||||
"DATA\t{}_{}_{}\t{}\t{}\t{}\t{}\t{}".format(
|
||||
build,
|
||||
chr["chromosome"],
|
||||
"GeneMarkHMM",
|
||||
build,
|
||||
chr["chromosome"],
|
||||
"GeneMarkHMM",
|
||||
"bed",
|
||||
os.path.join(org["base_dir"], "%s.GeneMarkHMM.bed" % chr["chromosome"]),
|
||||
)
|
||||
)
|
||||
# Glimmer3
|
||||
if os.path.exists(os.path.join(org['base_dir'], "%s.Glimmer3.bed" % chr['chromosome'])):
|
||||
print("DATA\t{}_{}_{}\t{}\t{}\t{}\t{}\t{}".format(build, chr['chromosome'], "Glimmer3", build, chr['chromosome'], "Glimmer3", "bed", os.path.join(org['base_dir'], "%s.Glimmer3.bed" % chr['chromosome'])))
|
||||
if os.path.exists(os.path.join(org["base_dir"], "%s.Glimmer3.bed" % chr["chromosome"])):
|
||||
print(
|
||||
"DATA\t{}_{}_{}\t{}\t{}\t{}\t{}\t{}".format(
|
||||
build,
|
||||
chr["chromosome"],
|
||||
"Glimmer3",
|
||||
build,
|
||||
chr["chromosome"],
|
||||
"Glimmer3",
|
||||
"bed",
|
||||
os.path.join(org["base_dir"], "%s.Glimmer3.bed" % chr["chromosome"]),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -26,21 +26,21 @@ def __main__():
|
||||
for line in info:
|
||||
fields = line.replace("\n", "").split("=")
|
||||
tmp_dict[fields[0]] = "=".join(fields[1:])
|
||||
if 'genome project id' in tmp_dict.keys():
|
||||
name = tmp_dict['genome project id']
|
||||
if 'build' in tmp_dict.keys():
|
||||
name = tmp_dict['build']
|
||||
if "genome project id" in tmp_dict.keys():
|
||||
name = tmp_dict["genome project id"]
|
||||
if "build" in tmp_dict.keys():
|
||||
name = tmp_dict["build"]
|
||||
if name not in organisms.keys():
|
||||
organisms[name] = {'chrs': {}, 'base_dir': this_base_dir}
|
||||
organisms[name] = {"chrs": {}, "base_dir": this_base_dir}
|
||||
for key in tmp_dict.keys():
|
||||
organisms[name][key] = tmp_dict[key]
|
||||
else:
|
||||
if tmp_dict['organism'] not in organisms.keys():
|
||||
organisms[tmp_dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
|
||||
organisms[tmp_dict['organism']]['chrs'][tmp_dict['chromosome']] = tmp_dict
|
||||
if tmp_dict["organism"] not in organisms.keys():
|
||||
organisms[tmp_dict["organism"]] = {"chrs": {}, "base_dir": this_base_dir}
|
||||
organisms[tmp_dict["organism"]]["chrs"][tmp_dict["chromosome"]] = tmp_dict
|
||||
|
||||
for org_name, org in list(organisms.items()):
|
||||
if 'name' not in org:
|
||||
if "name" not in org:
|
||||
del organisms[org_name]
|
||||
|
||||
orgs = list(organisms.keys())
|
||||
@@ -48,7 +48,7 @@ def __main__():
|
||||
swap_test = False
|
||||
for i in range(0, len(orgs) - 1):
|
||||
for j in range(0, len(orgs) - i - 1):
|
||||
if organisms[orgs[j]]['name'] > organisms[orgs[j + 1]]['name']:
|
||||
if organisms[orgs[j]]["name"] > organisms[orgs[j + 1]]["name"]:
|
||||
orgs[j], orgs[j + 1] = orgs[j + 1], orgs[j]
|
||||
swap_test = True
|
||||
if swap_test is False:
|
||||
@@ -60,13 +60,13 @@ def __main__():
|
||||
at_ucsc = False
|
||||
# if no gpi, then must be a ncbi chr which corresponds to a UCSC org, w/o matching UCSC designation
|
||||
try:
|
||||
org['genome project id']
|
||||
org["genome project id"]
|
||||
except KeyError:
|
||||
continue
|
||||
if 'build' in org:
|
||||
if "build" in org:
|
||||
at_ucsc = True
|
||||
|
||||
out_str = "||" + org['name'] + "||" + org['kingdom'] + "||" + org['group'] + "||"
|
||||
out_str = "||" + org["name"] + "||" + org["kingdom"] + "||" + org["group"] + "||"
|
||||
if at_ucsc:
|
||||
out_str = out_str + "Yes"
|
||||
out_str = out_str + "||"
|
||||
|
||||
@@ -22,7 +22,7 @@ def __main__():
|
||||
|
||||
organisms = {}
|
||||
|
||||
loc_out = open(loc_out, 'wb')
|
||||
loc_out = open(loc_out, "wb")
|
||||
|
||||
for result in os.walk(base_dir):
|
||||
this_base_dir, sub_dirs, files = result
|
||||
@@ -35,29 +35,29 @@ def __main__():
|
||||
for line in info:
|
||||
fields = line.replace("\n", "").split("=")
|
||||
tmp_dict[fields[0]] = "=".join(fields[1:])
|
||||
if 'genome project id' in tmp_dict.keys():
|
||||
name = tmp_dict['genome project id']
|
||||
if 'build' in tmp_dict.keys():
|
||||
name = tmp_dict['build']
|
||||
if "genome project id" in tmp_dict.keys():
|
||||
name = tmp_dict["genome project id"]
|
||||
if "build" in tmp_dict.keys():
|
||||
name = tmp_dict["build"]
|
||||
if name not in organisms.keys():
|
||||
organisms[name] = {'chrs': {}, 'base_dir': this_base_dir}
|
||||
organisms[name] = {"chrs": {}, "base_dir": this_base_dir}
|
||||
for key in tmp_dict.keys():
|
||||
organisms[name][key] = tmp_dict[key]
|
||||
else:
|
||||
if tmp_dict['organism'] not in organisms.keys():
|
||||
organisms[tmp_dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
|
||||
organisms[tmp_dict['organism']]['chrs'][tmp_dict['chromosome']] = tmp_dict
|
||||
if tmp_dict["organism"] not in organisms.keys():
|
||||
organisms[tmp_dict["organism"]] = {"chrs": {}, "base_dir": this_base_dir}
|
||||
organisms[tmp_dict["organism"]]["chrs"][tmp_dict["chromosome"]] = tmp_dict
|
||||
|
||||
for org in organisms:
|
||||
org = organisms[org]
|
||||
try:
|
||||
build = org['genome project id']
|
||||
build = org["genome project id"]
|
||||
except KeyError:
|
||||
continue
|
||||
if 'build' in org:
|
||||
build = org['build']
|
||||
if "build" in org:
|
||||
build = org["build"]
|
||||
|
||||
seq_path = os.path.join(org['base_dir'], "seq")
|
||||
seq_path = os.path.join(org["base_dir"], "seq")
|
||||
|
||||
# create seq dir, if exists go to next org
|
||||
# TODO: add better checking, i.e. for updating
|
||||
@@ -71,11 +71,11 @@ def __main__():
|
||||
|
||||
# Print org info
|
||||
|
||||
for chr in org['chrs']:
|
||||
chr = org['chrs'][chr]
|
||||
for chr in org["chrs"]:
|
||||
chr = org["chrs"][chr]
|
||||
|
||||
fasta_file = os.path.join(org['base_dir'], "%s.fna" % chr['chromosome'])
|
||||
nib_out_file = os.path.join(seq_path, "%s.nib " % chr['chromosome'])
|
||||
fasta_file = os.path.join(org["base_dir"], "%s.fna" % chr["chromosome"])
|
||||
nib_out_file = os.path.join(seq_path, "%s.nib " % chr["chromosome"])
|
||||
# create nibs using faToNib binary
|
||||
# TODO: when bx supports writing nib, use it here instead
|
||||
command = f"faToNib {fasta_file} {nib_out_file}"
|
||||
|
||||
@@ -26,34 +26,34 @@ def __main__():
|
||||
for line in info:
|
||||
fields = line.replace("\n", "").split("=")
|
||||
tmp_dict[fields[0]] = "=".join(fields[1:])
|
||||
if 'genome project id' in tmp_dict.keys():
|
||||
name = tmp_dict['genome project id']
|
||||
if 'build' in tmp_dict.keys():
|
||||
name = tmp_dict['build']
|
||||
if "genome project id" in tmp_dict.keys():
|
||||
name = tmp_dict["genome project id"]
|
||||
if "build" in tmp_dict.keys():
|
||||
name = tmp_dict["build"]
|
||||
if name not in organisms.keys():
|
||||
organisms[name] = {'chrs': {}, 'base_dir': this_base_dir}
|
||||
organisms[name] = {"chrs": {}, "base_dir": this_base_dir}
|
||||
for key in tmp_dict.keys():
|
||||
organisms[name][key] = tmp_dict[key]
|
||||
else:
|
||||
if tmp_dict['organism'] not in organisms.keys():
|
||||
organisms[tmp_dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
|
||||
organisms[tmp_dict['organism']]['chrs'][tmp_dict['chromosome']] = tmp_dict
|
||||
if tmp_dict["organism"] not in organisms.keys():
|
||||
organisms[tmp_dict["organism"]] = {"chrs": {}, "base_dir": this_base_dir}
|
||||
organisms[tmp_dict["organism"]]["chrs"][tmp_dict["chromosome"]] = tmp_dict
|
||||
for org in organisms:
|
||||
org = organisms[org]
|
||||
# if no gpi, then must be a ncbi chr which corresponds to a UCSC org, w/o matching UCSC designation
|
||||
try:
|
||||
build = org['genome project id']
|
||||
build = org["genome project id"]
|
||||
except KeyError:
|
||||
continue
|
||||
|
||||
if 'build' in org:
|
||||
build = org['build']
|
||||
if "build" in org:
|
||||
build = org["build"]
|
||||
|
||||
chrs = []
|
||||
for chrom in org['chrs']:
|
||||
chrom = org['chrs'][chrom]
|
||||
chrs.append("{}={}".format(chrom['chromosome'], chrom['length']))
|
||||
print("{}\t{}\t{}".format(build, org['name'], ",".join(chrs)))
|
||||
for chrom in org["chrs"]:
|
||||
chrom = org["chrs"][chrom]
|
||||
chrs.append("{}={}".format(chrom["chromosome"], chrom["length"]))
|
||||
print("{}\t{}\t{}".format(build, org["name"], ",".join(chrs)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -12,6 +12,7 @@ from ftplib import FTP
|
||||
from urllib.request import urlretrieve
|
||||
|
||||
import requests
|
||||
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
except ImportError:
|
||||
@@ -21,44 +22,49 @@ from util import ( # noqa: I202
|
||||
get_bed_from_genbank,
|
||||
get_bed_from_GeneMark,
|
||||
get_bed_from_GeneMarkHMM,
|
||||
get_bed_from_glimmer3
|
||||
get_bed_from_glimmer3,
|
||||
)
|
||||
|
||||
assert sys.version_info[:2] >= (2, 6)
|
||||
|
||||
# this defines the types of ftp files we are interested in, and how to process/convert them to a form for our use
|
||||
desired_ftp_files = {'GeneMark': {'ext': 'GeneMark-2.5f', 'parser': 'process_GeneMark'},
|
||||
'GeneMarkHMM': {'ext': 'GeneMarkHMM-2.6m', 'parser': 'process_GeneMarkHMM'},
|
||||
'Glimmer3': {'ext': 'Glimmer3', 'parser': 'process_Glimmer3'},
|
||||
'fna': {'ext': 'fna', 'parser': 'process_FASTA'},
|
||||
'gbk': {'ext': 'gbk', 'parser': 'process_Genbank'}}
|
||||
desired_ftp_files = {
|
||||
"GeneMark": {"ext": "GeneMark-2.5f", "parser": "process_GeneMark"},
|
||||
"GeneMarkHMM": {"ext": "GeneMarkHMM-2.6m", "parser": "process_GeneMarkHMM"},
|
||||
"Glimmer3": {"ext": "Glimmer3", "parser": "process_Glimmer3"},
|
||||
"fna": {"ext": "fna", "parser": "process_FASTA"},
|
||||
"gbk": {"ext": "gbk", "parser": "process_Genbank"},
|
||||
}
|
||||
|
||||
|
||||
# number, name, chroms, kingdom, group, genbank, refseq, info_url, ftp_url
|
||||
def iter_genome_projects(url="http://www.ncbi.nlm.nih.gov/genomes/lproks.cgi?view=1", info_url_base="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=genomeprj&cmd=Retrieve&dopt=Overview&list_uids="):
|
||||
for row in BeautifulSoup(requests.get(url).text).findAll(name='tr', bgcolor=["#EEFFDD", "#E8E8DD"]):
|
||||
def iter_genome_projects(
|
||||
url="http://www.ncbi.nlm.nih.gov/genomes/lproks.cgi?view=1",
|
||||
info_url_base="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=genomeprj&cmd=Retrieve&dopt=Overview&list_uids=",
|
||||
):
|
||||
for row in BeautifulSoup(requests.get(url).text).findAll(name="tr", bgcolor=["#EEFFDD", "#E8E8DD"]):
|
||||
row = str(row).replace("\n", "").replace("\r", "")
|
||||
|
||||
fields = row.split("</td>")
|
||||
|
||||
org_num = fields[0].split("list_uids=")[-1].split("\"")[0]
|
||||
org_num = fields[0].split("list_uids=")[-1].split('"')[0]
|
||||
|
||||
name = fields[1].split("\">")[-1].split("<")[0]
|
||||
name = fields[1].split('">')[-1].split("<")[0]
|
||||
|
||||
kingdom = "archaea"
|
||||
if "<td class=\"bacteria\" align=\"center\">B" in fields[2]:
|
||||
if '<td class="bacteria" align="center">B' in fields[2]:
|
||||
kingdom = "bacteria"
|
||||
|
||||
group = fields[3].split(">")[-1]
|
||||
|
||||
info_url = f"{info_url_base}{org_num}"
|
||||
|
||||
org_genbank = fields[7].split("\">")[-1].split("<")[0].split(".")[0]
|
||||
org_refseq = fields[8].split("\">")[-1].split("<")[0].split(".")[0]
|
||||
org_genbank = fields[7].split('">')[-1].split("<")[0].split(".")[0]
|
||||
org_refseq = fields[8].split('">')[-1].split("<")[0].split(".")[0]
|
||||
|
||||
# seems some things donot have an ftp url, try and except it here:
|
||||
try:
|
||||
ftp_url = fields[22].split("href=\"")[1].split("\"")[0]
|
||||
ftp_url = fields[22].split('href="')[1].split('"')[0]
|
||||
except Exception:
|
||||
print("FAILED TO AQUIRE FTP ADDRESS:", org_num, info_url)
|
||||
ftp_url = None
|
||||
@@ -68,7 +74,9 @@ def iter_genome_projects(url="http://www.ncbi.nlm.nih.gov/genomes/lproks.cgi?vie
|
||||
yield org_num, name, chroms, kingdom, group, org_genbank, org_refseq, info_url, ftp_url
|
||||
|
||||
|
||||
def get_chroms_by_project_id(org_num, base_url="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=genomeprj&cmd=Retrieve&dopt=Overview&list_uids="):
|
||||
def get_chroms_by_project_id(
|
||||
org_num, base_url="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=genomeprj&cmd=Retrieve&dopt=Overview&list_uids="
|
||||
):
|
||||
html_count = 0
|
||||
html = None
|
||||
while html_count < 500 and html is None:
|
||||
@@ -114,7 +122,7 @@ def get_ftp_contents(ftp_url):
|
||||
|
||||
def scrape_ftp(ftp_contents, org_dir, org_num, refseq, ftp_url):
|
||||
for file_type, items in desired_ftp_files.items():
|
||||
ext = items['ext']
|
||||
ext = items["ext"]
|
||||
ftp_filename = f"{refseq}.{ext}"
|
||||
target_filename = os.path.join(org_dir, f"{refseq}.{ext}")
|
||||
if ftp_filename in ftp_contents:
|
||||
@@ -133,8 +141,8 @@ def scrape_ftp(ftp_contents, org_dir, org_num, refseq, ftp_url):
|
||||
return
|
||||
|
||||
# do special processing for each file type:
|
||||
if items['parser'] is not None:
|
||||
globals()[items['parser']](target_filename, org_num, refseq)
|
||||
if items["parser"] is not None:
|
||||
globals()[items["parser"]](target_filename, org_num, refseq)
|
||||
else:
|
||||
print("FTP filetype:", file_type, "not found for", org_num, refseq)
|
||||
# FTP Files have been Loaded
|
||||
@@ -142,7 +150,7 @@ def scrape_ftp(ftp_contents, org_dir, org_num, refseq, ftp_url):
|
||||
|
||||
def process_FASTA(filename, org_num, refseq):
|
||||
fasta = []
|
||||
fasta = [line.strip() for line in open(filename, 'rb').readlines()]
|
||||
fasta = [line.strip() for line in open(filename, "rb").readlines()]
|
||||
fasta_header = fasta.pop(0)[1:]
|
||||
fasta_header_split = fasta_header.split("|")
|
||||
chr_name = fasta_header_split.pop(-1).strip()
|
||||
@@ -150,14 +158,14 @@ def process_FASTA(filename, org_num, refseq):
|
||||
fasta = "".join(fasta)
|
||||
|
||||
# Create Chrom Info File:
|
||||
chrom_info_file = open(os.path.join(os.path.split(filename)[0], "%s.info" % refseq), 'wb+')
|
||||
chrom_info_file = open(os.path.join(os.path.split(filename)[0], "%s.info" % refseq), "wb+")
|
||||
chrom_info_file.write(f"chromosome={refseq}\nname={chr_name}\nlength={len(fasta)}\norganism={org_num}\n")
|
||||
try:
|
||||
chrom_info_file.write("gi=%s\n" % accesions['gi'])
|
||||
chrom_info_file.write("gi=%s\n" % accesions["gi"])
|
||||
except Exception:
|
||||
chrom_info_file.write("gi=None\n")
|
||||
try:
|
||||
chrom_info_file.write("gb=%s\n" % accesions['gb'])
|
||||
chrom_info_file.write("gb=%s\n" % accesions["gb"])
|
||||
except Exception:
|
||||
chrom_info_file.write("gb=None\n")
|
||||
try:
|
||||
@@ -169,10 +177,10 @@ def process_FASTA(filename, org_num, refseq):
|
||||
|
||||
def process_Genbank(filename, org_num, refseq):
|
||||
# extracts 'CDS', 'tRNA', 'rRNA' features from genbank file
|
||||
features = get_bed_from_genbank(filename, refseq, ['CDS', 'tRNA', 'rRNA'])
|
||||
features = get_bed_from_genbank(filename, refseq, ["CDS", "tRNA", "rRNA"])
|
||||
for feature, values in features.items():
|
||||
feature_file = open(os.path.join(os.path.split(filename)[0], f"{refseq}.{feature}.bed"), 'wb+')
|
||||
feature_file.write('\n'.join(values))
|
||||
feature_file = open(os.path.join(os.path.split(filename)[0], f"{refseq}.{feature}.bed"), "wb+")
|
||||
feature_file.write("\n".join(values))
|
||||
feature_file.close()
|
||||
print("Genbank extraction finished for chrom:", refseq, "file:", filename)
|
||||
|
||||
@@ -183,8 +191,8 @@ def process_Glimmer3(filename, org_num, refseq):
|
||||
except Exception as e:
|
||||
print("Converting Glimmer3 to bed FAILED! For chrom:", refseq, "file:", filename, e)
|
||||
glimmer3_bed = []
|
||||
glimmer3_bed_file = open(os.path.join(os.path.split(filename)[0], "%s.Glimmer3.bed" % refseq), 'wb+')
|
||||
glimmer3_bed_file.write('\n'.join(glimmer3_bed))
|
||||
glimmer3_bed_file = open(os.path.join(os.path.split(filename)[0], "%s.Glimmer3.bed" % refseq), "wb+")
|
||||
glimmer3_bed_file.write("\n".join(glimmer3_bed))
|
||||
glimmer3_bed_file.close()
|
||||
|
||||
|
||||
@@ -194,8 +202,8 @@ def process_GeneMarkHMM(filename, org_num, refseq):
|
||||
except Exception as e:
|
||||
print("Converting GeneMarkHMM to bed FAILED! For chrom:", refseq, "file:", filename, e)
|
||||
geneMarkHMM_bed = []
|
||||
geneMarkHMM_bed_bed_file = open(os.path.join(os.path.split(filename)[0], "%s.GeneMarkHMM.bed" % refseq), 'wb+')
|
||||
geneMarkHMM_bed_bed_file.write('\n'.join(geneMarkHMM_bed))
|
||||
geneMarkHMM_bed_bed_file = open(os.path.join(os.path.split(filename)[0], "%s.GeneMarkHMM.bed" % refseq), "wb+")
|
||||
geneMarkHMM_bed_bed_file.write("\n".join(geneMarkHMM_bed))
|
||||
geneMarkHMM_bed_bed_file.close()
|
||||
|
||||
|
||||
@@ -205,8 +213,8 @@ def process_GeneMark(filename, org_num, refseq):
|
||||
except Exception as e:
|
||||
print("Converting GeneMark to bed FAILED! For chrom:", refseq, "file:", filename, e)
|
||||
geneMark_bed = []
|
||||
geneMark_bed_bed_file = open(os.path.join(os.path.split(filename)[0], "%s.GeneMark.bed" % refseq), 'wb+')
|
||||
geneMark_bed_bed_file.write('\n'.join(geneMark_bed))
|
||||
geneMark_bed_bed_file = open(os.path.join(os.path.split(filename)[0], "%s.GeneMark.bed" % refseq), "wb+")
|
||||
geneMark_bed_bed_file.write("\n".join(geneMark_bed))
|
||||
geneMark_bed_bed_file.close()
|
||||
|
||||
|
||||
@@ -246,7 +254,7 @@ def __main__():
|
||||
print("Org:", org_num, "chrom:", refseq, "[", time.time() - start_time, "seconds elapsed. ]")
|
||||
|
||||
# Create org info file
|
||||
info_file = open(os.path.join(org_dir, "%s.info" % org_num), 'wb+')
|
||||
info_file = open(os.path.join(org_dir, "%s.info" % org_num), "wb+")
|
||||
info_file.write("genome project id=%s\n" % org_num)
|
||||
info_file.write("name=%s\n" % name)
|
||||
info_file.write("kingdom=%s\n" % kingdom)
|
||||
|
||||
@@ -30,15 +30,15 @@ def __main__():
|
||||
for line in info:
|
||||
fields = line.replace("\n", "").split("=")
|
||||
tmp_dict[fields[0]] = "=".join(fields[1:])
|
||||
if 'genome project id' in tmp_dict.keys():
|
||||
if tmp_dict['genome project id'] not in organisms.keys():
|
||||
organisms[tmp_dict['genome project id']] = {'chrs': {}, 'base_dir': this_base_dir}
|
||||
if "genome project id" in tmp_dict.keys():
|
||||
if tmp_dict["genome project id"] not in organisms.keys():
|
||||
organisms[tmp_dict["genome project id"]] = {"chrs": {}, "base_dir": this_base_dir}
|
||||
for key in tmp_dict.keys():
|
||||
organisms[tmp_dict['genome project id']][key] = tmp_dict[key]
|
||||
organisms[tmp_dict["genome project id"]][key] = tmp_dict[key]
|
||||
else:
|
||||
if tmp_dict['organism'] not in organisms.keys():
|
||||
organisms[tmp_dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
|
||||
organisms[tmp_dict['organism']]['chrs'][tmp_dict['chromosome']] = tmp_dict
|
||||
if tmp_dict["organism"] not in organisms.keys():
|
||||
organisms[tmp_dict["organism"]] = {"chrs": {}, "base_dir": this_base_dir}
|
||||
organisms[tmp_dict["organism"]]["chrs"][tmp_dict["chromosome"]] = tmp_dict
|
||||
|
||||
# get UCSC data
|
||||
|
||||
@@ -62,9 +62,16 @@ def __main__():
|
||||
builds = {}
|
||||
|
||||
for dsn in tree:
|
||||
build = dsn.find("SOURCE").attrib['id']
|
||||
build = dsn.find("SOURCE").attrib["id"]
|
||||
try:
|
||||
org_page = urlopen("http://archaea.ucsc.edu/cgi-bin/hgGateway?db=" + build).read().replace("\n", "").split("<table border=2 cellspacing=2 cellpadding=2>")[1].split("</table>")[0].split("</tr>")
|
||||
org_page = (
|
||||
urlopen("http://archaea.ucsc.edu/cgi-bin/hgGateway?db=" + build)
|
||||
.read()
|
||||
.replace("\n", "")
|
||||
.split("<table border=2 cellspacing=2 cellpadding=2>")[1]
|
||||
.split("</table>")[0]
|
||||
.split("</tr>")
|
||||
)
|
||||
except Exception:
|
||||
print("NO CHROMS FOR", build)
|
||||
continue
|
||||
@@ -76,36 +83,39 @@ def __main__():
|
||||
chr = row.split("</a>")[0].split(">")[-1]
|
||||
refseq = row.split("</a>")[-2].split(">")[-1]
|
||||
for org in organisms:
|
||||
for org_chr in organisms[org]['chrs']:
|
||||
if organisms[org]['chrs'][org_chr]['chromosome'] == refseq:
|
||||
for org_chr in organisms[org]["chrs"]:
|
||||
if organisms[org]["chrs"][org_chr]["chromosome"] == refseq:
|
||||
if org not in builds:
|
||||
builds[org] = {'chrs': {}, 'build': build}
|
||||
builds[org]['chrs'][refseq] = chr
|
||||
builds[org] = {"chrs": {}, "build": build}
|
||||
builds[org]["chrs"][refseq] = chr
|
||||
|
||||
print()
|
||||
ext_to_edit = ['bed', 'info', ]
|
||||
ext_to_edit = [
|
||||
"bed",
|
||||
"info",
|
||||
]
|
||||
for org in builds:
|
||||
print(org, "changed to", builds[org]['build'])
|
||||
print(org, "changed to", builds[org]["build"])
|
||||
|
||||
# org info file
|
||||
info_file_old = os.path.join(base_dir + org, org + ".info")
|
||||
info_file_new = os.path.join(base_dir + org, builds[org]['build'] + ".info")
|
||||
info_file_new = os.path.join(base_dir + org, builds[org]["build"] + ".info")
|
||||
|
||||
old_dir = base_dir + org
|
||||
new_dir = base_dir + builds[org]['build']
|
||||
new_dir = base_dir + builds[org]["build"]
|
||||
|
||||
# open and edit org info file
|
||||
info_file_contents = open(info_file_old).read()
|
||||
info_file_contents = info_file_contents + "build=" + builds[org]['build'] + "\n"
|
||||
for chrom in builds[org]['chrs']:
|
||||
info_file_contents = info_file_contents.replace(chrom, builds[org]['chrs'][chrom])
|
||||
info_file_contents = info_file_contents + "build=" + builds[org]["build"] + "\n"
|
||||
for chrom in builds[org]["chrs"]:
|
||||
info_file_contents = info_file_contents.replace(chrom, builds[org]["chrs"][chrom])
|
||||
for result in os.walk(base_dir + org):
|
||||
this_base_dir, sub_dirs, files = result
|
||||
for file in files:
|
||||
if file[0:len(chrom)] == chrom:
|
||||
if file[0 : len(chrom)] == chrom:
|
||||
# rename file
|
||||
old_name = os.path.join(this_base_dir, file)
|
||||
new_name = os.path.join(this_base_dir, builds[org]['chrs'][chrom] + file[len(chrom):])
|
||||
new_name = os.path.join(this_base_dir, builds[org]["chrs"][chrom] + file[len(chrom) :])
|
||||
move(old_name, new_name)
|
||||
|
||||
# edit contents of file, skiping those in list
|
||||
@@ -113,20 +123,22 @@ def __main__():
|
||||
continue
|
||||
|
||||
file_contents = open(new_name).read()
|
||||
file_contents = file_contents.replace(chrom, builds[org]['chrs'][chrom])
|
||||
file_contents = file_contents.replace(chrom, builds[org]["chrs"][chrom])
|
||||
|
||||
# special case fixes...
|
||||
if file[-5:] == ".info":
|
||||
file_contents = file_contents.replace("organism=" + org, "organism=" + builds[org]['build'])
|
||||
file_contents = file_contents.replace("refseq=" + builds[org]['chrs'][chrom], "refseq=" + chrom)
|
||||
file_contents = file_contents.replace("organism=" + org, "organism=" + builds[org]["build"])
|
||||
file_contents = file_contents.replace(
|
||||
"refseq=" + builds[org]["chrs"][chrom], "refseq=" + chrom
|
||||
)
|
||||
|
||||
# write out new file
|
||||
file_out = open(new_name, 'w')
|
||||
file_out = open(new_name, "w")
|
||||
file_out.write(file_contents)
|
||||
file_out.close()
|
||||
|
||||
# write out org info file and remove old file
|
||||
org_info_out = open(info_file_new, 'w')
|
||||
org_info_out = open(info_file_new, "w")
|
||||
org_info_out.write(info_file_contents)
|
||||
org_info_out.close()
|
||||
os.unlink(info_file_old)
|
||||
|
||||
+53
-20
@@ -12,21 +12,21 @@ class Region:
|
||||
self.qualifiers = {}
|
||||
self.start = None
|
||||
self.end = None
|
||||
self.strand = '+'
|
||||
self.strand = "+"
|
||||
|
||||
def set_coordinates_by_location(self, location):
|
||||
location = location.strip().lower().replace('..', ',')
|
||||
location = location.strip().lower().replace("..", ",")
|
||||
if "complement(" in location: # if part of the sequence is on the negative strand, it all is?
|
||||
self.strand = '-' # default of + strand
|
||||
self.strand = "-" # default of + strand
|
||||
for remove_text in ["join(", "order(", "complement(", ")"]:
|
||||
location = location.replace(remove_text, "")
|
||||
for number in location.split(','):
|
||||
number = number.strip('\n\r\t <>,()')
|
||||
for number in location.split(","):
|
||||
number = number.strip("\n\r\t <>,()")
|
||||
if number:
|
||||
if "^" in number:
|
||||
# a single point
|
||||
# check that this is correct for points, ie: 413/NC_005027.gbk: misc_feature 6636286^6636287 ===> 6636285,6636286
|
||||
end = int(number.split('^')[0])
|
||||
end = int(number.split("^")[0])
|
||||
start = end - 1
|
||||
else:
|
||||
end = int(number)
|
||||
@@ -39,6 +39,7 @@ class Region:
|
||||
|
||||
class GenBankFeatureParser:
|
||||
"""Parses Features from Single Locus GenBank file"""
|
||||
|
||||
def __init__(self, fh):
|
||||
self.fh = fh
|
||||
self.features = {}
|
||||
@@ -48,7 +49,7 @@ class GenBankFeatureParser:
|
||||
base_indent = 0
|
||||
last_attr_name = None
|
||||
for line in fh:
|
||||
if not in_features and line.startswith('FEATURES'):
|
||||
if not in_features and line.startswith("FEATURES"):
|
||||
in_features = True
|
||||
continue
|
||||
if in_features:
|
||||
@@ -71,8 +72,8 @@ class GenBankFeatureParser:
|
||||
else:
|
||||
# add info to last known feature
|
||||
line = line.strip()
|
||||
if line.startswith('/'):
|
||||
fields = line[1:].split('=', 1)
|
||||
if line.startswith("/"):
|
||||
fields = line[1:].split("=", 1)
|
||||
if len(fields) == 2:
|
||||
last_attr_name, content = fields
|
||||
else:
|
||||
@@ -88,10 +89,14 @@ class GenBankFeatureParser:
|
||||
self.features[last_feature_name][-1].set_coordinates_by_location(line)
|
||||
else:
|
||||
# continuation of multi-line qualifier content
|
||||
if last_feature_name.lower() in ['translation']:
|
||||
self.features[last_feature_name][-1].qualifiers[last_attr_name][-1] = "{}{}".format(self.features[last_feature_name][-1].qualifiers[last_attr_name][-1], line.rstrip('"'))
|
||||
if last_feature_name.lower() in ["translation"]:
|
||||
self.features[last_feature_name][-1].qualifiers[last_attr_name][-1] = "{}{}".format(
|
||||
self.features[last_feature_name][-1].qualifiers[last_attr_name][-1], line.rstrip('"')
|
||||
)
|
||||
else:
|
||||
self.features[last_feature_name][-1].qualifiers[last_attr_name][-1] = "{} {}".format(self.features[last_feature_name][-1].qualifiers[last_attr_name][-1], line.rstrip('"'))
|
||||
self.features[last_feature_name][-1].qualifiers[last_attr_name][-1] = "{} {}".format(
|
||||
self.features[last_feature_name][-1].qualifiers[last_attr_name][-1], line.rstrip('"')
|
||||
)
|
||||
|
||||
def get_features_by_type(self, feature_type):
|
||||
if feature_type not in self.features:
|
||||
@@ -108,7 +113,7 @@ def get_bed_from_genbank(gb_file, chrom, feature_list):
|
||||
features[feature_type] = []
|
||||
for feature in genbank_parser.get_features_by_type(feature_type):
|
||||
name = ""
|
||||
for name_tag in ['gene', 'locus_tag', 'db_xref']:
|
||||
for name_tag in ["gene", "locus_tag", "db_xref"]:
|
||||
if name_tag in feature.qualifiers:
|
||||
if name:
|
||||
name = name + ";"
|
||||
@@ -116,7 +121,9 @@ def get_bed_from_genbank(gb_file, chrom, feature_list):
|
||||
if not name:
|
||||
name = "unknown"
|
||||
|
||||
features[feature_type].append(f"{chrom}\t{feature.start}\t{feature.end}\t{name}\t{0}\t{feature.strand}") # append new bed field here
|
||||
features[feature_type].append(
|
||||
f"{chrom}\t{feature.start}\t{feature.end}\t{name}\t{0}\t{feature.strand}"
|
||||
) # append new bed field here
|
||||
return features
|
||||
|
||||
|
||||
@@ -136,7 +143,14 @@ def get_bed_from_GeneMark(geneMark_filename, chr):
|
||||
for block in orfs.split("\n\n"):
|
||||
if block.startswith("List of Regions of interest"):
|
||||
break
|
||||
best_block = {'start': 0, 'end': 0, 'strand': '+', 'avg_prob': -sys.maxsize, 'start_prob': -sys.maxsize, 'name': 'DNE'}
|
||||
best_block = {
|
||||
"start": 0,
|
||||
"end": 0,
|
||||
"strand": "+",
|
||||
"avg_prob": -sys.maxsize,
|
||||
"start_prob": -sys.maxsize,
|
||||
"name": "DNE",
|
||||
}
|
||||
ctr += 1
|
||||
ctr2 = 0
|
||||
for line in block.split("\n"):
|
||||
@@ -145,7 +159,7 @@ def get_bed_from_GeneMark(geneMark_filename, chr):
|
||||
start = int(fields.pop(0)) - 1
|
||||
end = int(fields.pop(0))
|
||||
strand = fields.pop(0)
|
||||
if strand == 'complement':
|
||||
if strand == "complement":
|
||||
strand = "-"
|
||||
else:
|
||||
strand = "+"
|
||||
@@ -157,9 +171,28 @@ def get_bed_from_GeneMark(geneMark_filename, chr):
|
||||
except Exception:
|
||||
start_prob = 0
|
||||
name = "orf_" + str(ctr) + "_" + str(ctr2)
|
||||
if avg_prob >= best_block['avg_prob'] and start_prob > best_block['start_prob']:
|
||||
best_block = {'start': start, 'end': end, 'strand': strand, 'avg_prob': avg_prob, 'start_prob': start_prob, 'name': name}
|
||||
regions.append(chr + "\t" + str(best_block['start']) + "\t" + str(best_block['end']) + "\t" + best_block['name'] + "\t" + str(int(best_block['avg_prob'] * 1000)) + "\t" + best_block['strand'])
|
||||
if avg_prob >= best_block["avg_prob"] and start_prob > best_block["start_prob"]:
|
||||
best_block = {
|
||||
"start": start,
|
||||
"end": end,
|
||||
"strand": strand,
|
||||
"avg_prob": avg_prob,
|
||||
"start_prob": start_prob,
|
||||
"name": name,
|
||||
}
|
||||
regions.append(
|
||||
chr
|
||||
+ "\t"
|
||||
+ str(best_block["start"])
|
||||
+ "\t"
|
||||
+ str(best_block["end"])
|
||||
+ "\t"
|
||||
+ best_block["name"]
|
||||
+ "\t"
|
||||
+ str(int(best_block["avg_prob"] * 1000))
|
||||
+ "\t"
|
||||
+ best_block["strand"]
|
||||
)
|
||||
return regions
|
||||
|
||||
|
||||
@@ -214,7 +247,7 @@ def get_bed_from_glimmer3(glimmer3_filename, chr):
|
||||
else:
|
||||
strand = "+"
|
||||
start = start - 1
|
||||
score = (float(fields.pop(0)))
|
||||
score = float(fields.pop(0))
|
||||
if score > max_score:
|
||||
max_score = score
|
||||
if score < min_score:
|
||||
|
||||
Reference in New Issue
Block a user