diff --git a/pyproject.toml b/pyproject.toml index fe6c0bd55d1..6a22047f74b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,6 @@ include = '\.pyi?$' extend-exclude = ''' ^/( | packages - | tools )/ ''' force-exclude = 'lib/galaxy/util/jstree.py' diff --git a/tools/data_export/export_remote.py b/tools/data_export/export_remote.py index 8044b8bb861..c4d38b541d7 100644 --- a/tools/data_export/export_remote.py +++ b/tools/data_export/export_remote.py @@ -18,7 +18,7 @@ def check_for_duplicate_name(files_to_export): seen = set() duplicates = set() for entry in files_to_export: - name = entry['staging_path'] + name = entry["staging_path"] if name in seen: duplicates.add(name) seen.add(name) @@ -56,10 +56,10 @@ def main(argv=None): if write_if_not_exists(file_sources, target_uri, real_data_path): exit_code = 1 if export_metadata_files: - metadata_files = entry.get('metadata_files', []) + metadata_files = entry.get("metadata_files", []) for metadata_file in metadata_files: metadata_file_uri = f"{directory_uri}{metadata_file['staging_path']}" - if write_if_not_exists(file_sources, metadata_file_uri, metadata_file['source_path']): + if write_if_not_exists(file_sources, metadata_file_uri, metadata_file["source_path"]): exit_code = 1 counter += 1 print(f"{counter} out of {len(files_to_export)} files have been exported.\n") diff --git a/tools/data_export/send.py b/tools/data_export/send.py index 3662822f8b2..fe4ce034882 100644 --- a/tools/data_export/send.py +++ b/tools/data_export/send.py @@ -77,32 +77,53 @@ def send(provider, credentials, bucket, object_label, filename, overwrite_existi print("Finished successfully.") print("Job runtime:\t{}".format(time.time() - start_time)) - print("Transfer ET:\t{}\tSpeed:\t{}MB/sec".format( - time.time() - transfer_start_time, - round((os.path.getsize(filename) >> 20) / (time.time() - transfer_start_time), 3))) + print( + "Transfer ET:\t{}\tSpeed:\t{}MB/sec".format( + time.time() - transfer_start_time, + round((os.path.getsize(filename) >> 20) / (time.time() - transfer_start_time), 3), + ) + ) def parse_args(args): parser = argparse.ArgumentParser() - parser.add_argument('-p', '--provider', type=str, required=True, help="Provider") + parser.add_argument("-p", "--provider", type=str, required=True, help="Provider") - parser.add_argument('-b', '--bucket', type=str, required=True, - help="The cloud-based storage bucket in which data should be written.") + parser.add_argument( + "-b", + "--bucket", + type=str, + required=True, + help="The cloud-based storage bucket in which data should be written.", + ) - parser.add_argument('-o', '--object_label', type=str, required=True, - help="The label of the object created on the cloud-based storage for " - "the data to be persisted.") + parser.add_argument( + "-o", + "--object_label", + type=str, + required=True, + help="The label of the object created on the cloud-based storage for " "the data to be persisted.", + ) - parser.add_argument('-f', '--filename', type=str, required=True, - help="The (absolute) filename of the data to be persisted on the " - "cloud-based storage.") + parser.add_argument( + "-f", + "--filename", + type=str, + required=True, + help="The (absolute) filename of the data to be persisted on the " "cloud-based storage.", + ) - parser.add_argument('-w', '--overwrite_existing', type=str, required=True, - help="Sets if an object with the given `object_label` exists, this tool " - "should overwrite it (true) or append a time stamp to avoid " - "overwriting (false).") + parser.add_argument( + "-w", + "--overwrite_existing", + type=str, + required=True, + help="Sets if an object with the given `object_label` exists, this tool " + "should overwrite it (true) or append a time stamp to avoid " + "overwriting (false).", + ) - parser.add_argument('--credentials_file', type=str, required=True, help="Credentials file") + parser.add_argument("--credentials_file", type=str, required=True, help="Credentials file") return parser.parse_args(args) diff --git a/tools/data_source/data_source.py b/tools/data_source/data_source.py index a90504f2036..7208446cf59 100644 --- a/tools/data_source/data_source.py +++ b/tools/data_source/data_source.py @@ -21,9 +21,9 @@ from galaxy.util import ( get_charset_from_http_headers, ) -GALAXY_PARAM_PREFIX = 'GALAXY' +GALAXY_PARAM_PREFIX = "GALAXY" GALAXY_ROOT_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) -GALAXY_DATATYPES_CONF_FILE = os.path.join(GALAXY_ROOT_DIR, 'datatypes_conf.xml') +GALAXY_DATATYPES_CONF_FILE = os.path.join(GALAXY_ROOT_DIR, "datatypes_conf.xml") def stop_err(msg): @@ -35,18 +35,18 @@ def load_input_parameters(filename, erase_file=True): datasource_params = {} try: json_params = loads(open(filename).read()) - datasource_params = json_params.get('param_dict') + datasource_params = json_params.get("param_dict") except Exception: json_params = None for line in open(filename): try: line = line.strip() - fields = line.split('\t') + fields = line.split("\t") datasource_params[fields[0]] = fields[1] except Exception: continue if erase_file: - open(filename, 'w').close() # open file for writing, then close, removes params from file + open(filename, "w").close() # open file for writing, then close, removes params from file return json_params, datasource_params @@ -61,54 +61,68 @@ def __main__(): if job_params is None: # using an older tabular file enhanced_handling = False job_params = dict(param_dict=params) - job_params['output_data'] = [dict(out_data_name='output', - ext='data', - file_name=filename, - extra_files_path=None)] - job_params['job_config'] = dict(GALAXY_ROOT_DIR=GALAXY_ROOT_DIR, GALAXY_DATATYPES_CONF_FILE=GALAXY_DATATYPES_CONF_FILE, TOOL_PROVIDED_JOB_METADATA_FILE=TOOL_PROVIDED_JOB_METADATA_FILE) + job_params["output_data"] = [ + dict(out_data_name="output", ext="data", file_name=filename, extra_files_path=None) + ] + job_params["job_config"] = dict( + GALAXY_ROOT_DIR=GALAXY_ROOT_DIR, + GALAXY_DATATYPES_CONF_FILE=GALAXY_DATATYPES_CONF_FILE, + TOOL_PROVIDED_JOB_METADATA_FILE=TOOL_PROVIDED_JOB_METADATA_FILE, + ) else: enhanced_handling = True - json_file = open(job_params['job_config']['TOOL_PROVIDED_JOB_METADATA_FILE'], 'w') # specially named file for output junk to pass onto set metadata + json_file = open( + job_params["job_config"]["TOOL_PROVIDED_JOB_METADATA_FILE"], "w" + ) # specially named file for output junk to pass onto set metadata datatypes_registry = Registry() - datatypes_registry.load_datatypes(root_dir=job_params['job_config']['GALAXY_ROOT_DIR'], config=job_params['job_config']['GALAXY_DATATYPES_CONF_FILE']) + datatypes_registry.load_datatypes( + root_dir=job_params["job_config"]["GALAXY_ROOT_DIR"], + config=job_params["job_config"]["GALAXY_DATATYPES_CONF_FILE"], + ) - URL = params.get('URL', None) # using exactly URL indicates that only one dataset is being downloaded - URL_method = params.get('URL_method', None) + URL = params.get("URL", None) # using exactly URL indicates that only one dataset is being downloaded + URL_method = params.get("URL_method", None) - for data_dict in job_params['output_data']: - cur_filename = data_dict.get('file_name', filename) - cur_URL = params.get('%s|%s|URL' % (GALAXY_PARAM_PREFIX, data_dict['out_data_name']), URL) - if not cur_URL or urlparse(cur_URL).scheme not in ('http', 'https', 'ftp'): - open(cur_filename, 'w').write("") - stop_err('The remote data source application has not sent back a URL parameter in the request.') + for data_dict in job_params["output_data"]: + cur_filename = data_dict.get("file_name", filename) + cur_URL = params.get("%s|%s|URL" % (GALAXY_PARAM_PREFIX, data_dict["out_data_name"]), URL) + if not cur_URL or urlparse(cur_URL).scheme not in ("http", "https", "ftp"): + open(cur_filename, "w").write("") + stop_err("The remote data source application has not sent back a URL parameter in the request.") # The following calls to urlopen() will use the above default timeout try: - if not URL_method or URL_method == 'get': + if not URL_method or URL_method == "get": page = urlopen(cur_URL, timeout=DEFAULT_SOCKET_TIMEOUT) - elif URL_method == 'post': + elif URL_method == "post": page = urlopen(cur_URL, urlencode(params).encode("utf-8"), timeout=DEFAULT_SOCKET_TIMEOUT) except Exception as e: - stop_err('The remote data source application may be off line, please try again later. Error: %s' % str(e)) + stop_err("The remote data source application may be off line, please try again later. Error: %s" % str(e)) if max_file_size: - file_size = int(page.info().get('Content-Length', 0)) + file_size = int(page.info().get("Content-Length", 0)) if file_size > max_file_size: - stop_err('The size of the data (%d bytes) you have requested exceeds the maximum allowed (%d bytes) on this server.' % (file_size, max_file_size)) + stop_err( + "The size of the data (%d bytes) you have requested exceeds the maximum allowed (%d bytes) on this server." + % (file_size, max_file_size) + ) try: - cur_filename = sniff.stream_to_open_named_file(page, os.open(cur_filename, os.O_WRONLY | os.O_CREAT), cur_filename, source_encoding=get_charset_from_http_headers(page.headers)) + cur_filename = sniff.stream_to_open_named_file( + page, + os.open(cur_filename, os.O_WRONLY | os.O_CREAT), + cur_filename, + source_encoding=get_charset_from_http_headers(page.headers), + ) except Exception as e: - stop_err('Unable to fetch %s:\n%s' % (cur_URL, e)) + stop_err("Unable to fetch %s:\n%s" % (cur_URL, e)) # here import checks that upload tool performs if enhanced_handling: try: - ext = sniff.handle_uploaded_dataset_file(filename, datatypes_registry, ext=data_dict['ext']) + ext = sniff.handle_uploaded_dataset_file(filename, datatypes_registry, ext=data_dict["ext"]) except Exception as e: stop_err(str(e)) - info = dict(type='dataset', - dataset_id=data_dict['dataset_id'], - ext=ext) + info = dict(type="dataset", dataset_id=data_dict["dataset_id"], ext=ext) json_file.write("%s\n" % dumps(info)) diff --git a/tools/data_source/genbank.py b/tools/data_source/genbank.py index cd21103be7a..ac8e688a89c 100644 --- a/tools/data_source/genbank.py +++ b/tools/data_source/genbank.py @@ -10,21 +10,21 @@ assert sys.version_info[:2] >= (2, 6) def make_fasta(rec): - '''Creates fasta format from a record''' - gi = rec.annotations.get('gi', '') - org = rec.annotations.get('organism', '') - date = rec.annotations.get('date', '') - head = '>gi:%s, id:%s, org:%s, date:%s\n' % (gi, rec.id, org, date) - body = '\n'.join(textwrap.wrap(rec.seq.data, width=80)) + """Creates fasta format from a record""" + gi = rec.annotations.get("gi", "") + org = rec.annotations.get("organism", "") + date = rec.annotations.get("date", "") + head = ">gi:%s, id:%s, org:%s, date:%s\n" % (gi, rec.id, org, date) + body = "\n".join(textwrap.wrap(rec.seq.data, width=80)) return head, body -if __name__ == '__main__': +if __name__ == "__main__": mode = sys.argv[1] text = sys.argv[2] output_file = sys.argv[3] - print('Searching for %s
' % text) + print("Searching for %s
" % text) # check if inputs are all numbers try: @@ -33,12 +33,12 @@ if __name__ == '__main__': except ValueError: gi_list = GenBank.search_for(text, max_ids=10) - fp = open(output_file, 'wt') + fp = open(output_file, "wt") record_parser = GenBank.FeatureParser() - ncbi_dict = GenBank.NCBIDictionary(mode, 'genbank', parser=record_parser) + ncbi_dict = GenBank.NCBIDictionary(mode, "genbank", parser=record_parser) for gid in gi_list: res = ncbi_dict[gid] head, body = make_fasta(res) - fp.write(head + body + '\n') + fp.write(head + body + "\n") print(head) fp.close() diff --git a/tools/data_source/hbvar_filter.py b/tools/data_source/hbvar_filter.py index f4e6aeb3eae..1f241f141a4 100644 --- a/tools/data_source/hbvar_filter.py +++ b/tools/data_source/hbvar_filter.py @@ -10,10 +10,10 @@ from galaxy.util import DEFAULT_SOCKET_TIMEOUT def exec_before_job(app, inp_data, out_data, param_dict, tool=None): """Sets the name of the data""" - data_name = param_dict.get('name', 'HbVar query') - data_type = param_dict.get('type', 'txt') - if data_type == 'txt': - data_type = 'interval' # All data is TSV, assume interval + data_name = param_dict.get("name", "HbVar query") + data_type = param_dict.get("type", "txt") + if data_type == "txt": + data_type = "interval" # All data is TSV, assume interval name, data = next(iter(out_data.items())) data = app.datatypes_registry.change_datatype(data, data_type) data.name = data_name @@ -23,10 +23,10 @@ def exec_before_job(app, inp_data, out_data, param_dict, tool=None): def exec_after_process(app, inp_data, out_data, param_dict, tool=None, stdout=None, stderr=None): """Verifies the data after the run""" - URL = param_dict.get('URL', None) - URL = URL + '&_export=1&GALAXY_URL=0' + URL = param_dict.get("URL", None) + URL = URL + "&_export=1&GALAXY_URL=0" if not URL: - raise Exception('Datasource has not sent back a URL parameter') + raise Exception("Datasource has not sent back a URL parameter") CHUNK_SIZE = 2**20 # 1Mb MAX_SIZE = CHUNK_SIZE * 100 @@ -34,18 +34,18 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool=None, stdout=No try: page = urlopen(URL, timeout=DEFAULT_SOCKET_TIMEOUT) except Exception as exc: - raise Exception('Problems connecting to %s (%s)' % (URL, exc)) + raise Exception("Problems connecting to %s (%s)" % (URL, exc)) data = next(iter(out_data.values())) - fp = open(data.file_name, 'wb') + fp = open(data.file_name, "wb") size = 0 while 1: chunk = page.read(CHUNK_SIZE) if not chunk: break if size > MAX_SIZE: - raise Exception('----- maximum datasize exceeded ---') + raise Exception("----- maximum datasize exceeded ---") size += len(chunk) fp.write(chunk) @@ -56,10 +56,10 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool=None, stdout=No # check for missing meta data, if all there, comment first line and process file if not data.missing_meta(): line_ctr = -1 - temp = tempfile.NamedTemporaryFile('w') + temp = tempfile.NamedTemporaryFile("w") temp_filename = temp.name temp.close() - temp = open(temp_filename, 'w') + temp = open(temp_filename, "w") int(data.metadata.chromCol) int(data.metadata.startCol) int(data.metadata.strandCol) @@ -67,15 +67,15 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool=None, stdout=No for line in open(data.file_name): line_ctr += 1 - fields = line.strip().split('\t') + fields = line.strip().split("\t") - temp.write("%s\n" % '\t'.join(fields)) + temp.write("%s\n" % "\t".join(fields)) temp.close() shutil.move(temp_filename, data.file_name) else: - data = app.datatypes_registry.change_datatype(data, 'tabular') + data = app.datatypes_registry.change_datatype(data, "tabular") data.set_size() data.set_peek() app.model.context.add(data) diff --git a/tools/data_source/import.py b/tools/data_source/import.py index 4f89d4f68d0..0e7d1b4af33 100644 --- a/tools/data_source/import.py +++ b/tools/data_source/import.py @@ -17,40 +17,40 @@ out_name = sys.argv[2] id2name = { - 'eryth': 'ErythPreCRMmm3_cusTrk.txt', - 'cishg16': 'ReglRegHBBhg16CusTrk.txt', - 'cishg17': 'ReglRegHBBhg17CusTrk.txt', - 'exons': 'ExonsKnownGenes_mm3.txt', - 'krhg16': 'known_regulatory_hg16.bed', - 'krhg17': 'known_regulatory_hg17.bed', - 'tARhg16mmc': 'hg16.mouse.t_AR.cold.bed', - 'tARhg16mmm': 'hg16.mouse.t_AR.medium.bed', - 'tARhg16mmh': 'hg16.mouse.t_AR.hot.bed', - 'tARhg16rnc': 'hg16.rat.t_AR.cold.bed', - 'tARhg16rnm': 'hg16.rat.t_AR.medium.bed', - 'tARhg16rnh': 'hg16.rat.t_AR.hot.bed', - 'phastConsHg16': 'phastConsMost_hg16.bed', - 'omimhg16': 'omimDisorders_hg16.tab', - 'omimhg17': 'omimDisorders_hg17.tab', + "eryth": "ErythPreCRMmm3_cusTrk.txt", + "cishg16": "ReglRegHBBhg16CusTrk.txt", + "cishg17": "ReglRegHBBhg17CusTrk.txt", + "exons": "ExonsKnownGenes_mm3.txt", + "krhg16": "known_regulatory_hg16.bed", + "krhg17": "known_regulatory_hg17.bed", + "tARhg16mmc": "hg16.mouse.t_AR.cold.bed", + "tARhg16mmm": "hg16.mouse.t_AR.medium.bed", + "tARhg16mmh": "hg16.mouse.t_AR.hot.bed", + "tARhg16rnc": "hg16.rat.t_AR.cold.bed", + "tARhg16rnm": "hg16.rat.t_AR.medium.bed", + "tARhg16rnh": "hg16.rat.t_AR.hot.bed", + "phastConsHg16": "phastConsMost_hg16.bed", + "omimhg16": "omimDisorders_hg16.tab", + "omimhg17": "omimDisorders_hg17.tab", } -fname = id2name.get(dataid, '') +fname = id2name.get(dataid, "") if not fname: - print('Importing invalid data %s' % dataid) + print("Importing invalid data %s" % dataid) sys.exit() else: - print('Imported %s' % fname) + print("Imported %s" % fname) # this path is hardcoded -inp_name = os.path.join('database', 'import', fname) +inp_name = os.path.join("database", "import", fname) try: inp = open(inp_name) except Exception: - print('Could not find file %s' % inp_name) + print("Could not find file %s" % inp_name) sys.exit() -out = open(out_name, 'wt') +out = open(out_name, "wt") while 1: data = inp.read(BUFFER) diff --git a/tools/data_source/microbial_import.py b/tools/data_source/microbial_import.py index 38a72e36636..8de22b1e693 100644 --- a/tools/data_source/microbial_import.py +++ b/tools/data_source/microbial_import.py @@ -19,7 +19,7 @@ out_file1 = sys.argv[2] have_none = True while have_none: try: - uids.remove('None') + uids.remove("None") except ValueError: have_none = False @@ -31,7 +31,7 @@ try: for line in open(filename): if not line or line[0:1] == "#": continue - fields = line.split('\t') + fields = line.split("\t") try: info_type = fields.pop(0) diff --git a/tools/data_source/microbial_import_code.py b/tools/data_source/microbial_import_code.py index b8186b060d0..8da26697a62 100644 --- a/tools/data_source/microbial_import_code.py +++ b/tools/data_source/microbial_import_code.py @@ -3,7 +3,7 @@ from __future__ import print_function from shutil import copyfile -def load_microbial_data(GALAXY_DATA_INDEX_DIR, sep='\t'): +def load_microbial_data(GALAXY_DATA_INDEX_DIR, sep="\t"): # FIXME: this function is duplicated in the DynamicOptions class. It is used here only to # set data.name in exec_after_process(). microbe_info = {} @@ -11,8 +11,8 @@ def load_microbial_data(GALAXY_DATA_INDEX_DIR, sep='\t'): filename = "%s/microbial_data.loc" % GALAXY_DATA_INDEX_DIR for line in open(filename): - line = line.rstrip('\r\n') - if line and not line.startswith('#'): + line = line.rstrip("\r\n") + if line and not line.startswith("#"): fields = line.split(sep) # read each line, if not enough fields, go to next line try: @@ -28,13 +28,13 @@ def load_microbial_data(GALAXY_DATA_INDEX_DIR, sep='\t'): link_site = fields.pop(0) if org_num not in orgs: orgs[org_num] = {} - orgs[org_num]['chrs'] = {} - orgs[org_num]['name'] = name - orgs[org_num]['kingdom'] = kingdom - orgs[org_num]['group'] = group - orgs[org_num]['chromosomes'] = chromosomes - orgs[org_num]['info_url'] = info_url - orgs[org_num]['link_site'] = link_site + orgs[org_num]["chrs"] = {} + orgs[org_num]["name"] = name + orgs[org_num]["kingdom"] = kingdom + orgs[org_num]["group"] = group + orgs[org_num]["chromosomes"] = chromosomes + orgs[org_num]["info_url"] = info_url + orgs[org_num]["link_site"] = link_site elif info_type.upper() == "CHR": # CHR 12521 CP000315 Clostridium perfringens phage phiSM101, complete genome 38092 110684521 CP000315.1 org_num = fields.pop(0) @@ -45,15 +45,15 @@ def load_microbial_data(GALAXY_DATA_INDEX_DIR, sep='\t'): gb = fields.pop(0) info_url = fields.pop(0) chr = {} - chr['name'] = name - chr['length'] = length - chr['gi'] = gi - chr['gb'] = gb - chr['info_url'] = info_url + chr["name"] = name + chr["length"] = length + chr["gi"] = gi + chr["gb"] = gb + chr["info_url"] = info_url if org_num not in orgs: orgs[org_num] = {} - orgs[org_num]['chrs'] = {} - orgs[org_num]['chrs'][chr_acc] = chr + orgs[org_num]["chrs"] = {} + orgs[org_num]["chrs"][chr_acc] = chr elif info_type.upper() == "DATA": # DATA 12521_12521_CDS 12521 CP000315 CDS bed /home/djb396/alignments/playground/bacteria/12521/CP000315.CDS.bed uid = fields.pop(0) @@ -63,26 +63,26 @@ def load_microbial_data(GALAXY_DATA_INDEX_DIR, sep='\t'): filetype = fields.pop(0) path = fields.pop(0) data = {} - data['filetype'] = filetype - data['path'] = path - data['feature'] = feature + data["filetype"] = filetype + data["path"] = path + data["feature"] = feature if org_num not in orgs: orgs[org_num] = {} - orgs[org_num]['chrs'] = {} - if 'data' not in orgs[org_num]['chrs'][chr_acc]: - orgs[org_num]['chrs'][chr_acc]['data'] = {} - orgs[org_num]['chrs'][chr_acc]['data'][uid] = data + orgs[org_num]["chrs"] = {} + if "data" not in orgs[org_num]["chrs"][chr_acc]: + orgs[org_num]["chrs"][chr_acc]["data"] = {} + orgs[org_num]["chrs"][chr_acc]["data"][uid] = data else: continue except Exception: continue for org_num in orgs: org = orgs[org_num] - if org['kingdom'] not in microbe_info: - microbe_info[org['kingdom']] = {} - if org_num not in microbe_info[org['kingdom']]: - microbe_info[org['kingdom']][org_num] = org + if org["kingdom"] not in microbe_info: + microbe_info[org["kingdom"]] = {} + if org_num not in microbe_info[org["kingdom"]]: + microbe_info[org["kingdom"]][org_num] = org return microbe_info @@ -93,15 +93,15 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr if history is None: print("unknown history!") return - kingdom = param_dict.get('kingdom', None) - org = param_dict.get('org', None) + kingdom = param_dict.get("kingdom", None) + org = param_dict.get("org", None) # if not (kingdom or group or org): if not (kingdom or org): print("Parameters are not available.") GALAXY_DATA_INDEX_DIR = app.config.tool_data_path - microbe_info = load_microbial_data(GALAXY_DATA_INDEX_DIR, sep='\t') + microbe_info = load_microbial_data(GALAXY_DATA_INDEX_DIR, sep="\t") split_stdout = stdout.split("\n") basic_name = "" for line in split_stdout: @@ -114,7 +114,16 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr data = next(iter(out_data.values())) data.set_size() basic_name = data.name - data.name = data.name + " (" + microbe_info[kingdom][org]['chrs'][chr]['data'][description]['feature'] + " for " + microbe_info[kingdom][org]['name'] + ":" + chr + ")" + data.name = ( + data.name + + " (" + + microbe_info[kingdom][org]["chrs"][chr]["data"][description]["feature"] + + " for " + + microbe_info[kingdom][org]["name"] + + ":" + + chr + + ")" + ) data.dbkey = dbkey data.info = data.name data = app.datatypes_registry.change_datatype(data, file_type) @@ -128,10 +137,21 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr dbkey = fields[3] filepath = fields[4] file_type = fields[5] - newdata = app.model.HistoryDatasetAssociation(create_dataset=True, sa_session=app.model.context) # This import should become a library + newdata = app.model.HistoryDatasetAssociation( + create_dataset=True, sa_session=app.model.context + ) # This import should become a library newdata.set_size() newdata.extension = file_type - newdata.name = basic_name + " (" + microbe_info[kingdom][org]['chrs'][chr]['data'][description]['feature'] + " for " + microbe_info[kingdom][org]['name'] + ":" + chr + ")" + newdata.name = ( + basic_name + + " (" + + microbe_info[kingdom][org]["chrs"][chr]["data"][description]["feature"] + + " for " + + microbe_info[kingdom][org]["name"] + + ":" + + chr + + ")" + ) app.model.context.add(newdata) app.model.context.flush() app.security_agent.copy_dataset_permissions(base_dataset.dataset, newdata.dataset) diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index 0e68d5689e4..b389cb6902a 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -40,6 +40,7 @@ def get_file_sources(): global _file_sources if _file_sources is None: from galaxy.files import ConfiguredFileSources + file_sources = None if os.path.exists("file_sources.json"): file_sources_as_dict = None @@ -55,16 +56,12 @@ def get_file_sources(): def file_err(msg, dataset): # never remove a server-side upload - if dataset.type not in ('server_dir', 'path_paste'): + if dataset.type not in ("server_dir", "path_paste"): try: os.remove(dataset.path) except Exception: pass - return dict(type='dataset', - ext='data', - dataset_id=dataset.dataset_id, - stderr=msg, - failed=True) + return dict(type="dataset", ext="data", dataset_id=dataset.dataset_id, stderr=msg, failed=True) def safe_dict(d): @@ -80,7 +77,7 @@ def safe_dict(d): def parse_outputs(args): rval = {} for arg in args: - id, files_path, path = arg.split(':', 2) + id, files_path, path = arg.split(":", 2) rval[int(id)] = (path, files_path) return rval @@ -89,16 +86,18 @@ def add_file(dataset, registry, output_path: str) -> Dict[str, str]: ext = None compression_type = None line_count = None - link_data_only_str = dataset.get('link_data_only', 'copy_files') - if link_data_only_str not in ['link_to_files', 'copy_files']: - raise UploadProblemException("Invalid setting '%s' for option link_data_only - upload request misconfigured" % link_data_only_str) - link_data_only = link_data_only_str == 'link_to_files' + link_data_only_str = dataset.get("link_data_only", "copy_files") + if link_data_only_str not in ["link_to_files", "copy_files"]: + raise UploadProblemException( + "Invalid setting '%s' for option link_data_only - upload request misconfigured" % link_data_only_str + ) + link_data_only = link_data_only_str == "link_to_files" # run_as_real_user is estimated from galaxy config (external chmod indicated of inputs executed) # If this is True we always purge supplied upload inputs so they are cleaned up and we reuse their # paths during data conversions since this user already owns that path. # Older in_place check for upload jobs created before 18.01, TODO remove in 19.XX. xref #5206 - run_as_real_user = dataset.get('run_as_real_user', False) or dataset.get("in_place", False) + run_as_real_user = dataset.get("run_as_real_user", False) or dataset.get("in_place", False) # purge_source defaults to True unless this is an FTP import and # ftp_upload_purge has been overridden to False in Galaxy's config. @@ -106,43 +105,45 @@ def add_file(dataset, registry, output_path: str) -> Dict[str, str]: # - the job does not have write access to the file, e.g. when running as the # real user # - the files are uploaded from external paths. - purge_source = dataset.get('purge_source', True) and not run_as_real_user and dataset.type not in ('server_dir', 'path_paste') + purge_source = ( + dataset.get("purge_source", True) and not run_as_real_user and dataset.type not in ("server_dir", "path_paste") + ) # in_place is True unless we are running as a real user or importing external paths (i.e. # this is a real upload and not a path paste or ftp import). # in_place should always be False if running as real user because the uploaded file will # be owned by Galaxy and not the user and it should be False for external paths so Galaxy doesn't # modify files not controlled by Galaxy. - in_place = not run_as_real_user and dataset.type not in ('server_dir', 'path_paste', 'ftp_import') + in_place = not run_as_real_user and dataset.type not in ("server_dir", "path_paste", "ftp_import") # Base on the check_upload_content Galaxy config option and on by default, this enables some # security related checks on the uploaded content, but can prevent uploads from working in some cases. - check_content = dataset.get('check_content', True) + check_content = dataset.get("check_content", True) # auto_decompress is a request flag that can be swapped off to prevent Galaxy from automatically # decompressing archive files before sniffing. - auto_decompress = dataset.get('auto_decompress', True) + auto_decompress = dataset.get("auto_decompress", True) try: dataset.file_type except AttributeError: - raise UploadProblemException('Unable to process uploaded file, missing file_type parameter.') + raise UploadProblemException("Unable to process uploaded file, missing file_type parameter.") - if dataset.type == 'url': + if dataset.type == "url": try: dataset.path = sniff.stream_url_to_file(dataset.path, file_sources=get_file_sources()) except Exception as e: - raise UploadProblemException('Unable to fetch %s\n%s' % (dataset.path, unicodify(e))) + raise UploadProblemException("Unable to fetch %s\n%s" % (dataset.path, unicodify(e))) # See if we have an empty file if not os.path.exists(dataset.path): - raise UploadProblemException('Uploaded temporary file (%s) does not exist.' % dataset.path) + raise UploadProblemException("Uploaded temporary file (%s) does not exist." % dataset.path) stdout, ext, datatype, is_binary, converted_path, _, _ = handle_upload( registry=registry, path=dataset.path, requested_ext=dataset.file_type, name=dataset.name, - tmp_prefix='data_id_%s_upload_' % dataset.dataset_id, + tmp_prefix="data_id_%s_upload_" % dataset.dataset_id, tmp_dir=output_adjacent_tmpdir(output_path), check_content=check_content, link_data_only=link_data_only, @@ -153,15 +154,21 @@ def add_file(dataset, registry, output_path: str) -> Dict[str, str]: ) # Strip compression extension from name - if compression_type and not getattr(datatype, 'compressed', False) and dataset.name.endswith('.' + compression_type): - dataset.name = dataset.name[:-len('.' + compression_type)] + if ( + compression_type + and not getattr(datatype, "compressed", False) + and dataset.name.endswith("." + compression_type) + ): + dataset.name = dataset.name[: -len("." + compression_type)] # Move dataset if link_data_only: # Never alter a file that will not be copied to Galaxy's local file store. if datatype.dataset_content_needs_grooming(dataset.path): - err_msg = 'The uploaded files need grooming, so change your Copy data into Galaxy? selection to be ' + \ - 'Copy files into Galaxy instead of Link to files without copying into Galaxy so grooming can be performed.' + err_msg = ( + "The uploaded files need grooming, so change your Copy data into Galaxy? selection to be " + + "Copy files into Galaxy instead of Link to files without copying into Galaxy so grooming can be performed." + ) raise UploadProblemException(err_msg) if not link_data_only: # Move the dataset to its "real" path. converted_path is a tempfile so we move it even if purge_source is False. @@ -181,15 +188,12 @@ def add_file(dataset, registry, output_path: str) -> Dict[str, str]: shutil.copy(dataset.path, output_path) # Write the job info - stdout = stdout or 'uploaded %s file' % ext - info = dict(type='dataset', - dataset_id=dataset.dataset_id, - ext=ext, - stdout=stdout, - name=dataset.name, - line_count=line_count) - if dataset.get('uuid', None) is not None: - info['uuid'] = dataset.get('uuid') + stdout = stdout or "uploaded %s file" % ext + info = dict( + type="dataset", dataset_id=dataset.dataset_id, ext=ext, stdout=stdout, name=dataset.name, line_count=line_count + ) + if dataset.get("uuid", None) is not None: + info["uuid"] = dataset.get("uuid") # FIXME: does this belong here? also not output-adjacent-tmpdir aware =/ if not link_data_only and datatype and datatype.dataset_content_needs_grooming(output_path): # Groom the dataset content if necessary @@ -211,7 +215,7 @@ def add_composite_file(dataset, registry, output_path, files_path): try: temp_name = sniff.stream_url_to_file(path_or_url, file_sources=file_sources) except Exception as e: - raise UploadProblemException('Unable to fetch %s\n%s' % (path_or_url, unicodify(e))) + raise UploadProblemException("Unable to fetch %s\n%s" % (path_or_url, unicodify(e))) return temp_name, isa_url @@ -221,13 +225,13 @@ def add_composite_file(dataset, registry, output_path, files_path): safe_makedirs(files_path) def stage_file(name, composite_file_path, is_binary=False): - dp = composite_file_path['path'] + dp = composite_file_path["path"] path, isa_url = to_path(dp) if isa_url: dataset.path = path dp = path - auto_decompress = composite_file_path.get('auto_decompress', True) + auto_decompress = composite_file_path.get("auto_decompress", True) if auto_decompress and not datatype.composite_type and CompressedFile.can_decompress(dp): # It isn't an explicitly composite datatype, so these are just extra files to attach # as composite data. It'd be better if Galaxy was communicating this to the tool @@ -237,7 +241,7 @@ def add_composite_file(dataset, registry, output_path, files_path): CompressedFile(dp).extract(files_path) else: tmpdir = output_adjacent_tmpdir(output_path) - tmp_prefix = 'data_id_%s_convert_' % dataset.dataset_id + tmp_prefix = "data_id_%s_convert_" % dataset.dataset_id sniff.handle_composite_file( datatype, dp, @@ -255,9 +259,11 @@ def add_composite_file(dataset, registry, output_path, files_path): for name, value in dataset.composite_files.items(): value = bunch.Bunch(**value) if value.name not in dataset.composite_file_paths: - raise UploadProblemException("Failed to find file_path %s in %s" % (value.name, dataset.composite_file_paths)) + raise UploadProblemException( + "Failed to find file_path %s in %s" % (value.name, dataset.composite_file_paths) + ) if dataset.composite_file_paths[value.name] is None and not value.optional: - raise UploadProblemException('A required composite data file was not provided (%s)' % name) + raise UploadProblemException("A required composite data file was not provided (%s)" % name) elif dataset.composite_file_paths[value.name] is not None: composite_file_path = dataset.composite_file_paths[value.name] stage_file(name, composite_file_path, value.is_binary) @@ -273,9 +279,7 @@ def add_composite_file(dataset, registry, output_path, files_path): shutil.move(primary_file_path, output_path) # Write the job info - return dict(type='dataset', - dataset_id=dataset.dataset_id, - stdout='uploaded %s file' % dataset.file_type) + return dict(type="dataset", dataset_id=dataset.dataset_id, stdout="uploaded %s file" % dataset.file_type) def __read_paramfile(path): @@ -296,14 +300,14 @@ def __read_old_paramfile(path): def __write_job_metadata(metadata): # TODO: make upload/set_metadata compatible with https://github.com/galaxyproject/galaxy/pull/4437 - with open('galaxy.json', 'w') as fh: + with open("galaxy.json", "w") as fh: for meta in metadata: dump(meta, fh) - fh.write('\n') + fh.write("\n") def output_adjacent_tmpdir(output_path): - """ For temp files that will ultimately be moved to output_path anyway + """For temp files that will ultimately be moved to output_path anyway just create the file directly in output_path's directory so shutil.move will work optimally. """ @@ -313,7 +317,7 @@ def output_adjacent_tmpdir(output_path): def __main__(): if len(sys.argv) < 4: - print('usage: upload.py ...', file=sys.stderr) + print("usage: upload.py ...", file=sys.stderr) sys.exit(1) output_paths = parse_outputs(sys.argv[4:]) @@ -332,10 +336,10 @@ def __main__(): try: output_path = output_paths[int(dataset.dataset_id)][0] except Exception: - print('Output path for dataset %s not found on command line' % dataset.dataset_id, file=sys.stderr) + print("Output path for dataset %s not found on command line" % dataset.dataset_id, file=sys.stderr) sys.exit(1) try: - if dataset.type == 'composite': + if dataset.type == "composite": files_path = output_paths[int(dataset.dataset_id)][1] metadata.append(add_composite_file(dataset, registry, output_path, files_path)) else: @@ -345,5 +349,5 @@ def __main__(): __write_job_metadata(metadata) -if __name__ == '__main__': +if __name__ == "__main__": __main__() diff --git a/tools/evolution/add_scores.py b/tools/evolution/add_scores.py index a498ce489b6..1a99fd94cd2 100755 --- a/tools/evolution/add_scores.py +++ b/tools/evolution/add_scores.py @@ -11,21 +11,21 @@ def die(message): sys.exit(1) -def open_or_die(filename, mode='r', message=None): +def open_or_die(filename, mode="r", message=None): if message is None: - message = 'Error opening %s' % filename + message = "Error opening %s" % filename try: fh = open(filename, mode) except IOError as err: - die('%s: %s' % (message, err.strerror)) + die("%s: %s" % (message, err.strerror)) return fh class LocationFile(object): - def __init__(self, filename, comment_chars=None, delimiter='\t', key_column=0): + def __init__(self, filename, comment_chars=None, delimiter="\t", key_column=0): self.filename = filename if comment_chars is None: - self.comment_chars = ('#') + self.comment_chars = "#" else: self.comment_chars = tuple(comment_chars) self.delimiter = delimiter @@ -39,20 +39,26 @@ class LocationFile(object): line_number = 0 for line in fh: line_number += 1 - line = line.rstrip('\r\n') + line = line.rstrip("\r\n") if not line.startswith(self.comment_chars): elems = line.split(self.delimiter) if len(elems) <= self.key_column: - die('Location file %s line %d: less than %d columns' % (self.filename, line_number, self.key_column + 1)) + die( + "Location file %s line %d: less than %d columns" + % (self.filename, line_number, self.key_column + 1) + ) else: key = elems.pop(self.key_column) if key in self._map: if self._map[key] != elems: - die('Location file %s line %d: duplicate key "%s"' % (self.filename, line_number, key)) + die( + 'Location file %s line %d: duplicate key "%s"' + % (self.filename, line_number, key) + ) else: self._map[key] = elems except IOError as err: - die('Error opening location file %s: %s' % (self.filename, err.strerror)) + die("Error opening location file %s: %s" % (self.filename, err.strerror)) def get_values(self, key): if key in self._map: @@ -71,10 +77,10 @@ def main(): # open input, output, and bigwig files location_file = LocationFile(loc_filename) bigwig_filename = location_file.get_values(loc_key) - bwfh = open_or_die(bigwig_filename, message='Error opening BigWig file %s' % bigwig_filename) + bwfh = open_or_die(bigwig_filename, message="Error opening BigWig file %s" % bigwig_filename) bw = BigWigFile(file=bwfh) - ifh = open_or_die(input_filename, message='Error opening input file %s' % input_filename) - ofh = open_or_die(output_filename, mode='w', message='Error opening output file %s' % output_filename) + ifh = open_or_die(input_filename, message="Error opening input file %s" % input_filename) + ofh = open_or_die(output_filename, mode="w", message="Error opening output file %s" % output_filename) # make column numbers 0-based chrom_col = int(chrom_col) - 1 @@ -85,8 +91,8 @@ def main(): line_number = 0 for line in ifh: line_number += 1 - line = line.rstrip('\r\n') - elems = line.split('\t') + line = line.rstrip("\r\n") + elems = line.split("\t") if len(elems) > min_cols: chrom = elems[chrom_col].strip() # base-0 position in chrom @@ -95,12 +101,15 @@ def main(): score_list_len = len(score_list) if score_list_len == 1: beg, end, score = score_list[0] - score_val = '%1.3f' % score + score_val = "%1.3f" % score elif score_list_len == 0: - score_val = 'NA' + score_val = "NA" else: - die('%s line %d: chrom=%s, start=%d, score_list_len = %d' % (input_filename, line_number, chrom, start, score_list_len)) - print('\t'.join((line, score_val)), file=ofh) + die( + "%s line %d: chrom=%s, start=%d, score_list_len = %d" + % (input_filename, line_number, chrom, start, score_list_len) + ) + print("\t".join((line, score_val)), file=ofh) else: print(line, file=ofh) diff --git a/tools/evolution/codingSnps_filter.py b/tools/evolution/codingSnps_filter.py index 4055d4287c1..442b0d8afd7 100755 --- a/tools/evolution/codingSnps_filter.py +++ b/tools/evolution/codingSnps_filter.py @@ -23,14 +23,18 @@ def validate_input(trans, error_map, param_values, page_param_map): if param.metadata.strandCol is not None: int(param.metadata.strandCol) except Exception: - error_msg = ("The attributes of this dataset are not properly set. " - "Click the pencil icon in the history item to set the chrom, start, end and strand columns.") + error_msg = ( + "The attributes of this dataset are not properly set. " + "Click the pencil icon in the history item to set the chrom, start, end and strand columns." + ) error_map[name] = error_msg data_param_names.add(name) if len(dbkeys) > 1: for name in data_param_names: - error_map[name] = "All datasets must belong to same genomic build, " \ + error_map[name] = ( + "All datasets must belong to same genomic build, " "this dataset is linked to build '%s'" % param_values[name].dbkey + ) if data_params != len(data_param_names): for name in data_param_names: error_map[name] = "A dataset of the appropriate type is required" diff --git a/tools/extract/extract_genomic_dna.py b/tools/extract/extract_genomic_dna.py index bf346aa5faf..e5155503fcc 100755 --- a/tools/extract/extract_genomic_dna.py +++ b/tools/extract/extract_genomic_dna.py @@ -34,7 +34,18 @@ def stop_err(msg): def reverse_complement(s): - complement_dna = {"A": "T", "T": "A", "C": "G", "G": "C", "a": "t", "t": "a", "c": "g", "g": "c", "N": "N", "n": "n"} + complement_dna = { + "A": "T", + "T": "A", + "C": "G", + "G": "C", + "a": "t", + "t": "a", + "c": "g", + "g": "c", + "N": "N", + "n": "n", + } reversed_s = [] for i in s: reversed_s.append(complement_dna[i]) @@ -46,9 +57,9 @@ def check_seq_file(dbkey, GALAXY_DATA_INDEX_DIR): # Checks for the presence of *.nib files matching the dbkey within alignseq.loc seq_file = "%s/alignseq.loc" % GALAXY_DATA_INDEX_DIR for line in open(seq_file): - line = line.rstrip('\r\n') - if line and not line.startswith("#") and line.startswith('seq'): - fields = line.split('\t') + line = line.rstrip("\r\n") + if line and not line.startswith("#") and line.startswith("seq"): + fields = line.split("\t") if len(fields) >= 3 and fields[1] == dbkey: print("Using *.nib genomic reference files") return fields[2].strip() @@ -56,14 +67,14 @@ def check_seq_file(dbkey, GALAXY_DATA_INDEX_DIR): # If no entry in aligseq.loc was found, check for the presence of a *.2bit file in twobit.loc seq_file = "%s/twobit.loc" % GALAXY_DATA_INDEX_DIR for line in open(seq_file): - line = line.rstrip('\r\n') - if line and not line.startswith("#") and line.endswith('.2bit'): - fields = line.split('\t') + line = line.rstrip("\r\n") + if line and not line.startswith("#") and line.endswith(".2bit"): + fields = line.split("\t") if len(fields) >= 2 and fields[0] == dbkey: print("Using a *.2bit genomic reference file") return fields[1].strip() - return '' + return "" def __main__(): @@ -72,7 +83,7 @@ def __main__(): # options, args = doc_optparse.parse(__doc__) try: - if len(options.cols.split(',')) == 5: + if len(options.cols.split(",")) == 5: # BED file chrom_col, start_col, end_col, strand_col, name_col = parse_cols_arg(options.cols) else: @@ -103,14 +114,14 @@ def __main__(): cmd = "faToTwoBit %s %s" % (fasta_file, seq_path) tmp_name = tempfile.NamedTemporaryFile(dir=".").name - tmp_stderr = open(tmp_name, 'wb') + tmp_stderr = open(tmp_name, "wb") proc = subprocess.Popen(args=cmd, shell=True, stderr=tmp_stderr.fileno()) returncode = proc.wait() tmp_stderr.close() # Get stderr, allowing for case where it's very large. - tmp_stderr = open(tmp_name, 'rb') - stderr = '' + tmp_stderr = open(tmp_name, "rb") + stderr = "" buffsize = 1048576 try: while True: @@ -125,7 +136,7 @@ def __main__(): if returncode != 0: raise Exception(stderr) except Exception as e: - stop_err('Error running faToTwoBit. ' + str(e)) + stop_err("Error running faToTwoBit. " + str(e)) else: seq_path = check_seq_file(dbkey, GALAXY_DATA_INDEX_DIR) if not os.path.exists(seq_path): @@ -141,14 +152,14 @@ def __main__(): if isinstance(feature, gff_util.GFFFeature): return feature.lines() else: - return [feature.rstrip('\r\n')] + return [feature.rstrip("\r\n")] skipped_lines = 0 first_invalid_line = 0 invalid_lines = [] fout = open(output_filename, "w") warnings = [] - warning = '' + warning = "" twobitfile = None file_iterator = open(input_filename) if gff_format and interpret_features: @@ -170,9 +181,9 @@ def __main__(): strand = feature.strand else: # Processing lines, either interval or GFF format. - line = feature.rstrip('\r\n') + line = feature.rstrip("\r\n") if line and not line.startswith("#"): - fields = line.split('\t') + fields = line.split("\t") try: chrom = fields[chrom_col] start = int(fields[start_col]) @@ -200,9 +211,9 @@ def __main__(): skipped_lines += len(invalid_lines) continue - if strand not in ['+', '-']: - strand = '+' - sequence = '' + if strand not in ["+", "-"]: + strand = "+" + sequence = "" else: continue @@ -216,7 +227,11 @@ def __main__(): try: sequence = nib.get(start, end - start) except Exception: - warning = "Unable to fetch the sequence from '%d' to '%d' for build '%s'. " % (start, end - start, dbkey) + warning = "Unable to fetch the sequence from '%d' to '%d' for build '%s'. " % ( + start, + end - start, + dbkey, + ) warnings.append(warning) if not invalid_lines: invalid_lines = get_lines(feature) @@ -224,18 +239,22 @@ def __main__(): skipped_lines += len(invalid_lines) continue elif seq_path and os.path.isfile(seq_path): - if not(twobitfile): - twobitfile = bx.seq.twobit.TwoBitFile(open(seq_path, 'rb')) + if not (twobitfile): + twobitfile = bx.seq.twobit.TwoBitFile(open(seq_path, "rb")) try: if options.gff and interpret_features: # Create sequence from intervals within a feature. - sequence = '' + sequence = "" for interval in feature.intervals: - sequence += twobitfile[interval.chrom][interval.start:interval.end] + sequence += twobitfile[interval.chrom][interval.start : interval.end] else: sequence = twobitfile[chrom][start:end] except Exception: - warning = "Unable to fetch the sequence from '%d' to '%d' for chrom '%s'. " % (start, end - start, chrom) + warning = "Unable to fetch the sequence from '%d' to '%d' for chrom '%s'. " % ( + start, + end - start, + chrom, + ) warnings.append(warning) if not invalid_lines: invalid_lines = get_lines(feature) @@ -250,9 +269,13 @@ def __main__(): first_invalid_line = line_count skipped_lines += len(invalid_lines) continue - if sequence == '': - warning = "Chrom: '%s', start: '%s', end: '%s' is either invalid or not present in build '%s'. " % \ - (chrom, start, end, dbkey) + if sequence == "": + warning = "Chrom: '%s', start: '%s', end: '%s' is either invalid or not present in build '%s'. " % ( + chrom, + start, + end, + dbkey, + ) warnings.append(warning) if not invalid_lines: invalid_lines = get_lines(feature) @@ -282,13 +305,22 @@ def __main__(): # TODO: need better GFF Reader to capture all information needed # to produce this line. meta_data = "\t".join( - [feature.chrom, "galaxy_extract_genomic_dna", "interval", - str(feature.start), str(feature.end), feature.score, feature.strand, - ".", gff_util.gff_attributes_to_str(feature.attributes, "GTF")]) + [ + feature.chrom, + "galaxy_extract_genomic_dna", + "interval", + str(feature.start), + str(feature.end), + feature.score, + feature.strand, + ".", + gff_util.gff_attributes_to_str(feature.attributes, "GTF"), + ] + ) else: meta_data = "\t".join(fields) if gff_format: - format_str = "%s seq \"%s\";\n" + format_str = '%s seq "%s";\n' else: format_str = "%s\t%s\n" fout.write(format_str % (meta_data, str(sequence))) @@ -307,7 +339,10 @@ def __main__(): print(warn_msg) if skipped_lines: # Error message includes up to the first 10 skipped lines. - print('Skipped %d invalid lines, 1st is #%d, "%s"' % (skipped_lines, first_invalid_line, '\n'.join(invalid_lines[:10]))) + print( + 'Skipped %d invalid lines, 1st is #%d, "%s"' + % (skipped_lines, first_invalid_line, "\n".join(invalid_lines[:10])) + ) # Clean up temp file. if fasta_file: diff --git a/tools/extract/liftOver_wrapper.py b/tools/extract/liftOver_wrapper.py index 90385814c8a..eaa9c7096b3 100644 --- a/tools/extract/liftOver_wrapper.py +++ b/tools/extract/liftOver_wrapper.py @@ -26,7 +26,7 @@ def safe_bed_file(infile): https://lists.soe.ucsc.edu/pipermail/genome/2007-May/013561.html """ fix_pat = re.compile("^(track|browser)") - with tempfile.NamedTemporaryFile(mode='w', delete=False) as out_handle, open(infile, 'r') as in_handle: + with tempfile.NamedTemporaryFile(mode="w", delete=False) as out_handle, open(infile, "r") as in_handle: for line in in_handle: if fix_pat.match(line): line = "#" + line @@ -35,7 +35,9 @@ def safe_bed_file(infile): if len(sys.argv) < 9: - stop_err("USAGE: prog input out_file1 out_file2 input_dbkey output_dbkey infile_type minMatch multiple ") + stop_err( + "USAGE: prog input out_file1 out_file2 input_dbkey output_dbkey infile_type minMatch multiple " + ) infile = sys.argv[1] outfile1 = sys.argv[2] @@ -64,10 +66,25 @@ if in_dbkey == "?": stop_err("Input dataset genome build unspecified, click the pencil icon in the history item to specify it.") if not os.path.isfile(mapfilepath): - stop_err("%s mapping is not currently available." % (mapfilepath.split('/')[-1].split('.')[0])) + stop_err("%s mapping is not currently available." % (mapfilepath.split("/")[-1].split(".")[0])) safe_infile = safe_bed_file(infile) -cmd_line = "liftOver " + gff_option + "-minMatch=" + str(minMatch) + multiple_option + " " + safe_infile + " " + mapfilepath + " " + outfile1 + " " + outfile2 + " > /dev/null" +cmd_line = ( + "liftOver " + + gff_option + + "-minMatch=" + + str(minMatch) + + multiple_option + + " " + + safe_infile + + " " + + mapfilepath + + " " + + outfile1 + + " " + + outfile2 + + " > /dev/null" +) try: # have to nest try-except in try-finally to handle 2.4 @@ -77,6 +94,6 @@ try: if proc.returncode != 0: raise Exception(stderr) except Exception as e: - raise Exception('Exception caught attempting conversion: ' + str(e)) + raise Exception("Exception caught attempting conversion: " + str(e)) finally: os.remove(safe_infile) diff --git a/tools/filters/axt_to_concat_fasta.py b/tools/filters/axt_to_concat_fasta.py index 23e53aba223..759913abed4 100644 --- a/tools/filters/axt_to_concat_fasta.py +++ b/tools/filters/axt_to_concat_fasta.py @@ -26,8 +26,7 @@ def main(): # convert the alignment blocks - reader = bx.align.axt.Reader(sys.stdin, support_ids=True, - species1=species1, species2=species2) + reader = bx.align.axt.Reader(sys.stdin, support_ids=True, species1=species1, species2=species2) sp1text = list() sp2text = list() for a in reader: diff --git a/tools/filters/axt_to_fasta.py b/tools/filters/axt_to_fasta.py index d0b08594c4f..3b949fb7782 100644 --- a/tools/filters/axt_to_fasta.py +++ b/tools/filters/axt_to_fasta.py @@ -26,11 +26,10 @@ def main(): # convert the alignment blocks - reader = bx.align.axt.Reader(sys.stdin, support_ids=True, - species1=species1, species2=species2) + reader = bx.align.axt.Reader(sys.stdin, support_ids=True, species1=species1, species2=species2) for a in reader: - if ("id" in a.attributes): + if "id" in a.attributes: id = a.attributes["id"] else: id = None diff --git a/tools/filters/axt_to_lav.py b/tools/filters/axt_to_lav.py index 557a688c34b..cfdb61d5046 100644 --- a/tools/filters/axt_to_lav.py +++ b/tools/filters/axt_to_lav.py @@ -55,8 +55,8 @@ def main(): # pick off options args = sys.argv[1:] - seq_file2 = open(args.pop(-1), 'w') - seq_file1 = open(args.pop(-1), 'w') + seq_file2 = open(args.pop(-1), "w") + seq_file1 = open(args.pop(-1), "w") lav_out = args.pop(-1) axt_in = args.pop(-1) while len(args) > 0: @@ -102,25 +102,32 @@ def main(): # read the alignments - out = bx.align.lav.Writer(open(lav_out, 'w'), - attributes={"name_format_1": primaryFile, - "name_format_2": secondaryFile}) + out = bx.align.lav.Writer( + open(lav_out, "w"), attributes={"name_format_1": primaryFile, "name_format_2": secondaryFile} + ) axtsRead = 0 axtsWritten = 0 for axtBlock in bx.align.axt.Reader( - open(axt_in), species_to_lengths=speciesToLengths, species1=primary, - species2=secondary, support_ids=True): + open(axt_in), species_to_lengths=speciesToLengths, species1=primary, species2=secondary, support_ids=True + ): axtsRead += 1 out.write(axtBlock) primary_c = axtBlock.get_component_by_src_start(primary) secondary_c = axtBlock.get_component_by_src_start(secondary) - print(">%s_%s_%s_%s" % (primary_c.src, secondary_c.strand, primary_c.start, primary_c.start + primary_c.size), file=seq_file1) + print( + ">%s_%s_%s_%s" % (primary_c.src, secondary_c.strand, primary_c.start, primary_c.start + primary_c.size), + file=seq_file1, + ) print(primary_c.text, file=seq_file1) print(file=seq_file1) - print(">%s_%s_%s_%s" % (secondary_c.src, secondary_c.strand, secondary_c.start, secondary_c.start + secondary_c.size), file=seq_file2) + print( + ">%s_%s_%s_%s" + % (secondary_c.src, secondary_c.strand, secondary_c.start, secondary_c.start + secondary_c.size), + file=seq_file2, + ) print(secondary_c.text, file=seq_file2) print(file=seq_file2) axtsWritten += 1 diff --git a/tools/filters/axt_to_lav_code.py b/tools/filters/axt_to_lav_code.py index 044a4396016..cc319bb655a 100644 --- a/tools/filters/axt_to_lav_code.py +++ b/tools/filters/axt_to_lav_code.py @@ -1,6 +1,5 @@ - def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr): data = out_data["seq_file2"] - data.dbkey = param_dict['dbkey_2'] + data.dbkey = param_dict["dbkey_2"] app.model.context.add(data) app.model.context.flush() diff --git a/tools/filters/bed_to_gff_converter.py b/tools/filters/bed_to_gff_converter.py index fb7e0bd258f..de1c47e755e 100644 --- a/tools/filters/bed_to_gff_converter.py +++ b/tools/filters/bed_to_gff_converter.py @@ -13,13 +13,13 @@ def __main__(): skipped_lines = 0 first_skipped_line = 0 i = 0 - with open(output_name, 'w') as out, open(input_name) as fh_in: + with open(output_name, "w") as out, open(input_name) as fh_in: for i, line in enumerate(fh_in): complete_bed = False - line = line.rstrip('\r\n') - if line and not line.startswith('#') and not line.startswith('track') and not line.startswith('browser'): + line = line.rstrip("\r\n") + if line and not line.startswith("#") and not line.startswith("track") and not line.startswith("browser"): try: - elems = line.split('\t') + elems = line.split("\t") if len(elems) == 12: complete_bed = True chrom = elems[0] @@ -29,34 +29,43 @@ def __main__(): try: feature = elems[3] except Exception: - feature = 'feature%d' % (i + 1) + feature = "feature%d" % (i + 1) start = int(elems[1]) + 1 end = int(elems[2]) try: score = elems[4] except Exception: - score = '0' + score = "0" try: strand = elems[5] except Exception: - strand = '+' + strand = "+" try: group = elems[3] except Exception: - group = 'group%d' % (i + 1) + group = "group%d" % (i + 1) if complete_bed: - out.write('%s\tbed2gff\t%s\t%d\t%d\t%s\t%s\t.\t%s %s;\n' % (chrom, feature, start, end, score, strand, feature, group)) + out.write( + "%s\tbed2gff\t%s\t%d\t%d\t%s\t%s\t.\t%s %s;\n" + % (chrom, feature, start, end, score, strand, feature, group) + ) else: - out.write('%s\tbed2gff\t%s\t%d\t%d\t%s\t%s\t.\t%s;\n' % (chrom, feature, start, end, score, strand, group)) + out.write( + "%s\tbed2gff\t%s\t%d\t%d\t%s\t%s\t.\t%s;\n" + % (chrom, feature, start, end, score, strand, group) + ) if complete_bed: # We have all the info necessary to annotate exons for genes and mRNAs block_count = int(elems[9]) - block_sizes = elems[10].split(',') - block_starts = elems[11].split(',') + block_sizes = elems[10].split(",") + block_starts = elems[11].split(",") for j in range(block_count): exon_start = int(start) + int(block_starts[j]) exon_end = exon_start + int(block_sizes[j]) - 1 - out.write('%s\tbed2gff\texon\t%d\t%d\t%s\t%s\t.\texon %s;\n' % (chrom, exon_start, exon_end, score, strand, group)) + out.write( + "%s\tbed2gff\texon\t%d\t%d\t%s\t%s\t.\texon %s;\n" + % (chrom, exon_start, exon_end, score, strand, group) + ) except Exception: skipped_lines += 1 if not first_skipped_line: @@ -67,7 +76,10 @@ def __main__(): first_skipped_line = i + 1 info_msg = "%i lines converted to GFF version 2. " % (i + 1 - skipped_lines) if skipped_lines > 0: - info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % (skipped_lines, first_skipped_line) + info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % ( + skipped_lines, + first_skipped_line, + ) print(info_msg) diff --git a/tools/filters/convert_characters.py b/tools/filters/convert_characters.py index 15a1a9eb497..434a3b2f2af 100644 --- a/tools/filters/convert_characters.py +++ b/tools/filters/convert_characters.py @@ -8,41 +8,29 @@ import re def __main__(): parser = optparse.OptionParser() - parser.add_option('--strip', action='store_true', - help='strip leading and trailing whitespaces') - parser.add_option('--condense', action='store_true', - help='condense consecutive delimiters') + parser.add_option("--strip", action="store_true", help="strip leading and trailing whitespaces") + parser.add_option("--condense", action="store_true", help="condense consecutive delimiters") (options, args) = parser.parse_args() if len(args) != 3: parser.error("usage: convert_characters.py infile from_char outfile") - char_dict = { - 'T': '\t', - 's': r'\s', - 'Dt': r'\.', - 'C': ',', - 'D': '-', - 'U': '_', - 'P': r'\|', - 'Co': ':', - 'Sc': ';' - } + char_dict = {"T": "\t", "s": r"\s", "Dt": r"\.", "C": ",", "D": "-", "U": "_", "P": r"\|", "Co": ":", "Sc": ";"} # regexp to match 1 or more occurences. from_char = args[1] from_ch = char_dict[from_char] if options.condense: - from_ch += '+' + from_ch += "+" skipped = 0 - with open(args[0], 'rU') as fin: - with open(args[2], 'w') as fout: + with open(args[0], "rU") as fin: + with open(args[2], "w") as fout: for line in fin: if options.strip: line = line.strip() else: - line = line.rstrip('\n') + line = line.rstrip("\n") try: - fout.write("%s\n" % (re.sub(from_ch, '\t', line))) + fout.write("%s\n" % (re.sub(from_ch, "\t", line))) except Exception: skipped += 1 diff --git a/tools/filters/gff/extract_GFF_Features.py b/tools/filters/gff/extract_GFF_Features.py index 321814c0f28..34c002b04cf 100644 --- a/tools/filters/gff/extract_GFF_Features.py +++ b/tools/filters/gff/extract_GFF_Features.py @@ -33,21 +33,21 @@ def main(): if features is None: stop_err("Column %d has no features to display, select another column." % (column + 1)) - fo = open(out_file, 'w') + fo = open(out_file, "w") for line in open(inp_file): - line = line.rstrip('\r\n') - if line and line.startswith('#'): + line = line.rstrip("\r\n") + if line and line.startswith("#"): # Keep valid comment lines in the output fo.write("%s\n" % line) else: try: - if line.split('\t')[column] in features.split(','): + if line.split("\t")[column] in features.split(","): fo.write("%s\n" % line) except Exception: pass fo.close() - print('Column %d features: %s' % (column + 1, features)) + print("Column %d features: %s" % (column + 1, features)) if __name__ == "__main__": diff --git a/tools/filters/gff/gff_filter_by_attribute.py b/tools/filters/gff/gff_filter_by_attribute.py index abc2a5011aa..7c9cbf1b5a9 100644 --- a/tools/filters/gff/gff_filter_by_attribute.py +++ b/tools/filters/gff/gff_filter_by_attribute.py @@ -17,16 +17,50 @@ from ast import ( from json import loads AST_NODE_TYPE_WHITELIST = [ - 'Expr', 'Load', 'Str', 'Num', 'BoolOp', 'Compare', 'And', 'Eq', 'NotEq', - 'Or', 'GtE', 'LtE', 'Lt', 'Gt', 'BinOp', 'Add', 'Div', 'Sub', 'Mult', 'Mod', - 'Pow', 'LShift', 'GShift', 'BitAnd', 'BitOr', 'BitXor', 'UnaryOp', 'Invert', - 'Not', 'NotIn', 'In', 'Is', 'IsNot', 'List', 'Index', 'Subscript', - 'Name', + "Expr", + "Load", + "Str", + "Num", + "BoolOp", + "Compare", + "And", + "Eq", + "NotEq", + "Or", + "GtE", + "LtE", + "Lt", + "Gt", + "BinOp", + "Add", + "Div", + "Sub", + "Mult", + "Mod", + "Pow", + "LShift", + "GShift", + "BitAnd", + "BitOr", + "BitXor", + "UnaryOp", + "Invert", + "Not", + "NotIn", + "In", + "Is", + "IsNot", + "List", + "Index", + "Subscript", + "Name", ] -BUILTIN_AND_MATH_FUNCTIONS = 'abs|all|any|bin|chr|cmp|complex|divmod|float|hex|int|len|long|max|min|oct|ord|pow|range|reversed|round|sorted|str|sum|type|unichr|unicode|log|exp|sqrt|ceil|floor'.split('|') -STRING_AND_LIST_METHODS = [name for name in dir('') + dir([]) if not name.startswith('_')] +BUILTIN_AND_MATH_FUNCTIONS = "abs|all|any|bin|chr|cmp|complex|divmod|float|hex|int|len|long|max|min|oct|ord|pow|range|reversed|round|sorted|str|sum|type|unichr|unicode|log|exp|sqrt|ceil|floor".split( + "|" +) +STRING_AND_LIST_METHODS = [name for name in dir("") + dir([]) if not name.startswith("_")] VALID_FUNCTIONS = BUILTIN_AND_MATH_FUNCTIONS + STRING_AND_LIST_METHODS # Name blacklist isn't strictly needed - but provides extra peace of mind. NAME_BLACKLIST = ["exec", "eval", "globals", "locals", "__import__", "__builtins__"] @@ -84,7 +118,7 @@ def check_simple_name(text): if not len(statements) == 1: return False expression = statements[0] - if expression.__class__.__name__ != 'Expr': + if expression.__class__.__name__ != "Expr": return False for ast_node in walk(expression): @@ -143,7 +177,7 @@ def check_expression(text): if not len(statements) == 1: return False expression = statements[0] - if expression.__class__.__name__ != 'Expr': + if expression.__class__.__name__ != "Expr": return False for ast_node in walk(expression): @@ -165,11 +199,39 @@ def check_expression(text): # def get_operands(filter_condition): # Note that the order of all_operators is important - items_to_strip = ['+', '-', '**', '*', '//', '/', '%', '<<', '>>', '&', '|', '^', '~', '<=', '<', '>=', '>', '==', '!=', '<>', ' and ', ' or ', ' not ', ' is ', ' is not ', ' in ', ' not in '] + items_to_strip = [ + "+", + "-", + "**", + "*", + "//", + "/", + "%", + "<<", + ">>", + "&", + "|", + "^", + "~", + "<=", + "<", + ">=", + ">", + "==", + "!=", + "<>", + " and ", + " or ", + " not ", + " is ", + " is not ", + " in ", + " not in ", + ] for item in items_to_strip: if filter_condition.find(item) >= 0: - filter_condition = filter_condition.replace(item, ' ') - operands = set(filter_condition.split(' ')) + filter_condition = filter_condition.replace(item, " ") + operands = set(filter_condition.split(" ")) return operands @@ -178,7 +240,7 @@ def stop_err(msg): sys.exit() -def check_for_executable(text, description=''): +def check_for_executable(text, description=""): # Attempt to determine if the condition includes executable stuff and, if so, exit. secured = dir() operands = get_operands(text) @@ -211,26 +273,37 @@ for name, a_type in attribute_types.items(): # To avoid a nasty error here, add the official terms from # the GFF3 specification (if not already defined). # (These all start with a capital letter, which is important): -for name in ["ID", "Name", "Alias", "Parent", "Target", "Gap", "Derives_from", - "Note", "Dbxref", "Ontology_term", "Is_circular"]: +for name in [ + "ID", + "Name", + "Alias", + "Parent", + "Target", + "Gap", + "Derives_from", + "Note", + "Dbxref", + "Ontology_term", + "Is_circular", +]: attribute_types[name] = str # Unescape if input has been escaped mapped_str = { - '__lt__': '<', - '__le__': '<=', - '__eq__': '==', - '__ne__': '!=', - '__gt__': '>', - '__ge__': '>=', - '__sq__': '\'', - '__dq__': '"', + "__lt__": "<", + "__le__": "<=", + "__eq__": "==", + "__ne__": "!=", + "__gt__": ">", + "__ge__": ">=", + "__sq__": "'", + "__dq__": '"', } for key, value in mapped_str.items(): cond_text = cond_text.replace(key, value) # Attempt to determine if the condition includes executable stuff and, if so, exit. -check_for_executable(cond_text, 'condition') +check_for_executable(cond_text, "condition") if not check_expression(cond_text): stop_err("Illegal/invalid in condition '%s'" % (cond_text)) @@ -240,11 +313,11 @@ if not check_expression(cond_text): attrs, type_casts = [], [] for name in attribute_types.keys(): attrs.append(name) - type_cast = "get_value('%(name)s', attribute_types['%(name)s'], attribute_values)" % ({'name': name}) + type_cast = "get_value('%(name)s', attribute_types['%(name)s'], attribute_values)" % ({"name": name}) type_casts.append(type_cast) -attr_str = ', '.join(attrs) # 'c1, c2, c3, c4' -type_cast_str = ', '.join(type_casts) # 'str(c1), int(c2), int(c3), str(c4)' +attr_str = ", ".join(attrs) # 'c1, c2, c3, c4' +type_cast_str = ", ".join(type_casts) # 'str(c1), int(c2), int(c3), str(c4)' wrap = "%s = %s" % (attr_str, type_cast_str) # Stats @@ -253,7 +326,7 @@ first_invalid_line = 0 invalid_line = None lines_kept = 0 total_lines = 0 -out = open(out_fname, 'wt') +out = open(out_fname, "wt") # Helper function to safely get and type cast a value in a dict. @@ -265,7 +338,7 @@ def get_value(name, a_type, values_dict): # Read and filter input file, skipping invalid lines -code = ''' +code = """ for i, line in enumerate( open( in_fname ) ): total_lines += 1 line = line.rstrip( '\\r\\n' ) @@ -298,14 +371,17 @@ for i, line in enumerate( open( in_fname ) ): if not invalid_line: first_invalid_line = i + 1 invalid_line = line -''' % (wrap, cond_text) +""" % ( + wrap, + cond_text, +) valid_filter = True try: exec(code) except Exception as e: out.close() - if str(e).startswith('invalid syntax'): + if str(e).startswith("invalid syntax"): valid_filter = False stop_err('Filter condition "%s" likely invalid. See tool tips, syntax and examples.' % cond_text) else: @@ -314,10 +390,13 @@ except Exception as e: if valid_filter: out.close() valid_lines = total_lines - skipped_lines - print('Filtering with %s, ' % (cond_text)) + print("Filtering with %s, " % (cond_text)) if valid_lines > 0: - print('kept %4.2f%% of %d lines.' % (100.0 * lines_kept / valid_lines, total_lines)) + print("kept %4.2f%% of %d lines." % (100.0 * lines_kept / valid_lines, total_lines)) else: - print('Possible invalid filter condition "%s" or non-existent column referenced. See tool tips, syntax and examples.' % cond_text) + print( + 'Possible invalid filter condition "%s" or non-existent column referenced. See tool tips, syntax and examples.' + % cond_text + ) if skipped_lines > 0: print('Skipped %d invalid lines starting at line #%d: "%s"' % (skipped_lines, first_invalid_line, invalid_line)) diff --git a/tools/filters/gff/gff_filter_by_feature_count.py b/tools/filters/gff/gff_filter_by_feature_count.py index 51ac1a4b7af..a407848f157 100644 --- a/tools/filters/gff/gff_filter_by_feature_count.py +++ b/tools/filters/gff/gff_filter_by_feature_count.py @@ -19,16 +19,50 @@ from bx.intervals.io import GenomicInterval from galaxy.datatypes.util.gff_util import GFFReaderWrapper AST_NODE_TYPE_WHITELIST = [ - 'Expr', 'Load', 'Str', 'Num', 'BoolOp', 'Compare', 'And', 'Eq', 'NotEq', - 'Or', 'GtE', 'LtE', 'Lt', 'Gt', 'BinOp', 'Add', 'Div', 'Sub', 'Mult', 'Mod', - 'Pow', 'LShift', 'GShift', 'BitAnd', 'BitOr', 'BitXor', 'UnaryOp', 'Invert', - 'Not', 'NotIn', 'In', 'Is', 'IsNot', 'List', 'Index', 'Subscript', - 'Name', + "Expr", + "Load", + "Str", + "Num", + "BoolOp", + "Compare", + "And", + "Eq", + "NotEq", + "Or", + "GtE", + "LtE", + "Lt", + "Gt", + "BinOp", + "Add", + "Div", + "Sub", + "Mult", + "Mod", + "Pow", + "LShift", + "GShift", + "BitAnd", + "BitOr", + "BitXor", + "UnaryOp", + "Invert", + "Not", + "NotIn", + "In", + "Is", + "IsNot", + "List", + "Index", + "Subscript", + "Name", ] -BUILTIN_AND_MATH_FUNCTIONS = 'abs|all|any|bin|chr|cmp|complex|divmod|float|hex|int|len|long|max|min|oct|ord|pow|range|reversed|round|sorted|str|sum|type|unichr|unicode|log|exp|sqrt|ceil|floor'.split('|') -STRING_AND_LIST_METHODS = [name for name in dir('') + dir([]) if not name.startswith('_')] +BUILTIN_AND_MATH_FUNCTIONS = "abs|all|any|bin|chr|cmp|complex|divmod|float|hex|int|len|long|max|min|oct|ord|pow|range|reversed|round|sorted|str|sum|type|unichr|unicode|log|exp|sqrt|ceil|floor".split( + "|" +) +STRING_AND_LIST_METHODS = [name for name in dir("") + dir([]) if not name.startswith("_")] VALID_FUNCTIONS = BUILTIN_AND_MATH_FUNCTIONS + STRING_AND_LIST_METHODS # Name blacklist isn't strictly needed - but provides extra peace of mind. NAME_BLACKLIST = ["exec", "eval", "globals", "locals", "__import__", "__builtins__"] @@ -84,7 +118,7 @@ def check_expression(text): if not len(statements) == 1: return False expression = statements[0] - if expression.__class__.__name__ != 'Expr': + if expression.__class__.__name__ != "Expr": return False for ast_node in walk(expression): @@ -103,23 +137,16 @@ def check_expression(text): # Valid operators, ordered so that complex operators (e.g. '>=') are # recognized before simple operators (e.g. '>') -ops = [ - '>=', - '<=', - '<', - '>', - '==', - '!=' -] +ops = [">=", "<=", "<", ">", "==", "!="] # Escape sequences for valid operators. mapped_ops = { - '__ge__': ops[0], - '__le__': ops[1], - '__lt__': ops[2], - '__gt__': ops[3], - '__eq__': ops[4], - '__ne__': ops[5], + "__ge__": ops[0], + "__le__": ops[1], + "__lt__": ops[2], + "__gt__": ops[3], + "__eq__": ops[4], + "__ne__": ops[5], } @@ -151,7 +178,7 @@ def __main__(): kept_features = 0 skipped_lines = 0 first_skipped_line = 0 - out = open(output_name, 'w') + out = open(output_name, "w") for i, feature in enumerate(GFFReaderWrapper(open(input_name))): # noqa: B007 if not isinstance(feature, GenomicInterval): continue @@ -159,7 +186,7 @@ def __main__(): for interval in feature.intervals: if interval.feature == feature_name: count += 1 - eval_text = '%s %s' % (count, condition) + eval_text = "%s %s" % (count, condition) if not check_expression(eval_text): print("Invalid condition: %s, cannot filter." % condition, file=sys.stderr) sys.exit(1) @@ -167,7 +194,7 @@ def __main__(): if eval(eval_text): # Keep feature. for interval in feature.intervals: - out.write("\t".join(interval.fields) + '\n') + out.write("\t".join(interval.fields) + "\n") kept_features += 1 # Needed because i is 0-based but want to display stats using 1-based. @@ -175,10 +202,17 @@ def __main__(): # Clean up. out.close() - info_msg = "%i of %i features kept (%.2f%%) using condition %s. " % \ - (kept_features, i, float(kept_features) / i * 100.0, feature_name + condition) + info_msg = "%i of %i features kept (%.2f%%) using condition %s. " % ( + kept_features, + i, + float(kept_features) / i * 100.0, + feature_name + condition, + ) if skipped_lines > 0: - info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % (skipped_lines, first_skipped_line) + info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % ( + skipped_lines, + first_skipped_line, + ) print(info_msg) diff --git a/tools/filters/gff/gtf_filter_by_attribute_values_list.py b/tools/filters/gff/gtf_filter_by_attribute_values_list.py index 8be10348557..da9fe1f04bb 100644 --- a/tools/filters/gff/gtf_filter_by_attribute_values_list.py +++ b/tools/filters/gff/gtf_filter_by_attribute_values_list.py @@ -30,19 +30,19 @@ def parse_gff_attributes(attr_str): if len(pair) == 1: # Could not split for some reason -- raise exception? continue - if pair == '': + if pair == "": continue name = pair[0].strip() - if name == '': + if name == "": continue # Need to strip double quote from values - value = pair[1].strip(" \"") + value = pair[1].strip(' "') attributes[name] = value if len(attributes) == 0: # Could not split attributes string, so entire string must be # 'group' attribute. This is the case for strictly GFF files. - attributes['group'] = attr_str + attributes["group"] = attr_str return attributes @@ -50,15 +50,15 @@ def gff_filter(gff_file, attribute_name, ids_file, output_file): # Put ids in dict for quick lookup. ids_dict = {} for line in open(ids_file): - ids_dict[line.split('\t')[0].strip()] = True + ids_dict[line.split("\t")[0].strip()] = True # Filter GFF file using ids. - with open(output_file, 'w') as output, open(gff_file) as ingff: + with open(output_file, "w") as output, open(gff_file) as ingff: for line in ingff: - if not line or line.startswith('#'): + if not line or line.startswith("#"): output.write(line) continue - fields = line.split('\t') + fields = line.split("\t") attributes = parse_gff_attributes(fields[8]) if attribute_name in attributes and attributes[attribute_name] in ids_dict: output.write(line) diff --git a/tools/filters/gff_to_bed_converter.py b/tools/filters/gff_to_bed_converter.py index 929347ce959..b45a71faf31 100644 --- a/tools/filters/gff_to_bed_converter.py +++ b/tools/filters/gff_to_bed_converter.py @@ -7,7 +7,7 @@ from galaxy.datatypes.util.gff_util import parse_gff_attributes def get_bed_line(chrom, name, strand, blocks): - """ Returns a BED line for given data. """ + """Returns a BED line for given data.""" if len(blocks) == 1: # Use simple BED format if there is only a single block: @@ -45,9 +45,18 @@ def get_bed_line(chrom, name, strand, blocks): # we analyze the block names, but making everything thick makes more sense than # making everything thin. # - return "%s\t%i\t%i\t%s\t0\t%s\t%i\t%i\t0\t%i\t%s\t%s\n" % \ - (chrom, t_start, t_end, name, strand, t_start, t_end, len(block_starts), - ",".join(block_sizes), ",".join(block_starts)) + return "%s\t%i\t%i\t%s\t0\t%s\t%i\t%i\t0\t%i\t%s\t%s\n" % ( + chrom, + t_start, + t_end, + name, + strand, + t_start, + t_end, + len(block_starts), + ",".join(block_sizes), + ",".join(block_starts), + ) def __main__(): @@ -60,18 +69,18 @@ def __main__(): cur_transcript_id = None cur_transcript_strand = None cur_transcripts_blocks = [] # (start, end) for each block. - with open(output_name, 'w') as out, open(input_name) as in_fh: + with open(output_name, "w") as out, open(input_name) as in_fh: for i, line in enumerate(in_fh): - line = line.rstrip('\r\n') - if line and not line.startswith('#'): + line = line.rstrip("\r\n") + if line and not line.startswith("#"): try: # GFF format: chrom source, name, chromStart, chromEnd, score, strand, attributes - elems = line.split('\t') + elems = line.split("\t") start = str(int(elems[3]) - 1) coords = [int(start), int(elems[4])] strand = elems[6] - if strand not in ['+', '-']: - strand = '+' + if strand not in ["+", "-"]: + strand = "+" attributes = parse_gff_attributes(elems[8]) t_id = attributes.get("transcript_id", None) @@ -83,7 +92,14 @@ def __main__(): # Write previous transcript. if cur_transcript_id: # Write BED entry. - out.write(get_bed_line(cur_transcript_chrome, cur_transcript_id, cur_transcript_strand, cur_transcripts_blocks)) + out.write( + get_bed_line( + cur_transcript_chrome, + cur_transcript_id, + cur_transcript_strand, + cur_transcripts_blocks, + ) + ) # Replace any spaces in the name with underscores so UCSC will not complain. name = elems[2].replace(" ", "_") @@ -104,7 +120,11 @@ def __main__(): # Write previous transcript. if cur_transcript_id: # Write BED entry. - out.write(get_bed_line(cur_transcript_chrome, cur_transcript_id, cur_transcript_strand, cur_transcripts_blocks)) + out.write( + get_bed_line( + cur_transcript_chrome, cur_transcript_id, cur_transcript_strand, cur_transcripts_blocks + ) + ) # Start new transcript. cur_transcript_chrome = elems[0] @@ -124,10 +144,15 @@ def __main__(): # Write last transcript. if cur_transcript_id: # Write BED entry. - out.write(get_bed_line(cur_transcript_chrome, cur_transcript_id, cur_transcript_strand, cur_transcripts_blocks)) + out.write( + get_bed_line(cur_transcript_chrome, cur_transcript_id, cur_transcript_strand, cur_transcripts_blocks) + ) info_msg = "%i lines converted to BED. " % (i + 1 - skipped_lines) if skipped_lines > 0: - info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % (skipped_lines, first_skipped_line) + info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % ( + skipped_lines, + first_skipped_line, + ) print(info_msg) diff --git a/tools/filters/grep.py b/tools/filters/grep.py index 98c4aff39b0..b6e72e2feef 100644 --- a/tools/filters/grep.py +++ b/tools/filters/grep.py @@ -28,7 +28,7 @@ from tempfile import NamedTemporaryFile def getopts(argv): opts = {} while argv: - if argv[0][0] == '-': + if argv[0][0] == "-": opts[argv[0]] = argv[1] argv = argv[2:] else: @@ -73,14 +73,16 @@ def main(): # replace if input has been escaped, remove sq # characters that are allowed but need to be escaped - mapped_chars = {'>': '__gt__', - '<': '__lt__', - '\'': '__sq__', - '"': '__dq__', - '[': '__ob__', - ']': '__cb__', - '{': '__oc__', - '}': '__cc__'} + mapped_chars = { + ">": "__gt__", + "<": "__lt__", + "'": "__sq__", + '"': "__dq__", + "[": "__ob__", + "]": "__cb__", + "{": "__oc__", + "}": "__cc__", + } # with new sanitizing we only need to replace for single quote, # but this needs to remain for backwards compatibility @@ -121,7 +123,7 @@ def main(): # create temp file holding pattern # by using a file to hold the pattern, we don't have worry about sanitizing grep commandline and can include single quotes in pattern pattern_file_name = NamedTemporaryFile().name - open(pattern_file_name, 'w').write(pattern) + open(pattern_file_name, "w").write(pattern) # generate grep command commandline = "grep %s %s -f %s %s > %s" % (versionflag, invertflag, pattern_file_name, inputfile, outputfile) diff --git a/tools/filters/gtf_to_bedgraph_converter.py b/tools/filters/gtf_to_bedgraph_converter.py index 711ebfeb73e..a1042e5ea82 100644 --- a/tools/filters/gtf_to_bedgraph_converter.py +++ b/tools/filters/gtf_to_bedgraph_converter.py @@ -21,29 +21,29 @@ def __main__(): # Do conversion. skipped_lines = 0 first_skipped_line = 0 - out = open(tmp_name1, 'w') + out = open(tmp_name1, "w") # Write track data to temporary file. i = 0 for i, line in enumerate(open(input_name)): - line = line.rstrip('\r\n') + line = line.rstrip("\r\n") - if line and not line.startswith('#'): + if line and not line.startswith("#"): try: - elems = line.split('\t') + elems = line.split("\t") start = str(int(elems[3]) - 1) # GTF coordinates are 1-based, BedGraph are 0-based. strand = elems[6] - if strand not in ['+', '-']: - strand = '+' + if strand not in ["+", "-"]: + strand = "+" attributes_list = elems[8].split(";") attributes = {} for name_value_pair in attributes_list: pair = name_value_pair.strip().split(" ") name = pair[0].strip() - if name == '': + if name == "": continue # Need to strip double quote from values - value = pair[1].strip(" \"") + value = pair[1].strip(' "') attributes[name] = value value = attributes[attribute_name] # GTF format: chrom source, name, chromStart, chromEnd, score, strand, frame, attributes. @@ -79,7 +79,10 @@ def __main__(): info_msg = "%i lines converted to BEDGraph. " % (i + 1 - skipped_lines) if skipped_lines > 0: - info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % (skipped_lines, first_skipped_line) + info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % ( + skipped_lines, + first_skipped_line, + ) print(info_msg) diff --git a/tools/filters/join.py b/tools/filters/join.py index 1fb76e16ac9..670921554f4 100644 --- a/tools/filters/join.py +++ b/tools/filters/join.py @@ -22,13 +22,13 @@ from galaxy.util.bunch import Bunch class OffsetList(object): def __init__(self, filesize=0, fmt=None): - self.file = tempfile.NamedTemporaryFile('w+b') + self.file = tempfile.NamedTemporaryFile("w+b") if fmt: self.fmt = fmt elif filesize and filesize <= sys.maxsize * 2: - self.fmt = 'I' + self.fmt = "I" else: - self.fmt = 'Q' + self.fmt = "Q" self.fmt_size = struct.calcsize(self.fmt) @property @@ -66,7 +66,7 @@ class OffsetList(object): if index >= self.size: self.add_offset(offset) else: - temp_file = tempfile.NamedTemporaryFile('w+b') + temp_file = tempfile.NamedTemporaryFile("w+b") self.file.seek(0) temp_file.write(self.file.read((index) * self.fmt_size)) for off in offset: @@ -129,10 +129,10 @@ class SortedOffsets(OffsetList): identifier1 = None index1 += 1 -# methods to help link offsets to lines, ids, etc + # methods to help link offsets to lines, ids, etc def get_identifier_by_line(self, line): if isinstance(line, str): - fields = line.rstrip('\r\n').split(self.split) + fields = line.rstrip("\r\n").split(self.split) if self.column < len(fields): return fields[self.column] return None @@ -162,8 +162,8 @@ class OffsetIndex(object): self._index[start_char] = {} for i, offset in enumerate(sorted_offsets.get_offsets()): identifier = sorted_offsets.get_identifier_by_offset(offset) - if identifier[0:self.index_depth] not in self._index[start_char]: - self._index[start_char][identifier[0:self.index_depth]] = i + if identifier[0 : self.index_depth] not in self._index[start_char]: + self._index[start_char][identifier[0 : self.index_depth]] = i def get_lines_by_identifier(self, identifier): if not identifier: @@ -173,10 +173,10 @@ class OffsetIndex(object): self._build_index() # identifier cannot exist - if identifier[0] not in self._index or identifier[0:self.index_depth] not in self._index[identifier[0]]: + if identifier[0] not in self._index or identifier[0 : self.index_depth] not in self._index[identifier[0]]: return # identifier might exist, search for it - offset_index = self._index[identifier[0]][identifier[0:self.index_depth]] + offset_index = self._index[identifier[0]][identifier[0 : self.index_depth]] while True: if offset_index >= self._offsets[identifier[0]].size: return @@ -208,7 +208,7 @@ class OffsetIndex(object): def get_identifier_by_line(self, line): if isinstance(line, str): - fields = line.rstrip('\r\n').split(self.split) + fields = line.rstrip("\r\n").split(self.split) if self.column < len(fields): return fields[self.column] return None @@ -281,30 +281,44 @@ def fill_empty_columns(line, split, fill_values): else: filled_columns.append(fill_values[i]) if len(fill_values) > len(filled_columns): - filled_columns.extend(fill_values[len(filled_columns):]) + filled_columns.extend(fill_values[len(filled_columns) :]) return split.join(filled_columns) -def join_files(filename1, column1, filename2, column2, out_filename, split=None, buffer=1000000, keep_unmatched=False, keep_partial=False, keep_headers=False, index_depth=3, fill_options=None): +def join_files( + filename1, + column1, + filename2, + column2, + out_filename, + split=None, + buffer=1000000, + keep_unmatched=False, + keep_partial=False, + keep_headers=False, + index_depth=3, + fill_options=None, +): # return identifier based upon line def get_identifier_by_line(line, column, split=None): if isinstance(line, str): - fields = line.rstrip('\r\n').split(split) + fields = line.rstrip("\r\n").split(split) if column < len(fields): return fields[column] return None + if fill_options is None: fill_options = Bunch(fill_unjoined_only=True, file1_columns=None, file2_columns=None) keep_headers_done = False - out = open(out_filename, 'w') + out = open(out_filename, "w") index = BufferedIndex(filename2, column2, split, buffer, index_depth) for line1 in open(filename1): if keep_headers and not keep_headers_done: header1 = line1 with open(filename2) as file2: header2 = file2.readline() - header2 = re.sub(r'^#', '', header2) - out.write("%s%s%s\n" % (header1.rstrip('\r\n'), split, header2.rstrip('\r\n'))) + header2 = re.sub(r"^#", "", header2) + out.write("%s%s%s\n" % (header1.rstrip("\r\n"), split, header2.rstrip("\r\n"))) keep_headers_done = True continue identifier = get_identifier_by_line(line1, column1, split) @@ -312,18 +326,25 @@ def join_files(filename1, column1, filename2, column2, out_filename, split=None, written = False for line2 in index.get_lines_by_identifier(identifier): if not fill_options.fill_unjoined_only: - out.write("%s%s%s\n" % (fill_empty_columns(line1.rstrip('\r\n'), split, fill_options.file1_columns), split, fill_empty_columns(line2.rstrip('\r\n'), split, fill_options.file2_columns))) + out.write( + "%s%s%s\n" + % ( + fill_empty_columns(line1.rstrip("\r\n"), split, fill_options.file1_columns), + split, + fill_empty_columns(line2.rstrip("\r\n"), split, fill_options.file2_columns), + ) + ) else: - out.write("%s%s%s\n" % (line1.rstrip('\r\n'), split, line2.rstrip('\r\n'))) + out.write("%s%s%s\n" % (line1.rstrip("\r\n"), split, line2.rstrip("\r\n"))) written = True if not written and keep_unmatched: - out.write(fill_empty_columns(line1.rstrip('\r\n'), split, fill_options.file1_columns)) + out.write(fill_empty_columns(line1.rstrip("\r\n"), split, fill_options.file1_columns)) if fill_options: if fill_options.file2_columns: out.write("%s%s" % (split, fill_empty_columns("", split, fill_options.file2_columns))) out.write("\n") elif keep_partial: - out.write(fill_empty_columns(line1.rstrip('\r\n'), split, fill_options.file1_columns)) + out.write(fill_empty_columns(line1.rstrip("\r\n"), split, fill_options.file1_columns)) if fill_options: if fill_options.file2_columns: out.write("%s%s" % (split, fill_empty_columns("", split, fill_options.file2_columns))) @@ -334,56 +355,66 @@ def join_files(filename1, column1, filename2, column2, out_filename, split=None, def main(): parser = optparse.OptionParser() parser.add_option( - '-b', '--buffer', - dest='buffer', - type='int', default=1000000, - help='Number of lines to buffer at a time. Default: 1,000,000 lines. A buffer of 0 will attempt to use memory only.' + "-b", + "--buffer", + dest="buffer", + type="int", + default=1000000, + help="Number of lines to buffer at a time. Default: 1,000,000 lines. A buffer of 0 will attempt to use memory only.", ) parser.add_option( - '-d', '--index_depth', - dest='index_depth', - type='int', default=3, - help='Depth to use on filebased offset indexing. Default: 3.' + "-d", + "--index_depth", + dest="index_depth", + type="int", + default=3, + help="Depth to use on filebased offset indexing. Default: 3.", ) parser.add_option( - '-p', '--keep_partial', - action='store_true', - dest='keep_partial', + "-p", + "--keep_partial", + action="store_true", + dest="keep_partial", default=False, - help='Keep rows in first input which are missing identifiers.') + help="Keep rows in first input which are missing identifiers.", + ) parser.add_option( - '-u', '--keep_unmatched', - action='store_true', - dest='keep_unmatched', + "-u", + "--keep_unmatched", + action="store_true", + dest="keep_unmatched", default=False, - help='Keep rows in first input which are not joined with the second input.') + help="Keep rows in first input which are not joined with the second input.", + ) parser.add_option( - '-f', '--fill_options_file', - dest='fill_options_file', - type='str', default=None, - help='Fill empty columns with a values from a JSONified file.') + "-f", + "--fill_options_file", + dest="fill_options_file", + type="str", + default=None, + help="Fill empty columns with a values from a JSONified file.", + ) parser.add_option( - '-H', '--keep_headers', - action='store_true', - dest='keep_headers', - default=False, - help='Keep the headers') + "-H", "--keep_headers", action="store_true", dest="keep_headers", default=False, help="Keep the headers" + ) options, args = parser.parse_args() fill_options = None if options.fill_options_file is not None: try: - fill_options = Bunch(**stringify_dictionary_keys(json.load(open(options.fill_options_file)))) # json.load( open( options.fill_options_file ) ) + fill_options = Bunch( + **stringify_dictionary_keys(json.load(open(options.fill_options_file))) + ) # json.load( open( options.fill_options_file ) ) except Exception as e: print("Warning: Ignoring fill options due to json error (%s)." % e) if fill_options is None: fill_options = Bunch() - if 'fill_unjoined_only' not in fill_options: + if "fill_unjoined_only" not in fill_options: fill_options.fill_unjoined_only = True - if 'file1_columns' not in fill_options: + if "file1_columns" not in fill_options: fill_options.file1_columns = None - if 'file2_columns' not in fill_options: + if "file2_columns" not in fill_options: fill_options.file2_columns = None try: @@ -399,7 +430,20 @@ def main(): # Character for splitting fields and joining lines split = "\t" - return join_files(filename1, column1, filename2, column2, out_filename, split, options.buffer, options.keep_unmatched, options.keep_partial, options.keep_headers, options.index_depth, fill_options=fill_options) + return join_files( + filename1, + column1, + filename2, + column2, + out_filename, + split, + options.buffer, + options.keep_unmatched, + options.keep_partial, + options.keep_headers, + options.index_depth, + fill_options=fill_options, + ) if __name__ == "__main__": diff --git a/tools/filters/joinWrapper.py b/tools/filters/joinWrapper.py index b308d46e968..e596bfba912 100644 --- a/tools/filters/joinWrapper.py +++ b/tools/filters/joinWrapper.py @@ -30,13 +30,13 @@ def main(): os.system("sort -t ' ' -k %d,%d -o %s %s" % (field1, field1, tmpfile1.name, infile1)) os.system("sort -t ' ' -k %d,%d -o %s %s" % (field2, field2, tmpfile2.name, infile2)) except Exception as exc: - stop_err('Initialization error -> %s' % str(exc)) + stop_err("Initialization error -> %s" % str(exc)) option = "" for line in open(tmpfile1.name): line = line.strip() if line: - elems = line.split('\t') + elems = line.split("\t") for j in range(1, len(elems) + 1): if j == 1: option = "1.1" @@ -46,14 +46,14 @@ def main(): # check if join has --version option. BSD join doens't have this option, while GNU join does. # The return value in the latter case will be 0, and non-zero in the latter case. - ret = subprocess.call('join --version 2>/dev/null', shell=True) + ret = subprocess.call("join --version 2>/dev/null", shell=True) # check if we are a version later than 7 of join. If so, we want to skip # checking the order since join will raise an error with duplicated items in # the two files being joined. if ret == 0: cl = subprocess.Popen(["join", "--version"], stdout=subprocess.PIPE) (stdout, _) = cl.communicate() - version_line = stdout.decode('utf-8').split("\n")[0] + version_line = stdout.decode("utf-8").split("\n")[0] (version, _) = version_line.split()[-1].split(".") if int(version) >= 7: flags = "--nocheck-order" @@ -63,14 +63,30 @@ def main(): flags = "" if mode == "V": - cmdline = "join %s -t ' ' -v 1 -o %s -1 %d -2 %d %s %s > %s" % (flags, option, field1, field2, tmpfile1.name, tmpfile2.name, outfile) + cmdline = "join %s -t ' ' -v 1 -o %s -1 %d -2 %d %s %s > %s" % ( + flags, + option, + field1, + field2, + tmpfile1.name, + tmpfile2.name, + outfile, + ) else: - cmdline = "join %s -t ' ' -o %s -1 %d -2 %d %s %s > %s" % (flags, option, field1, field2, tmpfile1.name, tmpfile2.name, outfile) + cmdline = "join %s -t ' ' -o %s -1 %d -2 %d %s %s > %s" % ( + flags, + option, + field1, + field2, + tmpfile1.name, + tmpfile2.name, + outfile, + ) try: os.system(cmdline) except Exception as exj: - stop_err('Error joining the two datasets -> %s' % str(exj)) + stop_err("Error joining the two datasets -> %s" % str(exj)) if __name__ == "__main__": diff --git a/tools/filters/lav_to_bed.py b/tools/filters/lav_to_bed.py index 77ad02cc27f..f5c3854dc4e 100644 --- a/tools/filters/lav_to_bed.py +++ b/tools/filters/lav_to_bed.py @@ -15,8 +15,8 @@ def stop_err(msg): def main(): try: lav_file = open(sys.argv[1]) - bed_file1 = open(sys.argv[2], 'w') - bed_file2 = open(sys.argv[3], 'w') + bed_file1 = open(sys.argv[2], "w") + bed_file2 = open(sys.argv[3], "w") except Exception as e: stop_err(str(e)) @@ -36,7 +36,9 @@ def main(): else: continue # this is a pairwise alignment... if spec in species: - species[spec].write("%s\t%i\t%i\t%s_%s\t%i\t%s\n" % (chrom, c.start, c.end, spec, str(bedsWritten), 0, c.strand)) + species[spec].write( + "%s\t%i\t%i\t%s_%s\t%i\t%s\n" % (chrom, c.start, c.end, spec, str(bedsWritten), 0, c.strand) + ) bedsWritten += 1 for spec, file in species.items(): diff --git a/tools/filters/mergeCols.py b/tools/filters/mergeCols.py index 3346ab8a94d..adbad2d53c5 100644 --- a/tools/filters/mergeCols.py +++ b/tools/filters/mergeCols.py @@ -11,22 +11,22 @@ def stop_err(msg): def __main__(): try: infile = open(sys.argv[1]) - outfile = open(sys.argv[2], 'w') + outfile = open(sys.argv[2], "w") except Exception: - stop_err('Cannot open or create a file\n') + stop_err("Cannot open or create a file\n") if len(sys.argv) < 4: - stop_err('No columns to merge') + stop_err("No columns to merge") else: cols = sys.argv[3:] skipped_lines = 0 for line in infile: - line = line.rstrip('\r\n') - if line and not line.startswith('#'): - fields = line.split('\t') - line += '\t' + line = line.rstrip("\r\n") + if line and not line.startswith("#"): + fields = line.split("\t") + line += "\t" for col in cols: try: line += fields[int(col) - 1] @@ -36,7 +36,7 @@ def __main__(): print(line, file=outfile) if skipped_lines > 0: - print('Skipped %d invalid lines' % skipped_lines) + print("Skipped %d invalid lines" % skipped_lines) if __name__ == "__main__": diff --git a/tools/filters/random_lines_two_pass.py b/tools/filters/random_lines_two_pass.py index d00826b7f6e..66262074bf2 100644 --- a/tools/filters/random_lines_two_pass.py +++ b/tools/filters/random_lines_two_pass.py @@ -41,10 +41,10 @@ def sample(population, k): # An n-length list is smaller than a k-length set, or this is a # mapping type so the other algorithm wouldn't work. pool = list(population) - for i in range(k): # invariant: non-selected at [0,n-i) + for i in range(k): # invariant: non-selected at [0,n-i) j = int(random.random() * (n - i)) result[i] = pool[j] - pool[j] = pool[n - i - 1] # move non-selected item into vacancy + pool[j] = pool[n - i - 1] # move non-selected item into vacancy else: try: selected = set() @@ -55,7 +55,7 @@ def sample(population, k): j = int(random.random() * n) selected_add(j) result[i] = population[j] - except (TypeError, KeyError): # handle (at least) sets + except (TypeError, KeyError): # handle (at least) sets if isinstance(population, list): raise return sample(tuple(population), k) @@ -83,12 +83,14 @@ def get_random(line_offsets, num_lines): def __main__(): parser = optparse.OptionParser() - parser.add_option('-s', '--seed', dest='seed', action='store', type="string", default=None, help='Set the random seed.') + parser.add_option( + "-s", "--seed", dest="seed", action="store", type="string", default=None, help="Set the random seed." + ) (options, args) = parser.parse_args() assert len(args) == 3, "Invalid command line specified." - with open(args[0], 'rb') as input, open(args[1], 'wb') as output: + with open(args[0], "rb") as input, open(args[1], "wb") as output: num_lines = int(args[2]) assert num_lines > 0, "You must select at least one line." @@ -114,7 +116,10 @@ def __main__(): break total_lines = len(line_offsets) - assert num_lines <= total_lines, "Error: asked to select more lines (%i) than there were in the file (%i)." % (num_lines, total_lines) + assert num_lines <= total_lines, "Error: asked to select more lines (%i) than there were in the file (%i)." % ( + num_lines, + total_lines, + ) # get random line offsets line_offsets = get_random(line_offsets, num_lines) diff --git a/tools/filters/randomlines.py b/tools/filters/randomlines.py index 8cc75734d28..152f98d125a 100644 --- a/tools/filters/randomlines.py +++ b/tools/filters/randomlines.py @@ -19,7 +19,7 @@ def main(): for line in infile: line = line.rstrip("\n") n += 1 - if (n <= total_lines): + if n <= total_lines: kept.append(line) elif random.randint(1, n) <= total_lines: kept.pop(random.randint(0, total_lines - 1)) @@ -29,7 +29,7 @@ def main(): sys.stderr.write("Error: asked to select more lines than there were in the file.") sys.exit() - open(sys.argv[3], 'w').write("\n".join(kept)) + open(sys.argv[3], "w").write("\n".join(kept)) if __name__ == "__main__": diff --git a/tools/filters/secure_hash_message_digest.py b/tools/filters/secure_hash_message_digest.py index 8bd7c596542..6834fafff68 100644 --- a/tools/filters/secure_hash_message_digest.py +++ b/tools/filters/secure_hash_message_digest.py @@ -7,16 +7,23 @@ import hashlib import optparse from collections import OrderedDict -HASH_ALGORITHMS = ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'] -CHUNK_SIZE = 2 ** 20 # 1mb +HASH_ALGORITHMS = ["md5", "sha1", "sha224", "sha256", "sha384", "sha512"] +CHUNK_SIZE = 2**20 # 1mb def __main__(): # Parse Command Line parser = optparse.OptionParser() - parser.add_option('-a', '--algorithm', dest='algorithms', action='append', type="string", help='Algorithms to use, eg. (md5, sha1, sha224, sha256, sha384, sha512)') - parser.add_option('-i', '--input', dest='input', action='store', type="string", help='Input filename') - parser.add_option('-o', '--output', dest='output', action='store', type="string", help='Output filename') + parser.add_option( + "-a", + "--algorithm", + dest="algorithms", + action="append", + type="string", + help="Algorithms to use, eg. (md5, sha1, sha224, sha256, sha384, sha512)", + ) + parser.add_option("-i", "--input", dest="input", action="store", type="string", help="Input filename") + parser.add_option("-o", "--output", dest="output", action="store", type="string", help="Output filename") (options, args) = parser.parse_args() algorithms = OrderedDict() @@ -28,7 +35,7 @@ def __main__(): assert options.input, "You must provide an input filename." assert options.output, "You must provide an output filename." - with open(options.input, 'rb') as fh: + with open(options.input, "rb") as fh: while True: chunk = fh.read(CHUNK_SIZE) if chunk: @@ -37,9 +44,9 @@ def __main__(): else: break - with open(options.output, 'w') as output: - output.write('#%s\n' % ('\t'.join(algorithms.keys()))) - output.write('%s\n' % ('\t'.join(x.hexdigest() for x in algorithms.values()))) + with open(options.output, "w") as output: + output.write("#%s\n" % ("\t".join(algorithms.keys()))) + output.write("%s\n" % ("\t".join(x.hexdigest() for x in algorithms.values()))) if __name__ == "__main__": diff --git a/tools/filters/sff_extract.py b/tools/filters/sff_extract.py index bed37395c23..81d52ff035d 100644 --- a/tools/filters/sff_extract.py +++ b/tools/filters/sff_extract.py @@ -1,10 +1,10 @@ #!/usr/bin/env python -'''This software extracts the seq, qual and ancillary information from an sff +"""This software extracts the seq, qual and ancillary information from an sff file, like the ones used by the 454 sequencer. Optionally, it can also split paired-end reads if given the linker sequence. The splitting is done with maximum match, i.e., every occurence of the linker -sequence will be removed, even if occuring multiple times.''' +sequence will be removed, even if occuring multiple times.""" # copyright Jose Blanca and Bastien Chevreux # COMAV institute, Universidad Politecnica de Valencia (UPV) @@ -36,14 +36,14 @@ from typing import ( List, ) -__author__ = 'Jose Blanca and Bastien Chevreux' -__copyright__ = 'Copyright 2008, Jose Blanca, COMAV, and Bastien Chevreux' -__license__ = 'GPLv3 or later' -__version__ = '0.2.10' -__email__ = 'jblanca@btc.upv.es' -__status__ = 'beta' +__author__ = "Jose Blanca and Bastien Chevreux" +__copyright__ = "Copyright 2008, Jose Blanca, COMAV, and Bastien Chevreux" +__license__ = "GPLv3 or later" +__version__ = "0.2.10" +__email__ = "jblanca@btc.upv.es" +__status__ = "beta" -fake_sff_name = 'fake_sff_name' +fake_sff_name = "fake_sff_name" # readname as key: lines with matches from SSAHA, one best match ssahapematches: Dict[str, List] = {} @@ -55,7 +55,7 @@ stern_warning = True def read_bin_fragment(struct_def, fileh, offset=0, data=None, byte_padding=None): - '''It reads a chunk of a binary file. + """It reads a chunk of a binary file. You have to provide the struct, a file object, the offset (where to start reading). @@ -64,7 +64,7 @@ def read_bin_fragment(struct_def, fileh, offset=0, data=None, byte_padding=None) If a byte_padding is given the number of bytes read will be a multiple of that number, adding the required pad at the end. It returns the number of bytes reads and the data dict. - ''' + """ if data is None: data = {} @@ -75,7 +75,7 @@ def read_bin_fragment(struct_def, fileh, offset=0, data=None, byte_padding=None) fileh.seek(offset + bytes_read) n_bytes = struct.calcsize(item[1]) buffer = fileh.read(n_bytes) - read = struct.unpack('>' + item[1], buffer) + read = struct.unpack(">" + item[1], buffer) if len(read) == 1: read = read[0] data[item[0]] = read @@ -91,106 +91,105 @@ def read_bin_fragment(struct_def, fileh, offset=0, data=None, byte_padding=None) def check_magic(magic): - '''It checks that the magic number of the file matches the sff magic.''' + """It checks that the magic number of the file matches the sff magic.""" if magic != 779314790: - raise RuntimeError('This file does not seems to be an sff file.') + raise RuntimeError("This file does not seems to be an sff file.") def check_version(version): - '''It checks that the version is supported, otherwise it raises an error.''' - if version != ('\x00', '\x00', '\x00', '\x01'): - raise RuntimeError('SFF version not supported. Please contact the author of the software.') + """It checks that the version is supported, otherwise it raises an error.""" + if version != ("\x00", "\x00", "\x00", "\x01"): + raise RuntimeError("SFF version not supported. Please contact the author of the software.") def read_header(fileh): - '''It reads the header from the sff file and returns a dict with the - information''' + """It reads the header from the sff file and returns a dict with the + information""" # first we read the first part of the header head_struct = [ - ('magic_number', 'I'), - ('version', 'cccc'), - ('index_offset', 'Q'), - ('index_length', 'I'), - ('number_of_reads', 'I'), - ('header_length', 'H'), - ('key_length', 'H'), - ('number_of_flows_per_read', 'H'), - ('flowgram_format_code', 'B'), + ("magic_number", "I"), + ("version", "cccc"), + ("index_offset", "Q"), + ("index_length", "I"), + ("number_of_reads", "I"), + ("header_length", "H"), + ("key_length", "H"), + ("number_of_flows_per_read", "H"), + ("flowgram_format_code", "B"), ] data = {} - first_bytes, data = read_bin_fragment(struct_def=head_struct, fileh=fileh, - offset=0, data=data) - check_magic(data['magic_number']) - check_version(data['version']) + first_bytes, data = read_bin_fragment(struct_def=head_struct, fileh=fileh, offset=0, data=data) + check_magic(data["magic_number"]) + check_version(data["version"]) # now that we know the number_of_flows_per_read and the key_length # we can read the second part of the header struct2 = [ - ('flow_chars', str(data['number_of_flows_per_read']) + 'c'), - ('key_sequence', str(data['key_length']) + 'c') + ("flow_chars", str(data["number_of_flows_per_read"]) + "c"), + ("key_sequence", str(data["key_length"]) + "c"), ] read_bin_fragment(struct_def=struct2, fileh=fileh, offset=first_bytes, data=data) return data def read_sequence(header, fileh, fposition): - '''It reads one read from the sff file located at the fposition and - returns a dict with the information.''' + """It reads one read from the sff file located at the fposition and + returns a dict with the information.""" # the sequence struct read_header_1 = [ - ('read_header_length', 'H'), - ('name_length', 'H'), - ('number_of_bases', 'I'), - ('clip_qual_left', 'H'), - ('clip_qual_right', 'H'), - ('clip_adapter_left', 'H'), - ('clip_adapter_right', 'H'), + ("read_header_length", "H"), + ("name_length", "H"), + ("number_of_bases", "I"), + ("clip_qual_left", "H"), + ("clip_qual_right", "H"), + ("clip_adapter_left", "H"), + ("clip_adapter_right", "H"), ] def read_header_2(name_length): - '''It returns the struct definition for the second part of the header''' - return [('name', str(name_length) + 'c')] + """It returns the struct definition for the second part of the header""" + return [("name", str(name_length) + "c")] def read_data(number_of_bases): - '''It returns the struct definition for the read data section.''' - if header['flowgram_format_code'] == 1: - flow_type = 'H' + """It returns the struct definition for the read data section.""" + if header["flowgram_format_code"] == 1: + flow_type = "H" else: - raise Exception('file version not supported') + raise Exception("file version not supported") number_of_bases = str(number_of_bases) return [ - ('flowgram_values', str(header['number_of_flows_per_read']) + flow_type), - ('flow_index_per_base', number_of_bases + 'B'), - ('bases', number_of_bases + 'c'), - ('quality_scores', number_of_bases + 'B'), + ("flowgram_values", str(header["number_of_flows_per_read"]) + flow_type), + ("flow_index_per_base", number_of_bases + "B"), + ("bases", number_of_bases + "c"), + ("quality_scores", number_of_bases + "B"), ] data = {} # we read the first part of the header - bytes_read, data = read_bin_fragment(struct_def=read_header_1, - fileh=fileh, offset=fposition, data=data) + bytes_read, data = read_bin_fragment(struct_def=read_header_1, fileh=fileh, offset=fposition, data=data) - read_bin_fragment(struct_def=read_header_2(data['name_length']), - fileh=fileh, offset=fposition + bytes_read, data=data) + read_bin_fragment( + struct_def=read_header_2(data["name_length"]), fileh=fileh, offset=fposition + bytes_read, data=data + ) # we join the letters of the name - data['name'] = ''.join(data['name']) - offset = data['read_header_length'] + data["name"] = "".join(data["name"]) + offset = data["read_header_length"] # we read the sequence and the quality - read_data_st = read_data(data['number_of_bases']) - bytes_read, data = read_bin_fragment(struct_def=read_data_st, - fileh=fileh, offset=fposition + offset, - data=data, byte_padding=8) + read_data_st = read_data(data["number_of_bases"]) + bytes_read, data = read_bin_fragment( + struct_def=read_data_st, fileh=fileh, offset=fposition + offset, data=data, byte_padding=8 + ) # we join the bases - data['bases'] = ''.join(data['bases']) + data["bases"] = "".join(data["bases"]) # correct for the case the right clip is <= than the left clip # in this case, left clip is 0 are set to 0 (right clip == 0 means # "whole sequence") - if data['clip_qual_right'] <= data['clip_qual_left']: - data['clip_qual_right'] = 0 - data['clip_qual_left'] = 0 - if data['clip_adapter_right'] <= data['clip_adapter_left']: - data['clip_adapter_right'] = 0 - data['clip_adapter_left'] = 0 + if data["clip_qual_right"] <= data["clip_qual_left"]: + data["clip_qual_right"] = 0 + data["clip_qual_left"] = 0 + if data["clip_adapter_right"] <= data["clip_adapter_left"]: + data["clip_adapter_right"] = 0 + data["clip_adapter_left"] = 0 # the clipping section follows the NCBI's guidelines Trace Archive RFC # http://www.ncbi.nlm.nih.gov/Traces/trace.cgi?cmd=show&f=rfc&m=doc&s=rfc @@ -198,54 +197,53 @@ def read_sequence(header, fileh, fposition): # else: qual-> qual # adapter -> vector - if not data['clip_adapter_left']: - data['clip_adapter_left'], data['clip_qual_left'] = data['clip_qual_left'], data['clip_adapter_left'] - if not data['clip_adapter_right']: - data['clip_adapter_right'], data['clip_qual_right'] = data['clip_qual_right'], data['clip_adapter_right'] + if not data["clip_adapter_left"]: + data["clip_adapter_left"], data["clip_qual_left"] = data["clip_qual_left"], data["clip_adapter_left"] + if not data["clip_adapter_right"]: + data["clip_adapter_right"], data["clip_qual_right"] = data["clip_qual_right"], data["clip_adapter_right"] # see whether we have to override the minimum left clips - if config['min_leftclip'] > 0: - if data['clip_adapter_left'] > 0 and data['clip_adapter_left'] < config['min_leftclip']: - data['clip_adapter_left'] = config['min_leftclip'] - if data['clip_qual_left'] > 0 and data['clip_qual_left'] < config['min_leftclip']: - data['clip_qual_left'] = config['min_leftclip'] + if config["min_leftclip"] > 0: + if data["clip_adapter_left"] > 0 and data["clip_adapter_left"] < config["min_leftclip"]: + data["clip_adapter_left"] = config["min_leftclip"] + if data["clip_qual_left"] > 0 and data["clip_qual_left"] < config["min_leftclip"]: + data["clip_qual_left"] = config["min_leftclip"] # for handling the -c (clip) option gently, we already clip here # and set all clip points to the sequence end points - if config['clip']: - data['bases'], data['quality_scores'] = clip_read(data) + if config["clip"]: + data["bases"], data["quality_scores"] = clip_read(data) - data['number_of_bases'] = len(data['bases']) - data['clip_qual_right'] = data['number_of_bases'] - data['clip_adapter_right'] = data['number_of_bases'] - data['clip_qual_left'] = 0 - data['clip_adapter_left'] = 0 + data["number_of_bases"] = len(data["bases"]) + data["clip_qual_right"] = data["number_of_bases"] + data["clip_adapter_right"] = data["number_of_bases"] + data["clip_qual_left"] = 0 + data["clip_adapter_left"] = 0 - return data['read_header_length'] + bytes_read, data + return data["read_header_length"] + bytes_read, data def sequences(fileh, header): - '''It returns a generator with the data for each read.''' + """It returns a generator with the data for each read.""" # now we can read all the sequences - fposition = header['header_length'] # position in the file + fposition = header["header_length"] # position in the file reads_read = 0 while True: - if fposition == header['index_offset']: + if fposition == header["index_offset"]: # we have to skip the index section - fposition += header['index_length'] + fposition += header["index_length"] continue else: - bytes_read, seq_data = read_sequence(header=header, fileh=fileh, - fposition=fposition) + bytes_read, seq_data = read_sequence(header=header, fileh=fileh, fposition=fposition) yield seq_data fposition += bytes_read reads_read += 1 - if reads_read >= header['number_of_reads']: + if reads_read >= header["number_of_reads"]: break def remove_last_xmltag_in_file(fname, tag=None): - '''Given an xml file name and a tag, it removes the last tag of the + """Given an xml file name and a tag, it removes the last tag of the file if it matches the given tag. Tag removal is performed via file truncation. @@ -254,9 +252,9 @@ def remove_last_xmltag_in_file(fname, tag=None): The resulting xml file will be not xml valid. This function is a hack that allows to append records to xml files in a quick and dirty way. - ''' + """ - fh = open(fname, 'r+') + fh = open(fname, "r+") # we have to read from the end to the start of the file and keep the # string enclosed by i = -1 @@ -266,18 +264,18 @@ def remove_last_xmltag_in_file(fname, tag=None): char = fh.read(1) if not char.isspace(): last_tag.append(char) - if char == '<': + if char == "<": break i -= 1 # we have read the last tag backwards - last_tag = ''.join(last_tag[::-1]) + last_tag = "".join(last_tag[::-1]) # we remove the - last_tag = last_tag.rstrip('>').lstrip('").lstrip("\n'] - to_print.append(' ') + """ + to_print = [" \n"] + to_print.append(" ") to_print.append(readname) - to_print.append('\n') + to_print.append("\n") # extra information # do we have extra info for this file? info = None - if config['xml_info']: + if config["xml_info"]: # with this name? - if fname in config['xml_info']: - info = config['xml_info'][fname] + if fname in config["xml_info"]: + info = config["xml_info"][fname] else: # with no name? try: - info = config['xml_info'][fake_sff_name] + info = config["xml_info"][fake_sff_name] except KeyError: pass # we print the info that we have if info: for key in info: - to_print.append(' <' + key + '>' + info[key] - + '\n') + to_print.append(" <" + key + ">" + info[key] + "\n") - return ''.join(to_print) + return "".join(to_print) def create_clip_xml_info(readlen, adapl, adapr, quall, qualr): - '''Takes the clip values of the read and formats them into XML + """Takes the clip values of the read and formats them into XML Corrects "wrong" values that might have resulted through simplified calculations earlier in the process of conversion (especially during splitting of paired-end reads) - ''' + """ to_print = [""] @@ -361,76 +358,84 @@ def create_clip_xml_info(readlen, adapl, adapr, quall, qualr): quall = 0 if quall: - to_print.append(' ') + to_print.append(" ") to_print.append(str(quall)) - to_print.append('\n') + to_print.append("\n") if qualr: - to_print.append(' ') + to_print.append(" ") to_print.append(str(qualr)) - to_print.append('\n') + to_print.append("\n") if adapl: - to_print.append(' ') + to_print.append(" ") to_print.append(str(adapl)) - to_print.append('\n') + to_print.append("\n") if adapr: - to_print.append(' ') + to_print.append(" ") to_print.append(str(adapr)) - to_print.append('\n') - return ''.join(to_print) + to_print.append("\n") + return "".join(to_print) def create_xml_for_unpaired_read(data, fname): - '''Given the data for one read it returns an str with the xml ancillary - data.''' - to_print = [create_basic_xml_info(data['name'], fname)] + """Given the data for one read it returns an str with the xml ancillary + data.""" + to_print = [create_basic_xml_info(data["name"], fname)] # clippings in the XML only if we do not hard clip - if not config['clip']: - to_print.append(create_clip_xml_info(data['number_of_bases'], data['clip_adapter_left'], data['clip_adapter_right'], data['clip_qual_left'], data['clip_qual_right'])) - to_print.append(' \n') - return ''.join(to_print) + if not config["clip"]: + to_print.append( + create_clip_xml_info( + data["number_of_bases"], + data["clip_adapter_left"], + data["clip_adapter_right"], + data["clip_qual_left"], + data["clip_qual_right"], + ) + ) + to_print.append(" \n") + return "".join(to_print) def format_as_fasta(name, seq, qual): - name_line = ''.join(('>', name, '\n')) - seqstring = ''.join((name_line, seq, '\n')) - qual_line = ' '.join(str(q) for q in qual) - qualstring = ''.join((name_line, qual_line, '\n')) + name_line = "".join((">", name, "\n")) + seqstring = "".join((name_line, seq, "\n")) + qual_line = " ".join(str(q) for q in qual) + qualstring = "".join((name_line, qual_line, "\n")) return seqstring, qualstring def format_as_fastq(name, seq, qual): - qual_line = ''.join(chr(q + 33) for q in qual) - seqstring = ''.join(('@', name, '\n', seq, '\n+\n', qual_line, '\n')) + qual_line = "".join(chr(q + 33) for q in qual) + seqstring = "".join(("@", name, "\n", seq, "\n+\n", qual_line, "\n")) return seqstring def get_read_data(data): - '''Given the data for one read it returns 2 strs with the fasta seq - and fasta qual.''' + """Given the data for one read it returns 2 strs with the fasta seq + and fasta qual.""" # seq and qual - if config['mix_case']: + if config["mix_case"]: seq = sequence_case(data) - qual = data['quality_scores'] + qual = data["quality_scores"] else: - seq = data['bases'] - qual = data['quality_scores'] + seq = data["bases"] + qual = data["quality_scores"] return seq, qual def extract_read_info(data, fname): - '''Given the data for one read it returns 3 strs with the fasta seq, fasta - qual and xml ancillary data.''' + """Given the data for one read it returns 3 strs with the fasta seq, fasta + qual and xml ancillary data.""" seq, qual = get_read_data(data) - seqstring, qualstring = format_as_fasta(data['name'], seq, qual) + seqstring, qualstring = format_as_fasta(data["name"], seq, qual) xmlstring = create_xml_for_unpaired_read(data, fname) return seqstring, qualstring, xmlstring def write_sequence(name, seq, qual, seq_fh, qual_fh): - '''Write sequence and quality FASTA and FASTA qual filehandles + """Write sequence and quality FASTA and FASTA qual filehandles (or into FASTQ and XML) - if sequence length is 0, don't write''' + if sequence length is 0, don't write""" if len(seq) == 0: return @@ -444,14 +449,14 @@ def write_sequence(name, seq, qual, seq_fh, qual_fh): def write_unpaired_read(data, sff_fh, seq_fh, qual_fh, xml_fh): - '''Writes an unpaired read into FASTA, FASTA qual and XML filehandles + """Writes an unpaired read into FASTA, FASTA qual and XML filehandles (or into FASTQ and XML) - if sequence length is 0, don't write''' + if sequence length is 0, don't write""" seq, qual = get_read_data(data) if len(seq) == 0: return - write_sequence(data['name'], seq, qual, seq_fh, qual_fh) + write_sequence(data["name"], seq, qual, seq_fh, qual_fh) anci = create_xml_for_unpaired_read(data, sff_fh.name) if anci is not None: @@ -460,54 +465,55 @@ def write_unpaired_read(data, sff_fh, seq_fh, qual_fh, xml_fh): def reverse_complement(seq): - '''Returns the reverse complement of a DNA sequence as string''' + """Returns the reverse complement of a DNA sequence as string""" compdict = { - 'a': 't', - 'c': 'g', - 'g': 'c', - 't': 'a', - 'u': 't', - 'm': 'k', - 'r': 'y', - 'w': 'w', - 's': 's', - 'y': 'r', - 'k': 'm', - 'v': 'b', - 'h': 'd', - 'd': 'h', - 'b': 'v', - 'x': 'x', - 'n': 'n', - 'A': 'T', - 'C': 'G', - 'G': 'C', - 'T': 'A', - 'U': 'T', - 'M': 'K', - 'R': 'Y', - 'W': 'W', - 'S': 'S', - 'Y': 'R', - 'K': 'M', - 'V': 'B', - 'H': 'D', - 'D': 'H', - 'B': 'V', - 'X': 'X', - 'N': 'N', - '*': '*'} + "a": "t", + "c": "g", + "g": "c", + "t": "a", + "u": "t", + "m": "k", + "r": "y", + "w": "w", + "s": "s", + "y": "r", + "k": "m", + "v": "b", + "h": "d", + "d": "h", + "b": "v", + "x": "x", + "n": "n", + "A": "T", + "C": "G", + "G": "C", + "T": "A", + "U": "T", + "M": "K", + "R": "Y", + "W": "W", + "S": "S", + "Y": "R", + "K": "M", + "V": "B", + "H": "D", + "D": "H", + "B": "V", + "X": "X", + "N": "N", + "*": "*", + } - complseq = ''.join(compdict[base] for base in seq) + complseq = "".join(compdict[base] for base in seq) # python hack to reverse a list/string/etc complseq = complseq[::-1] return complseq def mask_sequence(seq, maskchar, fpos, tpos): - '''Given a sequence, mask it with maskchar starting at fpos (including) and + """Given a sequence, mask it with maskchar starting at fpos (including) and ending at tpos (excluding) - ''' + """ if len(maskchar) > 1: raise RuntimeError("Internal error: more than one character given to mask_sequence") if fpos < 0: @@ -515,16 +521,16 @@ def mask_sequence(seq, maskchar, fpos, tpos): if tpos > len(seq): tpos = len(seq) - newseq = ''.join((seq[:fpos], maskchar * (tpos - fpos), seq[tpos:])) + newseq = "".join((seq[:fpos], maskchar * (tpos - fpos), seq[tpos:])) return newseq def fragment_sequences(sequence, qualities, splitchar): - '''Works like split() on strings, except it does this on a sequence + """Works like split() on strings, except it does this on a sequence and the corresponding list with quality values. Returns a tuple for each fragment, each sublist has the fragment - sequence as first and the fragment qualities as second elemnt''' + sequence as first and the fragment qualities as second elemnt""" # this is slow (due to zip and list appends... use an iterator over # the sequence find find variations and splices on seq and qual @@ -532,12 +538,12 @@ def fragment_sequences(sequence, qualities, splitchar): print(sequence, qualities) raise RuntimeError("Internal error: length of sequence and qualities don't match???") - retlist = ([]) + retlist = [] if len(sequence) == 0: return retlist - actseq = ([]) - actqual = ([]) + actseq = [] + actqual = [] if sequence[0] != splitchar: inseq = True else: @@ -548,9 +554,9 @@ def fragment_sequences(sequence, qualities, splitchar): actseq.append(char) actqual.append(qual) else: - retlist.append((''.join(actseq), actqual)) - actseq = ([]) - actqual = ([]) + retlist.append(("".join(actseq), actqual)) + actseq = [] + actqual = [] inseq = False else: if char != splitchar: @@ -559,18 +565,18 @@ def fragment_sequences(sequence, qualities, splitchar): actqual.append(qual) if inseq and len(actseq): - retlist.append((''.join(actseq), actqual)) + retlist.append(("".join(actseq), actqual)) return retlist def calc_subseq_boundaries(maskedseq, maskchar): - '''E.g.: - ........xxxxxxxx..........xxxxxxxxxxxxxxxxxxxxx......... - to - (0,8),(8,16),(16,26),(26,47),(47,56) - ''' - blist = ([]) + """E.g.: + ........xxxxxxxx..........xxxxxxxxxxxxxxxxxxxxx......... + to + (0,8),(8,16),(16,26),(26,47),(47,56) + """ + blist = [] if len(maskedseq) == 0: return blist @@ -595,11 +601,11 @@ def calc_subseq_boundaries(maskedseq, maskchar): def correct_for_smallhits(maskedseq, maskchar, linkername): - '''If partial hits were found, take preventive measure: grow - the masked areas by 20 bases in each direction - Returns either unchanged "maskedseq" or a new sequence - with some more characters masked. - ''' + """If partial hits were found, take preventive measure: grow + the masked areas by 20 bases in each direction + Returns either unchanged "maskedseq" or a new sequence + with some more characters masked. + """ global linkerlengths if len(maskedseq) == 0: @@ -652,7 +658,7 @@ def correct_for_smallhits(maskedseq, maskchar, linkername): def split_paired_end(data, sff_fh, seq_fh, qual_fh, xml_fh): - '''Splits a paired end read and writes sequences into FASTA, FASTA qual + """Splits a paired end read and writes sequences into FASTA, FASTA qual and XML traceinfo file. Returns the number of sequences created. As the linker sequence may be anywhere in the read, including the ends @@ -679,14 +685,14 @@ def split_paired_end(data, sff_fh, seq_fh, qual_fh, xml_fh): For multiple or partial linker, the "good" parts of the reads are stored with a ".part" name, additionally they will not get template information in the XML - ''' + """ global ssahapematches maskchar = "#" numseqs = 0 - readname = data['name'] - readlen = data['number_of_bases'] + readname = data["name"] + readlen = data["number_of_bases"] leftclip, rightclip = return_merged_clips(data) seq, qual = get_read_data(data) @@ -698,9 +704,9 @@ def split_paired_end(data, sff_fh, seq_fh, qual_fh, xml_fh): maskedseq = mask_sequence(maskedseq, maskchar, rightclip, len(maskedseq)) leftclip, rightclip = return_merged_clips(data) - readlen = data['number_of_bases'] + readlen = data["number_of_bases"] - for match in ssahapematches[data['name']]: + for match in ssahapematches[data["name"]]: int(match[0]) linkername = match[2] leftreadhit = int(match[3]) @@ -745,13 +751,15 @@ def split_paired_end(data, sff_fh, seq_fh, qual_fh, xml_fh): # only two, the fact we had multiple linkers # says something went wrong, so simply do not # write any paired-end information for all these fragments - to_print.append(' \n') - xml_fh.write(''.join(to_print)) + to_print.append(" \n") + xml_fh.write("".join(to_print)) numseqs += 1 fragcounter += 1 else: if len(fragments) > 2: - raise RuntimeError("Unexpected: more than two fragments detected in " + readname + ". please contact the authors.") + raise RuntimeError( + "Unexpected: more than two fragments detected in " + readname + ". please contact the authors." + ) # nothing will happen for 0 fragments if len(fragments) == 1: boundaries = calc_subseq_boundaries(maskedseq, maskchar) @@ -759,17 +767,17 @@ def split_paired_end(data, sff_fh, seq_fh, qual_fh, xml_fh): raise RuntimeError("Unexpected case: ", str(len(boundaries)), "boundaries for 1 fragment of ", readname) if len(boundaries) == 3: # case: mask char on both sides of sequence - data['clip_adapter_left'] = boundaries[0][1] - data['clip_adapter_right'] = boundaries[2][0] + data["clip_adapter_left"] = boundaries[0][1] + data["clip_adapter_right"] = boundaries[2][0] elif len(boundaries) == 2: # case: mask char left or right of sequence if maskedseq[0] == maskchar: # case: mask char left - data['clip_adapter_left'] = boundaries[0][1] + data["clip_adapter_left"] = boundaries[0][1] else: # case: mask char right - data['clip_adapter_right'] = boundaries[1][0] - data['name'] = data['name'] + ".fn" + data["clip_adapter_right"] = boundaries[1][0] + data["name"] = data["name"] + ".fn" write_unpaired_read(data, sff_fh, seq_fh, qual_fh, xml_fh) numseqs = 1 elif len(fragments) == 2: @@ -794,13 +802,17 @@ def split_paired_end(data, sff_fh, seq_fh, qual_fh, xml_fh): write_sequence(oname, actseq, lqual, seq_fh, qual_fh) to_print = [create_basic_xml_info(oname, sff_fh.name)] - to_print.append(create_clip_xml_info(lreadlen, 0, lreadlen + 1 - data['clip_adapter_left'], 0, lreadlen + 1 - data['clip_qual_left'])) - to_print.append(' ') + to_print.append( + create_clip_xml_info( + lreadlen, 0, lreadlen + 1 - data["clip_adapter_left"], 0, lreadlen + 1 - data["clip_qual_left"] + ) + ) + to_print.append(" ") to_print.append(readname) - to_print.append('\n') - to_print.append(' r\n') - to_print.append(' \n') - xml_fh.write(''.join(to_print)) + to_print.append("\n") + to_print.append(" r\n") + to_print.append(" \n") + xml_fh.write("".join(to_print)) oname = readname + ".f" startsearch = False @@ -811,33 +823,41 @@ def split_paired_end(data, sff_fh, seq_fh, qual_fh, xml_fh): if startsearch: break - actseq = seq[spos + 1:] - actqual = qual[spos + 1:] + actseq = seq[spos + 1 :] + actqual = qual[spos + 1 :] write_sequence(oname, actseq, actqual, seq_fh, qual_fh) rreadlen = len(actseq) to_print = [create_basic_xml_info(oname, sff_fh.name)] - to_print.append(create_clip_xml_info(rreadlen, 0, rreadlen - (readlen - data['clip_adapter_right']), 0, rreadlen - (readlen - data['clip_qual_right']))) - to_print.append(' ') + to_print.append( + create_clip_xml_info( + rreadlen, + 0, + rreadlen - (readlen - data["clip_adapter_right"]), + 0, + rreadlen - (readlen - data["clip_qual_right"]), + ) + ) + to_print.append(" ") to_print.append(readname) - to_print.append('\n') - to_print.append(' f\n') - to_print.append(' \n') - xml_fh.write(''.join(to_print)) + to_print.append("\n") + to_print.append(" f\n") + to_print.append(" \n") + xml_fh.write("".join(to_print)) numseqs = 2 return numseqs def extract_reads_from_sff(config, sff_files): - '''Given the configuration and the list of sff_files it writes the seqs, + """Given the configuration and the list of sff_files it writes the seqs, qualities and ancillary data into the output file(s). If file for paired-end linker was given, first extracts all sequences of an SFF and searches these against the linker(s) with SSAHA2 to create needed information to split reads. - ''' + """ global ssahapematches if len(sff_files) == 0: @@ -846,67 +866,65 @@ def extract_reads_from_sff(config, sff_files): # we go through all input files for sff_file in sff_files: if not os.path.getsize(sff_file): - raise RuntimeError('Empty file? : ' + sff_file) + raise RuntimeError("Empty file? : " + sff_file) fh = open(sff_file) fh.close() - openmode = 'w' - if config['append']: - openmode = 'a' + openmode = "w" + if config["append"]: + openmode = "a" - seq_fh = open(config['seq_fname'], openmode) - xml_fh = open(config['xml_fname'], openmode) - if config['want_fastq']: + seq_fh = open(config["seq_fname"], openmode) + xml_fh = open(config["xml_fname"], openmode) + if config["want_fastq"]: qual_fh = None try: - os.remove(config['qual_fname']) + os.remove(config["qual_fname"]) except Exception: pass else: - qual_fh = open(config['qual_fname'], openmode) + qual_fh = open(config["qual_fname"], openmode) - if not config['append']: + if not config["append"]: xml_fh.write('\n\n') else: - remove_last_xmltag_in_file(config['xml_fname'], "trace_volume") + remove_last_xmltag_in_file(config["xml_fname"], "trace_volume") # we go through all input files for sff_file in sff_files: ssahapematches.clear() - seqcheckstore = ([]) + seqcheckstore = [] debug = 0 - if not debug and config['pelinker_fname']: + if not debug and config["pelinker_fname"]: sys.stdout.flush() if 0: # for debugging pid = os.getpid() - tmpfasta_fname = 'sffe.tmp.' + str(pid) + '.fasta' - tmpfasta_fh = open(tmpfasta_fname, 'w') + tmpfasta_fname = "sffe.tmp." + str(pid) + ".fasta" + tmpfasta_fh = open(tmpfasta_fname, "w") else: - tmpfasta_fh = tempfile.NamedTemporaryFile(prefix='sffeseqs_', - suffix='.fasta') + tmpfasta_fh = tempfile.NamedTemporaryFile(prefix="sffeseqs_", suffix=".fasta") - sff_fh = open(sff_file, 'rb') + sff_fh = open(sff_file, "rb") header_data = read_header(fileh=sff_fh) for seq_data in sequences(fileh=sff_fh, header=header_data): seq, qual = get_read_data(seq_data) - seqstring, qualstring = format_as_fasta(seq_data['name'], seq, qual) + seqstring, qualstring = format_as_fasta(seq_data["name"], seq, qual) tmpfasta_fh.write(seqstring) tmpfasta_fh.seek(0) if 0: # for debugging - tmpssaha_fname = 'sffe.tmp.' + str(pid) + '.ssaha2' - tmpssaha_fh = open(tmpssaha_fname, 'w+') + tmpssaha_fname = "sffe.tmp." + str(pid) + ".ssaha2" + tmpssaha_fh = open(tmpssaha_fname, "w+") else: - tmpssaha_fh = tempfile.NamedTemporaryFile(prefix='sffealig_', - suffix='.ssaha2') + tmpssaha_fh = tempfile.NamedTemporaryFile(prefix="sffealig_", suffix=".ssaha2") - launch_ssaha(config['pelinker_fname'], tmpfasta_fh.name, tmpssaha_fh) + launch_ssaha(config["pelinker_fname"], tmpfasta_fh.name, tmpssaha_fh) tmpfasta_fh.close() tmpssaha_fh.seek(0) @@ -918,7 +936,7 @@ def extract_reads_from_sff(config, sff_files): read_ssaha_data(tmpssaha_fh) sys.stdout.flush() - sff_fh = open(sff_file, 'rb') + sff_fh = open(sff_file, "rb") header_data = read_header(fileh=sff_fh) # now convert all reads @@ -930,19 +948,19 @@ def extract_reads_from_sff(config, sff_files): seq, qual = clip_read(seq_data) seqcheckstore.append(seq[0:50]) - if seq_data['name'] in ssahapematches: + if seq_data["name"] in ssahapematches: nseqs_out += split_paired_end(seq_data, sff_fh, seq_fh, qual_fh, xml_fh) else: - if config['pelinker_fname']: - seq_data['name'] = seq_data['name'] + ".fn" + if config["pelinker_fname"]: + seq_data["name"] = seq_data["name"] + ".fn" write_unpaired_read(seq_data, sff_fh, seq_fh, qual_fh, xml_fh) nseqs_out += 1 sff_fh.close() check_for_dubious_startseq(seqcheckstore, sff_file, seq_data) - seqcheckstore = ([]) + seqcheckstore = [] - xml_fh.write('\n') + xml_fh.write("\n") xml_fh.close() seq_fh.close() @@ -974,14 +992,18 @@ def check_for_dubious_startseq(seqcheckstore, sffname, seqdata): foundproblem += "\nWARNING: " foundproblem += "weird sequences in file " + sffname + "\n\n" foundproblem += "After applying left clips, " + str(count) + " sequences (=" - foundproblem += '%.0f' % (100.0 * float(count) / len(seqcheckstore)) + foundproblem += "%.0f" % (100.0 * float(count) / len(seqcheckstore)) foundproblem += "%) start with these bases:\n" + shortseq foundproblem += "\n\nThis does not look sane.\n\n" foundproblem += "Countermeasures you *probably* must take:\n" foundproblem += "1) Make your sequence provider aware of that problem and ask whether this can be\n corrected in the SFF.\n" foundproblem += "2) If you decide that this is not normal and your sequence provider does not\n react, use the --min_left_clip of sff_extract.\n" left, right = return_merged_clips(seqdata) - foundproblem += " (Probably '--min_left_clip=" + str(left + len(shortseq)) + "' but you should cross-check that)\n" + foundproblem += ( + " (Probably '--min_left_clip=" + + str(left + len(shortseq)) + + "' but you should cross-check that)\n" + ) foundproblem += "*" * 80 + "\n" if not foundinloop: break @@ -990,20 +1012,20 @@ def check_for_dubious_startseq(seqcheckstore, sffname, seqdata): def parse_extra_info(info): - '''It parses the information that will go in the xml file. + """It parses the information that will go in the xml file. There are two formats accepted for the extra information: key1:value1, key2:value2 or: file1.sff{key1:value1, key2:value2};file2.sff{key3:value3} - ''' + """ if not info: return info - finfos = info.split(';') # information for each file + finfos = info.split(";") # information for each file data_for_files = {} for finfo in finfos: # we split the file name from the rest - items = finfo.split('{') + items = finfo.split("{") if len(items) == 1: fname = fake_sff_name info = items[0] @@ -1011,10 +1033,10 @@ def parse_extra_info(info): fname = items[0] info = items[1] # now we get each key,value pair in the info - info = info.replace('}', '') + info = info.replace("}", "") data = {} - for item in info.split(','): - key, value = item.strip().split(':') + for item in info.split(","): + key, value = item.strip().split(":") key = key.strip() value = value.strip() data[key] = value @@ -1023,12 +1045,13 @@ def parse_extra_info(info): def return_merged_clips(data): - '''It returns the left and right positions to clip.''' + """It returns the left and right positions to clip.""" + def max(a, b): - '''It returns the max of the two given numbers. + """It returns the max of the two given numbers. It won't take into account the zero values. - ''' + """ if not a and not b: return None if not a: @@ -1041,10 +1064,10 @@ def return_merged_clips(data): return b def min(a, b): - '''It returns the min of the two given numbers. + """It returns the min of the two given numbers. It won't take into account the zero values. - ''' + """ if not a and not b: return None if not a: @@ -1055,60 +1078,61 @@ def return_merged_clips(data): return a else: return b - left = max(data['clip_adapter_left'], data['clip_qual_left']) - right = min(data['clip_adapter_right'], data['clip_qual_right']) + + left = max(data["clip_adapter_left"], data["clip_qual_left"]) + right = min(data["clip_adapter_right"], data["clip_qual_right"]) # maybe both clips where zero if left is None: left = 1 if right is None: - right = data['number_of_bases'] + right = data["number_of_bases"] return left, right def sequence_case(data): - '''Given the data for one read it returns the seq with mixed case. + """Given the data for one read it returns the seq with mixed case. The regions to be clipped will be lower case and the rest upper case. - ''' + """ left, right = return_merged_clips(data) - seq = data['bases'] + seq = data["bases"] if left >= right: new_seq = seq.lower() else: - new_seq = ''.join((seq[:left - 1].lower(), seq[left - 1:right], seq[right:].lower())) + new_seq = "".join((seq[: left - 1].lower(), seq[left - 1 : right], seq[right:].lower())) return new_seq def clip_read(data): - '''Given the data for one read it returns clipped seq and qual.''' - qual = data['quality_scores'] + """Given the data for one read it returns clipped seq and qual.""" + qual = data["quality_scores"] left, right = return_merged_clips(data) - seq = data['bases'] - qual = data['quality_scores'] - new_seq = seq[left - 1:right] - new_qual = qual[left - 1:right] + seq = data["bases"] + qual = data["quality_scores"] + new_seq = seq[left - 1 : right] + new_qual = qual[left - 1 : right] return new_seq, new_qual def tests_for_ssaha(): - '''Tests whether SSAHA2 can be successfully called.''' + """Tests whether SSAHA2 can be successfully called.""" try: - print("Testing whether SSAHA2 is installed and can be launched ... ", end=' ') + print("Testing whether SSAHA2 is installed and can be launched ... ", end=" ") sys.stdout.flush() - fh = open('/dev/null', 'w') + fh = open("/dev/null", "w") subprocess.call(["ssaha2"], stdout=fh) fh.close() print("ok.") except Exception: print("nope? Uh oh ...\n\n") - raise RuntimeError('Could not launch ssaha2. Have you installed it? Is it in your path?') + raise RuntimeError("Could not launch ssaha2. Have you installed it? Is it in your path?") def load_linker_sequences(linker_fname): - '''Loads all linker sequences into memory, storing only the length - of each linker.''' + """Loads all linker sequences into memory, storing only the length + of each linker.""" global linkerlengths if not os.path.getsize(linker_fname): @@ -1125,41 +1149,46 @@ def load_linker_sequences(linker_fname): def launch_ssaha(linker_fname, query_fname, output_fh): - '''Launches SSAHA2 on the linker and query file, string SSAHA2 output - into the output filehandle''' + """Launches SSAHA2 on the linker and query file, string SSAHA2 output + into the output filehandle""" tests_for_ssaha() try: - print("Searching linker sequences with SSAHA2 (this may take a while) ... ", end=' ') + print("Searching linker sequences with SSAHA2 (this may take a while) ... ", end=" ") sys.stdout.flush() - retcode = subprocess.call(["ssaha2", "-output", "ssaha2", "-solexa", "-kmer", "4", "-skip", "1", linker_fname, query_fname], stdout=output_fh) + retcode = subprocess.call( + ["ssaha2", "-output", "ssaha2", "-solexa", "-kmer", "4", "-skip", "1", linker_fname, query_fname], + stdout=output_fh, + ) if retcode: - raise RuntimeError('Ups.') + raise RuntimeError("Ups.") else: print("ok.") except Exception: print("\n") - raise RuntimeError('An error occurred during the SSAHA2 execution, aborting.') + raise RuntimeError("An error occurred during the SSAHA2 execution, aborting.") def read_ssaha_data(ssahadata_fh): - '''Given file handle, reads file generated with SSAHA2 (with default + """Given file handle, reads file generated with SSAHA2 (with default output format) and stores all matches as list ssahapematches - (ssaha paired-end matches) dictionary''' + (ssaha paired-end matches) dictionary""" global ssahapematches - print("Parsing SSAHA2 result file ... ", end=' ') + print("Parsing SSAHA2 result file ... ", end=" ") sys.stdout.flush() for line in ssahadata_fh: - if line.startswith('ALIGNMENT'): + if line.startswith("ALIGNMENT"): ml = line.split() if len(ml) != 12: - print("\n", line, end=' ') - raise RuntimeError('Expected 12 elements in the SSAHA2 line with ALIGMENT keyword, but found ' + str(len(ml))) + print("\n", line, end=" ") + raise RuntimeError( + "Expected 12 elements in the SSAHA2 line with ALIGMENT keyword, but found " + str(len(ml)) + ) if ml[2] not in ssahapematches: - ssahapematches[ml[2]] = ([]) - if ml[8] == 'F': + ssahapematches[ml[2]] = [] + if ml[8] == "F": # store everything except the first element (output # format name (ALIGNMENT)) and the last element # (length) @@ -1180,6 +1209,7 @@ def read_ssaha_data(ssahadata_fh): # ########################################################################## + class Fasta(object): def __init__(self, name, sequence): self.name = name @@ -1188,30 +1218,32 @@ class Fasta(object): def read_fasta(file): items = [] - aninstance = Fasta('', '') + aninstance = Fasta("", "") linenum = 0 for line in file: linenum += 1 if line.startswith(">"): if len(aninstance.sequence): items.append(aninstance) - aninstance = Fasta('', '') + aninstance = Fasta("", "") # name == all characters until the first whitespace # (split()[0]) but without the starting ">" ([1:]) aninstance.name = line.split()[0][1:] - aninstance.sequence = '' + aninstance.sequence = "" if len(aninstance.name) == 0: - raise RuntimeError(file.name + ': no name in line ' + str(linenum) + '?') + raise RuntimeError(file.name + ": no name in line " + str(linenum) + "?") else: if len(aninstance.name) == 0: - raise RuntimeError(file.name + ': no sequence header at line ' + str(linenum) + '?') + raise RuntimeError(file.name + ": no sequence header at line " + str(linenum) + "?") aninstance.sequence += line.strip() if len(aninstance.name) and len(aninstance.sequence): items.append(aninstance) return items + + ########################################################################## @@ -1220,57 +1252,81 @@ def version_string(): def read_config(): - '''It reads the configuration options from the command line arguments and - it returns a dict with them.''' + """It reads the configuration options from the command line arguments and + it returns a dict with them.""" from optparse import ( OptionGroup, OptionParser, ) + usage = "usage: %prog [options] sff1 sff2 ..." - desc = "Extract sequences from 454 SFF files into FASTA, FASTA quality"\ - " and XML traceinfo format. When a paired-end linker sequence"\ - " is given (-l), use SSAHA2 to scan the sequences for the linker,"\ - " then split the sequences, removing the linker." + desc = ( + "Extract sequences from 454 SFF files into FASTA, FASTA quality" + " and XML traceinfo format. When a paired-end linker sequence" + " is given (-l), use SSAHA2 to scan the sequences for the linker," + " then split the sequences, removing the linker." + ) parser = OptionParser(usage=usage, version=version_string(), description=desc) - parser.add_option('-a', '--append', action="store_true", dest='append', - help='append output to existing files', default=False) - parser.add_option('-i', '--xml_info', dest='xml_info', - help='extra info to write in the xml file') - parser.add_option("-l", "--linker_file", dest="pelinker_fname", - help="FASTA file with paired-end linker sequences", metavar="FILE") + parser.add_option( + "-a", "--append", action="store_true", dest="append", help="append output to existing files", default=False + ) + parser.add_option("-i", "--xml_info", dest="xml_info", help="extra info to write in the xml file") + parser.add_option( + "-l", "--linker_file", dest="pelinker_fname", help="FASTA file with paired-end linker sequences", metavar="FILE" + ) group = OptionGroup(parser, "File name options", "") - group.add_option('-c', '--clip', action="store_true", dest='clip', - help='clip (completely remove) ends with low qual and/or adaptor sequence', default=False) - group.add_option('-u', '--upper_case', action="store_false", dest='mix_case', - help='all bases in upper case, including clipped ends', default=True) - group.add_option('', '--min_left_clip', dest='min_leftclip', - metavar="INTEGER", type="int", - help='if the left clip coming from the SFF is smaller than this value, override it', default=0) - group.add_option('-Q', '--fastq', action="store_true", dest='want_fastq', - help='store as FASTQ file instead of FASTA + FASTA quality file', default=False) + group.add_option( + "-c", + "--clip", + action="store_true", + dest="clip", + help="clip (completely remove) ends with low qual and/or adaptor sequence", + default=False, + ) + group.add_option( + "-u", + "--upper_case", + action="store_false", + dest="mix_case", + help="all bases in upper case, including clipped ends", + default=True, + ) + group.add_option( + "", + "--min_left_clip", + dest="min_leftclip", + metavar="INTEGER", + type="int", + help="if the left clip coming from the SFF is smaller than this value, override it", + default=0, + ) + group.add_option( + "-Q", + "--fastq", + action="store_true", + dest="want_fastq", + help="store as FASTQ file instead of FASTA + FASTA quality file", + default=False, + ) parser.add_option_group(group) group = OptionGroup(parser, "File name options", "") - group.add_option("-o", "--out_basename", dest="basename", - help="base name for all output files") - group.add_option("-s", "--seq_file", dest="seq_fname", - help="output sequence file name", metavar="FILE") - group.add_option("-q", "--qual_file", dest="qual_fname", - help="output quality file name", metavar="FILE") - group.add_option("-x", "--xml_file", dest="xml_fname", - help="output ancillary xml file name", metavar="FILE") + group.add_option("-o", "--out_basename", dest="basename", help="base name for all output files") + group.add_option("-s", "--seq_file", dest="seq_fname", help="output sequence file name", metavar="FILE") + group.add_option("-q", "--qual_file", dest="qual_fname", help="output quality file name", metavar="FILE") + group.add_option("-x", "--xml_file", dest="xml_fname", help="output ancillary xml file name", metavar="FILE") parser.add_option_group(group) # default fnames # is there an sff file? - basename = 'reads' - if sys.argv[-1][-4:].lower() == '.sff': + basename = "reads" + if sys.argv[-1][-4:].lower() == ".sff": basename = sys.argv[-1][:-4] - def_seq_fname = basename + '.fasta' - def_qual_fname = basename + '.fasta.qual' - def_xml_fname = basename + '.xml' - def_pelinker_fname = '' + def_seq_fname = basename + ".fasta" + def_qual_fname = basename + ".fasta.qual" + def_xml_fname = basename + ".xml" + def_pelinker_fname = "" parser.set_defaults(seq_fname=def_seq_fname) parser.set_defaults(qual_fname=def_qual_fname) parser.set_defaults(xml_fname=def_xml_fname) @@ -1283,30 +1339,30 @@ def read_config(): global config config = {} for property in dir(options): - if property[0] == '_' or property in ('ensure_value', 'read_file', 'read_module'): + if property[0] == "_" or property in ("ensure_value", "read_file", "read_module"): continue config[property] = getattr(options, property) - if config['basename'] is None: - config['basename'] = basename + if config["basename"] is None: + config["basename"] = basename # if we have not set a file name with -s, -q or -x we set the basename # based file name - if config['want_fastq']: - config['qual_fname'] = '' - if config['seq_fname'] == def_seq_fname: - config['seq_fname'] = config['basename'] + '.fastq' + if config["want_fastq"]: + config["qual_fname"] = "" + if config["seq_fname"] == def_seq_fname: + config["seq_fname"] = config["basename"] + ".fastq" else: - if config['seq_fname'] == def_seq_fname: - config['seq_fname'] = config['basename'] + '.fasta' - if config['qual_fname'] == def_qual_fname: - config['qual_fname'] = config['basename'] + '.fasta.qual' + if config["seq_fname"] == def_seq_fname: + config["seq_fname"] = config["basename"] + ".fasta" + if config["qual_fname"] == def_qual_fname: + config["qual_fname"] = config["basename"] + ".fasta.qual" - if config['xml_fname'] == def_xml_fname: - config['xml_fname'] = config['basename'] + '.xml' + if config["xml_fname"] == def_xml_fname: + config["xml_fname"] = config["basename"] + ".xml" # we parse the extra info for the xml file - config['xml_info'] = parse_extra_info(config['xml_info']) + config["xml_info"] = parse_extra_info(config["xml_info"]) return config, args @@ -1318,14 +1374,14 @@ def testsome(): def main(): argv = sys.argv if len(argv) == 1: - sys.argv.append('-h') + sys.argv.append("-h") read_config() sys.exit() try: config, args = read_config() - if config['pelinker_fname']: - load_linker_sequences(config['pelinker_fname']) + if config["pelinker_fname"]: + load_linker_sequences(config["pelinker_fname"]) if len(args) == 0: raise RuntimeError("No SFF file given?") extract_reads_from_sff(config, args) diff --git a/tools/filters/sorter.py b/tools/filters/sorter.py index 1eeea7b4354..e9ccb53d9a0 100644 --- a/tools/filters/sorter.py +++ b/tools/filters/sorter.py @@ -31,35 +31,35 @@ def main(): header_lines = args.header_lines key_args = [] for k in args.key: - key_args.extend(['-k', k]) + key_args.extend(["-k", k]) # sed header if header_lines > 0: - sed_header = ['sed', '-n', f"1,{header_lines:d}p"] + sed_header = ["sed", "-n", f"1,{header_lines:d}p"] subprocess.check_call(sed_header, stdin=input_fh, stdout=output_fh) input_fh.seek(0) # grep comments - grep_comments = ['grep', '^#'] + grep_comments = ["grep", "^#"] exit_code = subprocess.call(grep_comments, stdout=output_fh) if exit_code not in [0, 1]: - stop_err('Searching for comment lines failed') + stop_err("Searching for comment lines failed") # grep and sort columns if header_lines > 0: - sed_cmd = ['sed', f'1,{header_lines:d}d'] + sed_cmd = ["sed", f"1,{header_lines:d}d"] sed_header_restore = subprocess.Popen(sed_cmd, stdin=input_fh, stdout=subprocess.PIPE) pipe_stdin = sed_header_restore.stdout else: pipe_stdin = input_fh - grep = subprocess.Popen(['grep', '^[^#]'], stdin=pipe_stdin, stdout=subprocess.PIPE) - sort = subprocess.Popen(['sort', '-f', '-t', '\t'] + key_args, stdin=grep.stdout, stdout=output_fh) + grep = subprocess.Popen(["grep", "^[^#]"], stdin=pipe_stdin, stdout=subprocess.PIPE) + sort = subprocess.Popen(["sort", "-f", "-t", "\t"] + key_args, stdin=grep.stdout, stdout=output_fh) # wait for commands to complete sort.communicate() assert sort.returncode == 0, f"sort pipeline exited with non-zero exit code: {sort.returncode:d}" except Exception as ex: - stop_err('Error running sorter.py\n' + str(ex)) + stop_err("Error running sorter.py\n" + str(ex)) # exit sys.exit(0) diff --git a/tools/filters/trimmer.py b/tools/filters/trimmer.py index 65e53eaf865..9374c0561f0 100644 --- a/tools/filters/trimmer.py +++ b/tools/filters/trimmer.py @@ -17,45 +17,38 @@ options (listed below) default to 'None' if omitted parser = optparse.OptionParser(usage=usage) parser.add_option( - '-a', '--ascii', - action='store_true', + "-a", + "--ascii", + action="store_true", default=False, - help='Use ascii codes to defined ignored beginnings instead of raw characters') + help="Use ascii codes to defined ignored beginnings instead of raw characters", + ) parser.add_option( - '-q', '--fastq', - action='store_true', + "-q", + "--fastq", + action="store_true", default=False, - help='The input data in fastq format. It selected the script skips every even line since they contain sequence ids') + help="The input data in fastq format. It selected the script skips every even line since they contain sequence ids", + ) parser.add_option( - '-i', '--ignore', - help='A comma separated list on ignored beginnings (e.g., ">,@"), or its ascii codes (e.g., "60,42") if option -a is enabled') + "-i", + "--ignore", + help='A comma separated list on ignored beginnings (e.g., ">,@"), or its ascii codes (e.g., "60,42") if option -a is enabled', + ) + + parser.add_option("-s", "--start", type="int", default=0, help="Trim from beginning to here (1-based)") + + parser.add_option("-e", "--end", type="int", default="0", help="Trim from here to the ned (1-based)") parser.add_option( - '-s', '--start', - type='int', - default=0, - help='Trim from beginning to here (1-based)') + "-f", "--file", dest="input_txt", default=False, help="Name of file to be chopped. STDIN is default" + ) parser.add_option( - '-e', '--end', - type='int', - default='0', - help='Trim from here to the ned (1-based)') - - parser.add_option( - '-f', '--file', - dest='input_txt', - default=False, - help='Name of file to be chopped. STDIN is default') - - parser.add_option( - '-c', '--column', - type='int', - dest='col', - default='0', - help='Column to chop. If 0 = chop the whole line') + "-c", "--column", type="int", dest="col", default="0", help="Column to chop. If 0 = chop the whole line" + ) options, args = parser.parse_args() invalid_starts = [] @@ -66,13 +59,10 @@ options (listed below) default to 'None' if omitted infile = sys.stdin if options.ignore and options.ignore != "None": - invalid_starts = { - chr(int(c)) if options.ascii else c - for c in options.ignore.split(',') - } + invalid_starts = {chr(int(c)) if options.ascii else c for c in options.ignore.split(",")} for i, line in enumerate(infile): - line = line.rstrip('\r\n') + line = line.rstrip("\r\n") if line: if options.fastq and i % 2 == 0: print(line) @@ -81,19 +71,19 @@ options (listed below) default to 'None' if omitted if line[0] not in invalid_starts: if options.col == 0: if options.end == 0: - line = line[options.start - 1:] + line = line[options.start - 1 :] else: - line = line[options.start - 1:options.end] + line = line[options.start - 1 : options.end] else: - fields = line.split('\t') + fields = line.split("\t") if options.col > len(fields): - stop_err('Column %d does not exist. Check input parameters\n' % options.col) + stop_err("Column %d does not exist. Check input parameters\n" % options.col) if options.end == 0: - fields[options.col - 1] = fields[options.col - 1][options.start - 1:] + fields[options.col - 1] = fields[options.col - 1][options.start - 1 :] else: - fields[options.col - 1] = fields[options.col - 1][options.start - 1:options.end] - line = '\t'.join(fields) + fields[options.col - 1] = fields[options.col - 1][options.start - 1 : options.end] + line = "\t".join(fields) print(line) diff --git a/tools/filters/ucsc_gene_bed_to_exon_bed.py b/tools/filters/ucsc_gene_bed_to_exon_bed.py index dbeae5006b8..e40a353faa5 100755 --- a/tools/filters/ucsc_gene_bed_to_exon_bed.py +++ b/tools/filters/ucsc_gene_bed_to_exon_bed.py @@ -23,18 +23,21 @@ assert sys.version_info[:2] >= (2, 6) def main(): parser = optparse.OptionParser(usage="%prog [options] ") - parser.add_option("-r", "--region", dest="region", default="transcribed", - help="Limit to region: one of coding, utr3, utr5, transcribed [default]") - parser.add_option("-e", "--exons", action="store_true", dest="exons", - help="Only print intervals overlapping an exon") - parser.add_option("-s", "--strand", action="store_true", dest="strand", - help="Print strand after interval") - parser.add_option("-i", "--input", dest="input", default=None, - help="Input file") - parser.add_option("-o", "--output", dest="output", default=None, - help="Output file") + parser.add_option( + "-r", + "--region", + dest="region", + default="transcribed", + help="Limit to region: one of coding, utr3, utr5, transcribed [default]", + ) + parser.add_option( + "-e", "--exons", action="store_true", dest="exons", help="Only print intervals overlapping an exon" + ) + parser.add_option("-s", "--strand", action="store_true", dest="strand", help="Print strand after interval") + parser.add_option("-i", "--input", dest="input", default=None, help="Input file") + parser.add_option("-o", "--output", dest="output", default=None, help="Output file") options, args = parser.parse_args() - assert options.region in ('coding', 'utr3', 'utr5', 'transcribed', 'intron', 'codon'), "Invalid region argument" + assert options.region in ("coding", "utr3", "utr5", "transcribed", "intron", "codon"), "Invalid region argument" try: out_file = open(options.output, "w") @@ -56,7 +59,7 @@ def main(): if line[0:1] == "#": continue # Parse fields from gene tabls - fields = line.split('\t') + fields = line.split("\t") chrom = fields[0] tx_start = int(fields[1]) tx_end = int(fields[2]) @@ -66,31 +69,31 @@ def main(): cds_end = int(fields[7]) # Determine the subset of the transcribed region we are interested in - if options.region == 'utr3': - if strand == '-': + if options.region == "utr3": + if strand == "-": region_start, region_end = tx_start, cds_start else: region_start, region_end = cds_end, tx_end - elif options.region == 'utr5': - if strand == '-': + elif options.region == "utr5": + if strand == "-": region_start, region_end = cds_end, tx_end else: region_start, region_end = tx_start, cds_start - elif options.region == 'coding' or options.region == 'codon': + elif options.region == "coding" or options.region == "codon": region_start, region_end = cds_start, cds_end else: region_start, region_end = tx_start, tx_end # If only interested in exons, print the portion of each exon overlapping # the region of interest, otherwise print the span of the region - # options.exons is always TRUE + # options.exons is always TRUE if options.exons: - exon_starts = [int(_) + tx_start for _ in fields[11].rstrip(',\n').split(',')] - exon_ends = [int(_) for _ in fields[10].rstrip(',\n').split(',')] + exon_starts = [int(_) + tx_start for _ in fields[11].rstrip(",\n").split(",")] + exon_ends = [int(_) for _ in fields[10].rstrip(",\n").split(",")] exon_ends = [x + y for x, y in zip(exon_starts, exon_ends)] - # for Intron regions: - if options.region == 'intron': + # for Intron regions: + if options.region == "intron": i = 0 while i < len(exon_starts) - 1: intron_starts = exon_ends[i] @@ -100,13 +103,13 @@ def main(): else: print_tab_sep(out_file, chrom, intron_starts, intron_ends) i += 1 - # for non-intron regions: + # for non-intron regions: else: for start, end in zip(exon_starts, exon_ends): start = max(start, region_start) end = min(end, region_end) if start < end: - if options.region == 'codon': + if options.region == "codon": start += (3 - ((start - region_start) % 3)) % 3 c_start = start while c_start + 3 <= end: @@ -126,7 +129,7 @@ def main(): def print_tab_sep(out_file, *args): """Print items in `l` to stdout separated by tabs""" - print('\t'.join(str(f) for f in args), file=out_file) + print("\t".join(str(f) for f in args), file=out_file) if __name__ == "__main__": diff --git a/tools/filters/ucsc_gene_bed_to_intron_bed.py b/tools/filters/ucsc_gene_bed_to_intron_bed.py index 91bd49b30be..54de6bddc8f 100755 --- a/tools/filters/ucsc_gene_bed_to_intron_bed.py +++ b/tools/filters/ucsc_gene_bed_to_intron_bed.py @@ -23,12 +23,9 @@ assert sys.version_info[:2] >= (2, 6) def main(): parser = optparse.OptionParser(usage="%prog [options] ") - parser.add_option("-s", "--strand", action="store_true", dest="strand", - help="Print strand after interval") - parser.add_option("-i", "--input", dest="input", default=None, - help="Input file") - parser.add_option("-o", "--output", dest="output", default=None, - help="Output file") + parser.add_option("-s", "--strand", action="store_true", dest="strand", help="Print strand after interval") + parser.add_option("-i", "--input", dest="input", default=None, help="Input file") + parser.add_option("-o", "--output", dest="output", default=None, help="Output file") options, args = parser.parse_args() try: @@ -50,7 +47,7 @@ def main(): continue # Parse fields from gene tabls - fields = line.split('\t') + fields = line.split("\t") chrom = fields[0] tx_start = int(fields[1]) int(fields[2]) @@ -59,8 +56,8 @@ def main(): int(fields[6]) int(fields[7]) - exon_starts = [int(_) + tx_start for _ in fields[11].rstrip(',\n').split(',')] - exon_ends = [int(_) for _ in fields[10].rstrip(',\n').split(',')] + exon_starts = [int(_) + tx_start for _ in fields[11].rstrip(",\n").split(",")] + exon_ends = [int(_) for _ in fields[10].rstrip(",\n").split(",")] exon_ends = [x + y for x, y in zip(exon_starts, exon_ends)] i = 0 @@ -78,7 +75,7 @@ def main(): def print_tab_sep(out_file, *args): """Print items in `l` to stdout separated by tabs""" - print('\t'.join(str(f) for f in args), file=out_file) + print("\t".join(str(f) for f in args), file=out_file) if __name__ == "__main__": diff --git a/tools/filters/ucsc_gene_table_to_intervals.py b/tools/filters/ucsc_gene_table_to_intervals.py index aec59641659..f23a9fce211 100755 --- a/tools/filters/ucsc_gene_table_to_intervals.py +++ b/tools/filters/ucsc_gene_table_to_intervals.py @@ -23,18 +23,21 @@ assert sys.version_info[:2] >= (2, 6) def main(): parser = optparse.OptionParser(usage="%prog [options] ") - parser.add_option("-r", "--region", dest="region", default="transcribed", - help="Limit to region: one of coding, utr3, utr5, transcribed [default]") - parser.add_option("-e", "--exons", action="store_true", dest="exons", - help="Only print intervals overlapping an exon") - parser.add_option("-s", "--strand", action="store_true", dest="strand", - help="Print strand after interval") - parser.add_option("-i", "--input", dest="input", default=None, - help="Input file") - parser.add_option("-o", "--output", dest="output", default=None, - help="Output file") + parser.add_option( + "-r", + "--region", + dest="region", + default="transcribed", + help="Limit to region: one of coding, utr3, utr5, transcribed [default]", + ) + parser.add_option( + "-e", "--exons", action="store_true", dest="exons", help="Only print intervals overlapping an exon" + ) + parser.add_option("-s", "--strand", action="store_true", dest="strand", help="Print strand after interval") + parser.add_option("-i", "--input", dest="input", default=None, help="Input file") + parser.add_option("-o", "--output", dest="output", default=None, help="Output file") options, args = parser.parse_args() - assert options.region in ('coding', 'utr3', 'utr5', 'transcribed'), "Invalid region argument" + assert options.region in ("coding", "utr3", "utr5", "transcribed"), "Invalid region argument" try: out_file = open(options.output, "w") @@ -49,7 +52,7 @@ def main(): sys.exit(0) print("Region:", options.region + ";") - print("Only overlap with Exons:", end=' ') + print("Only overlap with Exons:", end=" ") if options.exons: print("Yes") else: @@ -61,7 +64,7 @@ def main(): if line[0:1] == "#": continue # Parse fields from gene tabls - fields = line.split('\t') + fields = line.split("\t") name = fields[0] chrom = fields[1] strand = fields[2].replace(" ", "_") @@ -71,17 +74,17 @@ def main(): cds_end = int(fields[6]) # Determine the subset of the transcribed region we are interested in - if options.region == 'utr3': - if strand == '-': + if options.region == "utr3": + if strand == "-": region_start, region_end = tx_start, cds_start else: region_start, region_end = cds_end, tx_end - elif options.region == 'utr5': - if strand == '-': + elif options.region == "utr5": + if strand == "-": region_start, region_end = cds_end, tx_end else: region_start, region_end = tx_start, cds_start - elif options.region == 'coding': + elif options.region == "coding": region_start, region_end = cds_start, cds_end else: region_start, region_end = tx_start, tx_end @@ -89,8 +92,8 @@ def main(): # If only interested in exons, print the portion of each exon overlapping # the region of interest, otherwise print the span of the region if options.exons: - exon_starts = map(int, fields[8].rstrip(',\n').split(',')) - exon_ends = map(int, fields[9].rstrip(',\n').split(',')) + exon_starts = map(int, fields[8].rstrip(",\n").split(",")) + exon_ends = map(int, fields[9].rstrip(",\n").split(",")) for start, end in zip(exon_starts, exon_ends): start = max(start, region_start) end = min(end, region_end) @@ -110,7 +113,7 @@ def main(): def print_tab_sep(out_file, *args): """Print items in `l` to stdout separated by tabs""" - print('\t'.join(str(f) for f in args), file=out_file) + print("\t".join(str(f) for f in args), file=out_file) if __name__ == "__main__": diff --git a/tools/filters/uniq.py b/tools/filters/uniq.py index 9822aebcc5a..5243109c714 100644 --- a/tools/filters/uniq.py +++ b/tools/filters/uniq.py @@ -26,7 +26,7 @@ import sys def getopts(argv): opts = {} while argv: - if argv[0][0] == '-': + if argv[0][0] == "-": opts[argv[0]] = argv[1] argv = argv[2:] else: @@ -71,7 +71,7 @@ def main(): return -3 columns = opts.get("-c") - if columns is None or columns == 'None': + if columns is None or columns == "None": print("Columns not specified.") return -4 @@ -101,18 +101,18 @@ def main(): commandline = "cut " # Set delimiter - if delim == 'C': - commandline += "-d \",\" " - if delim == 'D': - commandline += "-d \"-\" " - if delim == 'U': - commandline += "-d \"_\" " - if delim == 'P': - commandline += "-d \"|\" " - if delim == 'Dt': - commandline += "-d \".\" " - if delim == 'Sp': - commandline += "-d \" \" " + if delim == "C": + commandline += '-d "," ' + if delim == "D": + commandline += '-d "-" ' + if delim == "U": + commandline += '-d "_" ' + if delim == "P": + commandline += '-d "|" ' + if delim == "Dt": + commandline += '-d "." ' + if delim == "Sp": + commandline += '-d " " ' # set columns commandline += "-f " + columns diff --git a/tools/filters/wiggle_to_simple.py b/tools/filters/wiggle_to_simple.py index eb911108e55..ae1953f7745 100755 --- a/tools/filters/wiggle_to_simple.py +++ b/tools/filters/wiggle_to_simple.py @@ -23,7 +23,9 @@ def main(): out_file.write("%s\n" % "\t".join(map(str, fields))) except UCSCLimitException: # Wiggle data was truncated, at the very least need to warn the user. - sys.stderr.write('Encountered message from UCSC: "Reached output limit of 100000 data values", so be aware your data was truncated.') + sys.stderr.write( + 'Encountered message from UCSC: "Reached output limit of 100000 data values", so be aware your data was truncated.' + ) if __name__ == "__main__": diff --git a/tools/interactive/isee/render.py b/tools/interactive/isee/render.py index 334436353e8..101e90e7971 100644 --- a/tools/interactive/isee/render.py +++ b/tools/interactive/isee/render.py @@ -31,28 +31,26 @@ def render_plots(call, plots): ) for plot in plots ] - plot_calls = ',\ninitial=c(\n' + ',\n'.join(plot_calls_list) + ')' + plot_calls = ",\ninitial=c(\n" + ",\n".join(plot_calls_list) + ")" return call + plot_calls def get_render_func(plot): """Return the appropriate function to render plot.""" # This is probably broken and unused - return OPTIONS['plots'][plot['plot_types']['plot_type'].value] # type: ignore[index] + return OPTIONS["plots"][plot["plot_types"]["plot_type"].value] # type: ignore[index] def reduced_dimension_plot(pw="6L"): """Render a ReducedDimensionPlot object call.""" - return ( - f'''ReducedDimensionPlot( - PanelWidth={pw})''') + return f"""ReducedDimensionPlot( + PanelWidth={pw})""" def feature_assay_plot(pw="6L"): """Render a FeatureAssayPlot object call.""" - return ( - f'''FeatureAssayPlot( - PanelWidth={pw})''') + return f"""FeatureAssayPlot( + PanelWidth={pw})""" def row_data_table(pw="12L"): @@ -66,14 +64,14 @@ def column_data_plot(pw="6L"): OPTIONS = { - 'plots': { + "plots": { "reduced_dimension_plot": reduced_dimension_plot, "feature_assay_plot": feature_assay_plot, "row_data_table": row_data_table, "column_data_plot": column_data_plot, }, - 'colormaps': {}, - 'extra': {}, + "colormaps": {}, + "extra": {}, } diff --git a/tools/maf/interval2maf.py b/tools/maf/interval2maf.py index 06aeef514ba..dd2641d6ea6 100755 --- a/tools/maf/interval2maf.py +++ b/tools/maf/interval2maf.py @@ -45,24 +45,32 @@ def __main__(): else: dbkey = None if dbkey in [None, "?"]: - maf_utilities.tool_fail("You must specify a proper build in order to extract alignments. You can specify your genome build by clicking on the pencil icon associated with your interval file.") + maf_utilities.tool_fail( + "You must specify a proper build in order to extract alignments. You can specify your genome build by clicking on the pencil icon associated with your interval file." + ) species = maf_utilities.parse_species_option(options.species) if options.chromCol: chromCol = int(options.chromCol) - 1 else: - maf_utilities.tool_fail("Chromosome column not set, click the pencil icon in the history item to set the metadata attributes.") + maf_utilities.tool_fail( + "Chromosome column not set, click the pencil icon in the history item to set the metadata attributes." + ) if options.startCol: startCol = int(options.startCol) - 1 else: - maf_utilities.tool_fail("Start column not set, click the pencil icon in the history item to set the metadata attributes.") + maf_utilities.tool_fail( + "Start column not set, click the pencil icon in the history item to set the metadata attributes." + ) if options.endCol: endCol = int(options.endCol) - 1 else: - maf_utilities.tool_fail("End column not set, click the pencil icon in the history item to set the metadata attributes.") + maf_utilities.tool_fail( + "End column not set, click the pencil icon in the history item to set the metadata attributes." + ) if options.strandCol: strandCol = int(options.strandCol) - 1 @@ -80,9 +88,9 @@ def __main__(): maf_utilities.tool_fail("Output file has not been specified.") split_blocks_by_species = remove_all_gap_columns = False - if options.split_blocks_by_species and options.split_blocks_by_species == 'split_blocks_by_species': + if options.split_blocks_by_species and options.split_blocks_by_species == "split_blocks_by_species": split_blocks_by_species = True - if options.remove_all_gap_columns and options.remove_all_gap_columns == 'remove_all_gap_columns': + if options.remove_all_gap_columns and options.remove_all_gap_columns == "remove_all_gap_columns": remove_all_gap_columns = True else: remove_all_gap_columns = True @@ -97,7 +105,9 @@ def __main__(): if index is None: maf_utilities.tool_fail("The MAF source specified (%s) appears to be invalid." % (options.mafType)) elif options.mafFile: - index, index_filename = maf_utilities.open_or_build_maf_index(options.mafFile, options.mafIndex, species=[dbkey]) + index, index_filename = maf_utilities.open_or_build_maf_index( + options.mafFile, options.mafIndex, species=[dbkey] + ) if index is None: maf_utilities.tool_fail("Your MAF file appears to be malformed.") else: @@ -124,7 +134,11 @@ def __main__(): src = maf_utilities.src_merge(dbkey, region.chrom) for block in index.get_as_iterator(src, region.start, region.end): if split_blocks_by_species: - blocks = [new_block for new_block in maf_utilities.iter_blocks_split_by_species(block) if maf_utilities.component_overlaps_region(new_block.get_component_by_src_start(dbkey), region)] + blocks = [ + new_block + for new_block in maf_utilities.iter_blocks_split_by_species(block) + if maf_utilities.component_overlaps_region(new_block.get_component_by_src_start(dbkey), region) + ] else: blocks = [block] for block in blocks: diff --git a/tools/maf/interval_maf_to_merged_fasta.py b/tools/maf/interval_maf_to_merged_fasta.py index 7ec62e2b2b3..133eba3e881 100644 --- a/tools/maf/interval_maf_to_merged_fasta.py +++ b/tools/maf/interval_maf_to_merged_fasta.py @@ -49,7 +49,9 @@ def __main__(): else: primary_species = None if primary_species in [None, "?", "None"]: - stop_err("You must specify a proper build in order to extract alignments. You can specify your genome build by clicking on the pencil icon associated with your interval file.") + stop_err( + "You must specify a proper build in order to extract alignments. You can specify your genome build by clicking on the pencil icon associated with your interval file." + ) include_primary = True secondary_species = maf_utilities.parse_species_option(options.species) @@ -76,7 +78,9 @@ def __main__(): if options.chromCol: chr_col = int(options.chromCol) - 1 else: - stop_err("Chromosome column not set, click the pencil icon in the history item to set the metadata attributes.") + stop_err( + "Chromosome column not set, click the pencil icon in the history item to set the metadata attributes." + ) if options.startCol: start_col = int(options.startCol) - 1 @@ -94,7 +98,7 @@ def __main__(): mafIndexFile = "%s/maf_index.loc" % options.mafIndexFileDir overwrite_with_gaps = True - if options.overwrite_with_gaps and options.overwrite_with_gaps.lower() == 'false': + if options.overwrite_with_gaps and options.overwrite_with_gaps.lower() == "false": overwrite_with_gaps = False # Finish parsing command line @@ -108,7 +112,9 @@ def __main__(): stop_err("The MAF source specified (%s) appears to be invalid." % (options.mafSource)) elif options.mafSourceType.lower() in ["user"]: # index maf for use here, need to remove index_file when finished - index, index_filename = maf_utilities.open_or_build_maf_index(options.mafSource, options.mafIndex, species=[primary_species]) + index, index_filename = maf_utilities.open_or_build_maf_index( + options.mafSource, options.mafIndex, species=[primary_species] + ) if index is None: stop_err("Your MAF file appears to be malformed.") else: @@ -120,10 +126,18 @@ def __main__(): if options.geneBED: region_enumerator = maf_utilities.line_enumerator(open(interval_file).readlines()) else: - region_enumerator = enumerate(bx.intervals.io.NiceReaderWrapper( - open(interval_file), chrom_col=chr_col, start_col=start_col, - end_col=end_col, strand_col=strand_col, fix_strand=True, - return_header=False, return_comments=False)) + region_enumerator = enumerate( + bx.intervals.io.NiceReaderWrapper( + open(interval_file), + chrom_col=chr_col, + start_col=start_col, + end_col=end_col, + strand_col=strand_col, + fix_strand=True, + return_header=False, + return_comments=False, + ) + ) # Step through intervals regions_extracted = 0 @@ -135,9 +149,16 @@ def __main__(): starts, ends, fields = maf_utilities.get_starts_ends_fields_from_gene_bed(line) # create spliced alignment object alignment = maf_utilities.get_spliced_region_alignment( - index, primary_species, fields[0], starts, ends, - strand='+', species=species, mincols=mincols, - overwrite_with_gaps=overwrite_with_gaps) + index, + primary_species, + fields[0], + starts, + ends, + strand="+", + species=species, + mincols=mincols, + overwrite_with_gaps=overwrite_with_gaps, + ) primary_name = secondary_name = fields[3] alignment_strand = fields[5] except Exception as e: @@ -147,9 +168,16 @@ def __main__(): try: # create spliced alignment object alignment = maf_utilities.get_region_alignment( - index, primary_species, line.chrom, line.start, - line.end, strand='+', species=species, mincols=mincols, - overwrite_with_gaps=overwrite_with_gaps) + index, + primary_species, + line.chrom, + line.start, + line.end, + strand="+", + species=species, + mincols=mincols, + overwrite_with_gaps=overwrite_with_gaps, + ) primary_name = "%s(%s):%s-%s" % (line.chrom, line.strand, line.start, line.end) secondary_name = "" alignment_strand = line.strand diff --git a/tools/maf/maf_by_block_number.py b/tools/maf/maf_by_block_number.py index ce42a553a9c..5f023cd99f4 100644 --- a/tools/maf/maf_by_block_number.py +++ b/tools/maf/maf_by_block_number.py @@ -23,7 +23,7 @@ def __main__(): sys.exit(0) species = maf_utilities.parse_species_option(sys.argv[5].strip()) - maf_writer = bx.align.maf.Writer(open(output_filename1, 'w')) + maf_writer = bx.align.maf.Writer(open(output_filename1, "w")) # we want to maintain order of block file and write blocks as many times as they are listed failed_lines = [] for ctr, line in enumerate(open(input_block_filename)): diff --git a/tools/maf/maf_filter.py b/tools/maf/maf_filter.py index f2134f9398d..3f54e3427fa 100644 --- a/tools/maf/maf_filter.py +++ b/tools/maf/maf_filter.py @@ -29,22 +29,25 @@ def main(): if species: num_species = len(species) else: - num_species = len(sys.argv.pop(1).split(',')) + num_species = len(sys.argv.pop(1).split(",")) except Exception: - print("One or more arguments is missing.\nUsage: maf_filter.py maf_filter_file input_maf output_maf path_to_save_debug species_to_keep", file=sys.stderr) + print( + "One or more arguments is missing.\nUsage: maf_filter.py maf_filter_file input_maf output_maf path_to_save_debug species_to_keep", + file=sys.stderr, + ) sys.exit() # Open input and output MAF files try: maf_reader = bx.align.maf.Reader(open(maf_file)) - maf_writer = bx.align.maf.Writer(open(out_file, 'w')) + maf_writer = bx.align.maf.Writer(open(out_file, "w")) except Exception: print("Your MAF file appears to be malformed.", file=sys.stderr) sys.exit() # Save script file for debuging/verification info later os.mkdir(additional_files_path) - shutil.copy(script_file, os.path.join(additional_files_path, 'debug.txt')) + shutil.copy(script_file, os.path.join(additional_files_path, "debug.txt")) # Loop through blocks, running filter on each # 'maf_block' and 'ret_val' are used/shared in the provided code file @@ -53,13 +56,15 @@ def main(): blocks_kept = 0 for i, maf_block in enumerate(maf_reader): # noqa: B007 if min_size <= maf_block.text_size <= max_size: - local = {'maf_block': maf_block, 'ret_val': False} - exec(compile(open(script_file).read(), script_file, 'exec'), {}, local) - if local['ret_val']: + local = {"maf_block": maf_block, "ret_val": False} + exec(compile(open(script_file).read(), script_file, "exec"), {}, local) + if local["ret_val"]: # Species limiting must be done after filters as filters could be run on non-requested output species if species: maf_block = maf_block.limit_to_species(species) - if len(maf_block.components) >= min_species_per_block and (not exclude_incomplete_blocks or len(maf_block.components) >= num_species): + if len(maf_block.components) >= min_species_per_block and ( + not exclude_incomplete_blocks or len(maf_block.components) >= num_species + ): maf_writer.write(maf_block) blocks_kept += 1 maf_writer.close() @@ -67,7 +72,7 @@ def main(): if i == 0: print("Your file contains no valid maf_blocks.") else: - print('Kept %s of %s blocks (%.2f%%).' % (blocks_kept, i + 1, float(blocks_kept) / float(i + 1) * 100.0)) + print("Kept %s of %s blocks (%.2f%%)." % (blocks_kept, i + 1, float(blocks_kept) / float(i + 1) * 100.0)) if __name__ == "__main__": diff --git a/tools/maf/maf_limit_size.py b/tools/maf/maf_limit_size.py index 62982397562..a2aa3f0d3c2 100644 --- a/tools/maf/maf_limit_size.py +++ b/tools/maf/maf_limit_size.py @@ -17,7 +17,7 @@ def __main__(): max_size = int(sys.argv[4].strip()) if max_size < 1: max_size = sys.maxsize - maf_writer = bx.align.maf.Writer(open(output_filename1, 'w')) + maf_writer = bx.align.maf.Writer(open(output_filename1, "w")) try: maf_reader = bx.align.maf.Reader(open(input_maf_filename)) except Exception: @@ -30,7 +30,7 @@ def __main__(): if min_size <= m.text_size <= max_size: maf_writer.write(m) blocks_kept += 1 - print('Kept %s of %s blocks (%.2f%%).' % (blocks_kept, i + 1, float(blocks_kept) / float(i + 1) * 100.0)) + print("Kept %s of %s blocks (%.2f%%)." % (blocks_kept, i + 1, float(blocks_kept) / float(i + 1) * 100.0)) if __name__ == "__main__": diff --git a/tools/maf/maf_limit_to_species.py b/tools/maf/maf_limit_to_species.py index e34ed350a92..28c70f5d8ac 100644 --- a/tools/maf/maf_limit_to_species.py +++ b/tools/maf/maf_limit_to_species.py @@ -24,7 +24,7 @@ def main(): spec_len = 0 try: maf_reader = bx.align.maf.Reader(open(sys.argv[2])) - maf_writer = bx.align.maf.Writer(open(sys.argv[3], 'w')) + maf_writer = bx.align.maf.Writer(open(sys.argv[3], "w")) except Exception: print("Your MAF file appears to be malformed.", file=sys.stderr) sys.exit() @@ -39,7 +39,9 @@ def main(): m = m.limit_to_species(species) m.remove_all_gap_columns() spec_in_block_len = len(maf_utilities.get_species_in_block(m)) - if (not species or allow_partial or spec_in_block_len == spec_len) and spec_in_block_len > min_species_per_block: + if ( + not species or allow_partial or spec_in_block_len == spec_len + ) and spec_in_block_len > min_species_per_block: maf_writer.write(m) maf_blocks_kept += 1 diff --git a/tools/maf/maf_reverse_complement.py b/tools/maf/maf_reverse_complement.py index 958fa81b9ee..a80efb6c6c6 100644 --- a/tools/maf/maf_reverse_complement.py +++ b/tools/maf/maf_reverse_complement.py @@ -22,7 +22,7 @@ def __main__(): species = maf_utilities.parse_species_option(sys.argv.pop(1)) try: - maf_writer = bx.align.maf.Writer(open(output_file, 'w')) + maf_writer = bx.align.maf.Writer(open(output_file, "w")) except Exception: print(sys.stderr, "Unable to open output file") sys.exit() diff --git a/tools/maf/maf_stats.py b/tools/maf/maf_stats.py index e839f9e1156..cc063bdd9c0 100644 --- a/tools/maf/maf_stats.py +++ b/tools/maf/maf_stats.py @@ -24,7 +24,10 @@ def __main__(): start_col = int(sys.argv[6].strip()) - 1 end_col = int(sys.argv[7].strip()) - 1 except Exception: - print("You appear to be missing metadata. You can specify your metadata by clicking on the pencil icon associated with your interval file.", file=sys.stderr) + print( + "You appear to be missing metadata. You can specify your metadata by clicking on the pencil icon associated with your interval file.", + file=sys.stderr, + ) sys.exit() summary = sys.argv[8].strip() if summary.lower() == "true": @@ -40,7 +43,9 @@ def __main__(): index = index_filename = None if maf_source_type == "user": # index maf for use here - index, index_filename = maf_utilities.open_or_build_maf_index(input_maf_filename, maf_index_filename, species=[dbkey]) + index, index_filename = maf_utilities.open_or_build_maf_index( + input_maf_filename, maf_index_filename, species=[dbkey] + ) if index is None: print("Your MAF file appears to be malformed.", file=sys.stderr) sys.exit() @@ -51,10 +56,10 @@ def __main__(): print("The MAF source specified (%s) appears to be invalid." % (input_maf_filename), file=sys.stderr) sys.exit() else: - print('Invalid source type specified: %s' % maf_source_type, file=sys.stdout) + print("Invalid source type specified: %s" % maf_source_type, file=sys.stdout) sys.exit() - out = open(output_filename, 'w') + out = open(output_filename, "w") num_region = None num_bad_region = 0 @@ -87,11 +92,15 @@ def __main__(): for block in maf_utilities.iter_blocks_split_by_species(block): if maf_utilities.component_overlaps_region(block.get_component_by_src(src), region): # need to chop and orient the block - block = maf_utilities.orient_block_by_region(maf_utilities.chop_block_by_region(block, src, region), src, region, force_strand='+') - start_offset, alignment = maf_utilities.reduce_block_by_primary_genome(block, dbkey, region.chrom, region.start) + block = maf_utilities.orient_block_by_region( + maf_utilities.chop_block_by_region(block, src, region), src, region, force_strand="+" + ) + start_offset, alignment = maf_utilities.reduce_block_by_primary_genome( + block, dbkey, region.chrom, region.start + ) for i in range(len(alignment[dbkey])): for spec, text in alignment.items(): - if text[i] != '-': + if text[i] != "-": coverage[spec].set(start_offset + i) if summary: # record summary @@ -102,13 +111,17 @@ def __main__(): else: # print coverage for interval coverage_sum = coverage[dbkey].count_range() - out.write("%s\t%s\t%s\t%s\n" % ("\t".join(region.fields), dbkey, coverage_sum, region_length - coverage_sum)) + out.write( + "%s\t%s\t%s\t%s\n" % ("\t".join(region.fields), dbkey, coverage_sum, region_length - coverage_sum) + ) keys = list(coverage.keys()) keys.remove(dbkey) keys.sort() for key in keys: coverage_sum = coverage[key].count_range() - out.write("%s\t%s\t%s\t%s\n" % ("\t".join(region.fields), key, coverage_sum, region_length - coverage_sum)) + out.write( + "%s\t%s\t%s\t%s\n" % ("\t".join(region.fields), key, coverage_sum, region_length - coverage_sum) + ) if summary: out.write("#species\tnucleotides\tcoverage\n") for spec in species_summary: diff --git a/tools/maf/maf_thread_for_species.py b/tools/maf/maf_thread_for_species.py index d60cee22544..6da4c522b9d 100644 --- a/tools/maf/maf_thread_for_species.py +++ b/tools/maf/maf_thread_for_species.py @@ -23,7 +23,7 @@ from bx.align.tools.thread import ( def main(): input_file = sys.argv.pop(1) output_file = sys.argv.pop(1) - species = sys.argv.pop(1).split(',') + species = sys.argv.pop(1).split(",") try: maf_reader = bx.align.maf.Reader(open(input_file)) @@ -31,14 +31,14 @@ def main(): print("Unable to open source MAF file", file=sys.stderr) sys.exit() try: - maf_writer = FusingAlignmentWriter(bx.align.maf.Writer(open(output_file, 'w'))) + maf_writer = FusingAlignmentWriter(bx.align.maf.Writer(open(output_file, "w"))) except Exception: print("Unable to open output file", file=sys.stderr) sys.exit() try: for m in maf_reader: new_components = m.components - if species != ['None']: + if species != ["None"]: new_components = get_components_for_species(m, species) if new_components: remove_all_gap_columns(new_components) diff --git a/tools/maf/maf_to_bed.py b/tools/maf/maf_to_bed.py index 6e0be5d3aec..3122501f5a7 100644 --- a/tools/maf/maf_to_bed.py +++ b/tools/maf/maf_to_bed.py @@ -18,7 +18,7 @@ def __main__(): # where to store files that become additional output database_tmp_dir = sys.argv[5] - species = sys.argv[3].split(',') + species = sys.argv[3].split(",") partial = sys.argv[4] output_id = sys.argv[6] out_files = {} @@ -44,10 +44,12 @@ def __main__(): for i, spec in enumerate(species): if i == 0: - out_files[spec] = open(output_filename, 'w') + out_files[spec] = open(output_filename, "w") primary_spec = spec else: - out_files[spec] = open(os.path.join(database_tmp_dir, 'primary_%s_%s_visible_bed_%s' % (output_id, spec, spec)), 'w+') + out_files[spec] = open( + os.path.join(database_tmp_dir, "primary_%s_%s_visible_bed_%s" % (output_id, spec, spec)), "w+" + ) num_species = len(species) print("Restricted to species:", ",".join(species)) @@ -69,12 +71,42 @@ def __main__(): if not spec or not chrom: spec = chrom = c.src if spec not in out_files.keys(): - out_files[spec] = open(os.path.join(database_tmp_dir, 'primary_%s_%s_visible_bed_%s' % (output_id, spec, spec)), 'wb+') + out_files[spec] = open( + os.path.join(database_tmp_dir, "primary_%s_%s_visible_bed_%s" % (output_id, spec, spec)), "wb+" + ) if c.strand == "-": - out_files[spec].write(chrom + "\t" + str(c.src_size - c.end) + "\t" + str(c.src_size - c.start) + "\t" + spec + "_" + str(block_num) + "\t" + "0\t" + c.strand + "\n") + out_files[spec].write( + chrom + + "\t" + + str(c.src_size - c.end) + + "\t" + + str(c.src_size - c.start) + + "\t" + + spec + + "_" + + str(block_num) + + "\t" + + "0\t" + + c.strand + + "\n" + ) else: - out_files[spec].write(chrom + "\t" + str(c.start) + "\t" + str(c.end) + "\t" + spec + "_" + str(block_num) + "\t" + "0\t" + c.strand + "\n") + out_files[spec].write( + chrom + + "\t" + + str(c.start) + + "\t" + + str(c.end) + + "\t" + + spec + + "_" + + str(block_num) + + "\t" + + "0\t" + + c.strand + + "\n" + ) for file_out in out_files.keys(): out_files[file_out].close() diff --git a/tools/maf/maf_to_fasta_concat.py b/tools/maf/maf_to_fasta_concat.py index 463c3da05f8..4dc174e9c3e 100755 --- a/tools/maf/maf_to_fasta_concat.py +++ b/tools/maf/maf_to_fasta_concat.py @@ -24,12 +24,12 @@ def __main__(): except Exception as e: maf_utilities.tool_fail("Error reading MAF filename: %s" % e) try: - file_out = open(sys.argv[3], 'w') + file_out = open(sys.argv[3], "w") except Exception as e: maf_utilities.tool_fail("Error opening file for output: %s" % e) if species: - print("Restricted to species: %s" % ', '.join(species)) + print("Restricted to species: %s" % ", ".join(species)) else: print("Not restricted to species.") @@ -45,7 +45,9 @@ def __main__(): for start_block in maf.Reader(open(input_filename)): for block in maf_utilities.iter_blocks_split_by_species(start_block): block.remove_all_gap_columns() # remove extra gaps - component = block.get_component_by_src_start(spec) # blocks only have one occurrence of a particular species, so this is safe + component = block.get_component_by_src_start( + spec + ) # blocks only have one occurrence of a particular species, so this is safe if component: file_out.write(component.text) else: diff --git a/tools/maf/maf_to_fasta_multiple_sets.py b/tools/maf/maf_to_fasta_multiple_sets.py index 713b48c3861..a08d2e8abc0 100755 --- a/tools/maf/maf_to_fasta_multiple_sets.py +++ b/tools/maf/maf_to_fasta_multiple_sets.py @@ -19,7 +19,7 @@ def __main__(): except Exception as e: maf_utilities.tool_fail("Error opening input MAF: %s" % e) try: - file_out = open(sys.argv[2], 'w') + file_out = open(sys.argv[2], "w") except Exception as e: maf_utilities.tool_fail("Error opening file for output: %s" % e) try: @@ -36,7 +36,7 @@ def __main__(): maf_utilities.tool_fail("Error determining keep partial value: %s" % e) if species: - print("Restricted to species: %s" % ', '.join(species)) + print("Restricted to species: %s" % ", ".join(species)) else: print("Not restricted to species.") @@ -52,8 +52,11 @@ def __main__(): spec_counts[spec] = 0 else: spec_counts[spec] += 1 - d = OrderedDict([('block_index', block_num), ('species', spec), ('sequence_index', spec_counts[spec])]) - file_out.write("%s\n" % maf_utilities.get_fasta_header(component, d, suffix="%s_%i_%i" % (spec, block_num, spec_counts[spec]))) + d = OrderedDict([("block_index", block_num), ("species", spec), ("sequence_index", spec_counts[spec])]) + file_out.write( + "%s\n" + % maf_utilities.get_fasta_header(component, d, suffix="%s_%i_%i" % (spec, block_num, spec_counts[spec])) + ) file_out.write("%s\n" % component.text) file_out.write("\n") file_out.close() diff --git a/tools/maf/maf_to_interval.py b/tools/maf/maf_to_interval.py index 0013e706aa6..fd7b18ae3d4 100644 --- a/tools/maf/maf_to_interval.py +++ b/tools/maf/maf_to_interval.py @@ -18,8 +18,8 @@ def __main__(): # where to store files that become additional output database_tmp_dir = sys.argv[4] primary_spec = sys.argv[5] - species = sys.argv[6].split(',') - all_species = sys.argv[7].split(',') + species = sys.argv[6].split(",") + all_species = sys.argv[7].split(",") partial = sys.argv[8] keep_gaps = sys.argv[9] out_files = {} @@ -35,10 +35,12 @@ def __main__(): all_species.sort() for spec in species: if spec == primary_spec: - out_files[spec] = open(output_filename, 'w+') + out_files[spec] = open(output_filename, "w+") else: - out_files[spec] = open(os.path.join(database_tmp_dir, 'primary_%s_%s_visible_interval_%s' % (output_id, spec, spec)), 'w+') - out_files[spec].write('#chrom\tstart\tend\tstrand\tscore\tname\t%s\n' % ('\t'.join(all_species))) + out_files[spec] = open( + os.path.join(database_tmp_dir, "primary_%s_%s_visible_interval_%s" % (output_id, spec, spec)), "w+" + ) + out_files[spec].write("#chrom\tstart\tend\tstrand\tscore\tname\t%s\n" % ("\t".join(all_species))) num_species = len(all_species) file_in = open(input_filename) @@ -51,17 +53,30 @@ def __main__(): sequences = {} for c in block.components: spec, chrom = maf_utilities.src_split(c.src) - if keep_gaps == 'remove_gaps': - sequences[spec] = c.text.replace('-', '') + if keep_gaps == "remove_gaps": + sequences[spec] = c.text.replace("-", "") else: sequences[spec] = c.text - sequences = '\t'.join(sequences.get(_, '') for _ in all_species) + sequences = "\t".join(sequences.get(_, "") for _ in all_species) for spec in species: c = block.get_component_by_src_start(spec) if c is not None: spec2, chrom = maf_utilities.src_split(c.src) - assert spec2 == spec, Exception('Species name inconsistancy found in component: %s != %s' % (spec, spec2)) - out_files[spec].write("%s\t%s\t%s\t%s\t%s\t%s\t%s\n" % (chrom, c.forward_strand_start, c.forward_strand_end, c.strand, m.score, "%s_%s_%s" % (spec, i, j), sequences)) + assert spec2 == spec, Exception( + "Species name inconsistancy found in component: %s != %s" % (spec, spec2) + ) + out_files[spec].write( + "%s\t%s\t%s\t%s\t%s\t%s\t%s\n" + % ( + chrom, + c.forward_strand_start, + c.forward_strand_end, + c.strand, + m.score, + "%s_%s_%s" % (spec, i, j), + sequences, + ) + ) file_in.close() for file_out in out_files.values(): file_out.close() diff --git a/tools/maf/vcf_to_maf_customtrack.py b/tools/maf/vcf_to_maf_customtrack.py index da67ad75ea4..22c629400ab 100644 --- a/tools/maf/vcf_to_maf_customtrack.py +++ b/tools/maf/vcf_to_maf_customtrack.py @@ -8,7 +8,7 @@ import bx.align.maf import galaxy_utils.sequence.vcf from six import Iterator -UNKNOWN_NUCLEOTIDE = '*' +UNKNOWN_NUCLEOTIDE = "*" class PopulationVCFParser(Iterator): @@ -21,7 +21,7 @@ class PopulationVCFParser(Iterator): for vc in self.reader: rval = [] for i, allele in enumerate(vc.alt): - rval.append(('%s_%i.%i' % (self.name, i + 1, self.counter + 1), allele)) + rval.append(("%s_%i.%i" % (self.name, i + 1, self.counter + 1), allele)) self.counter += 1 yield (vc, rval) @@ -36,18 +36,20 @@ class SampleVCFParser(Iterator): rval = [] alleles = [vc.ref] + vc.alt - if 'GT' in vc.format: - gt_index = vc.format.index('GT') + if "GT" in vc.format: + gt_index = vc.format.index("GT") for sample_name, sample_value in zip(vc.sample_names, vc.sample_values): gt_indexes = [] - for i in sample_value[gt_index].replace('|', '/').replace('\\', '/').split('/'): # Do we need to consider phase here? + for i in ( + sample_value[gt_index].replace("|", "/").replace("\\", "/").split("/") + ): # Do we need to consider phase here? try: gt_indexes.append(int(i)) except Exception: gt_indexes.append(None) for i, allele_i in enumerate(gt_indexes): if allele_i is not None: - rval.append(('%s_%i.%i' % (sample_name, i + 1, self.counter + 1), alleles[allele_i])) + rval.append(("%s_%i.%i" % (sample_name, i + 1, self.counter + 1), alleles[allele_i])) self.counter += 1 yield (vc, rval) @@ -55,22 +57,41 @@ class SampleVCFParser(Iterator): def main(): usage = "usage: %prog [options] output_file dbkey inputfile pop_name" parser = OptionParser(usage=usage) - parser.add_option("-p", "--population", action="store_true", dest="population", default=False, help="Create MAF on a per population basis") - parser.add_option("-s", "--sample", action="store_true", dest="sample", default=False, help="Create MAF on a per sample basis") - parser.add_option("-n", "--name", dest="name", default='Unknown Custom Track', help="Name for Custom Track") - parser.add_option("-g", "--galaxy", action="store_true", dest="galaxy", default=False, help="Tool is being executed by Galaxy (adds extra error messaging).") + parser.add_option( + "-p", + "--population", + action="store_true", + dest="population", + default=False, + help="Create MAF on a per population basis", + ) + parser.add_option( + "-s", "--sample", action="store_true", dest="sample", default=False, help="Create MAF on a per sample basis" + ) + parser.add_option("-n", "--name", dest="name", default="Unknown Custom Track", help="Name for Custom Track") + parser.add_option( + "-g", + "--galaxy", + action="store_true", + dest="galaxy", + default=False, + help="Tool is being executed by Galaxy (adds extra error messaging).", + ) (options, args) = parser.parse_args() if len(args) < 3: if options.galaxy: - print("It appears that you forgot to specify an input VCF file, click 'Add new VCF...' to add at least input.\n", file=sys.stderr) + print( + "It appears that you forgot to specify an input VCF file, click 'Add new VCF...' to add at least input.\n", + file=sys.stderr, + ) parser.error("Need to specify an output file, a dbkey and at least one input file") if not (options.population ^ options.sample): - parser.error('You must specify either a per population conversion or a per sample conversion, but not both') + parser.error("You must specify either a per population conversion or a per sample conversion, but not both") - out = open(args.pop(0), 'w') - out.write('track name="%s" visibility=pack\n' % options.name.replace("\"", "'")) + out = open(args.pop(0), "w") + out.write('track name="%s" visibility=pack\n' % options.name.replace('"', "'")) maf_writer = bx.align.maf.Writer(out) @@ -81,9 +102,9 @@ def main(): i = 0 while args: filename = args.pop(0) - pop_name = args.pop(0).replace(' ', '_') + pop_name = args.pop(0).replace(" ", "_") if not pop_name: - pop_name = 'population_%i' % (i + 1) + pop_name = "population_%i" % (i + 1) vcf_files.append(PopulationVCFParser(galaxy_utils.sequence.vcf.Reader(open(filename)), pop_name)) i += 1 else: @@ -97,28 +118,33 @@ def main(): num_ins = 0 num_dels = 0 for _variant_name, variant_text in variants: - if 'D' in variant_text: + if "D" in variant_text: num_dels = max(num_dels, int(variant_text[1:])) - elif 'I' in variant_text: + elif "I" in variant_text: num_ins = max(num_ins, len(variant_text) - 1) alignment = bx.align.maf.Alignment() - ref_text = vc.ref + '-' * num_ins + UNKNOWN_NUCLEOTIDE * (num_dels - len(vc.ref)) + ref_text = vc.ref + "-" * num_ins + UNKNOWN_NUCLEOTIDE * (num_dels - len(vc.ref)) start_pos = vc.pos - 1 if num_dels and start_pos: ref_text = UNKNOWN_NUCLEOTIDE + ref_text start_pos -= 1 - alignment.add_component(bx.align.maf.Component( - src='%s.%s%s' % (dbkey, ("chr" if not vc.chrom.startswith("chr") else ""), vc.chrom), - start=start_pos, size=len(ref_text.replace('-', '')), - strand='+', src_size=start_pos + len(ref_text), - text=ref_text)) + alignment.add_component( + bx.align.maf.Component( + src="%s.%s%s" % (dbkey, ("chr" if not vc.chrom.startswith("chr") else ""), vc.chrom), + start=start_pos, + size=len(ref_text.replace("-", "")), + strand="+", + src_size=start_pos + len(ref_text), + text=ref_text, + ) + ) for variant_name, variant_text in variants: # FIXME: # skip non-spec. compliant data, see: http://1000genomes.org/wiki/doku.php?id=1000_genomes:analysis:vcf3.3 for format spec # this check is due to data having indels not represented in the published format spec, # e.g. 1000 genomes pilot 1 indel data: ftp://ftp-trace.ncbi.nih.gov/1000genomes/ftp/pilot_data/release/2010_03/pilot1/indels/CEU.SRP000031.2010_03.indels.sites.vcf.gz - if variant_text and variant_text[0] in ['-', '+']: + if variant_text and variant_text[0] in ["-", "+"]: non_spec_skipped += 1 continue @@ -126,29 +152,40 @@ def main(): if num_dels and start_pos: var_text = UNKNOWN_NUCLEOTIDE else: - var_text = '' - if 'D' in variant_text: + var_text = "" + if "D" in variant_text: cur_num_del = int(variant_text[1:]) pre_del = min(len(vc.ref), cur_num_del) post_del = cur_num_del - pre_del - var_text = var_text + '-' * pre_del + '-' * num_ins + '-' * post_del + var_text = var_text + "-" * pre_del + "-" * num_ins + "-" * post_del var_text = var_text + UNKNOWN_NUCLEOTIDE * (len(ref_text) - len(var_text)) - elif 'I' in variant_text: + elif "I" in variant_text: cur_num_ins = len(variant_text) - 1 - var_text = var_text + vc.ref + variant_text[1:] + '-' * (num_ins - cur_num_ins) + UNKNOWN_NUCLEOTIDE * max(0, (num_dels - 1)) + var_text = ( + var_text + + vc.ref + + variant_text[1:] + + "-" * (num_ins - cur_num_ins) + + UNKNOWN_NUCLEOTIDE * max(0, (num_dels - 1)) + ) else: - var_text = var_text + variant_text + '-' * num_ins + UNKNOWN_NUCLEOTIDE * (num_dels - len(vc.ref)) - alignment.add_component(bx.align.maf.Component( - src=variant_name, start=0, - size=len(var_text.replace('-', '')), strand='+', - src_size=len(var_text.replace('-', '')), - text=var_text)) + var_text = var_text + variant_text + "-" * num_ins + UNKNOWN_NUCLEOTIDE * (num_dels - len(vc.ref)) + alignment.add_component( + bx.align.maf.Component( + src=variant_name, + start=0, + size=len(var_text.replace("-", "")), + strand="+", + src_size=len(var_text.replace("-", "")), + text=var_text, + ) + ) maf_writer.write(alignment) maf_writer.close() if non_spec_skipped: - print('Skipped %i non-specification compliant indels.' % non_spec_skipped) + print("Skipped %i non-specification compliant indels." % non_spec_skipped) if __name__ == "__main__": diff --git a/tools/meme/fimo_wrapper.py b/tools/meme/fimo_wrapper.py index 8a2d522273a..1d32ae4b349 100644 --- a/tools/meme/fimo_wrapper.py +++ b/tools/meme/fimo_wrapper.py @@ -36,7 +36,7 @@ def main(): proc = subprocess.Popen(args=fimo_cmd, shell=True, stderr=tmp_stderr) returncode = proc.wait() tmp_stderr.seek(0) - stderr = '' + stderr = "" try: while True: stderr += tmp_stderr.read(buffsize) @@ -48,17 +48,22 @@ def main(): if returncode != 0: raise Exception(stderr) except Exception as e: - raise Exception('Error running FIMO:\n' + str(e)) + raise Exception("Error running FIMO:\n" + str(e)) - shutil.move(os.path.join(html_path, 'fimo.txt'), txt_out) - shutil.move(os.path.join(html_path, 'fimo.gff'), gff_out) - shutil.move(os.path.join(html_path, 'fimo.xml'), xml_out) - shutil.move(os.path.join(html_path, 'fimo.html'), html_out) + shutil.move(os.path.join(html_path, "fimo.txt"), txt_out) + shutil.move(os.path.join(html_path, "fimo.gff"), gff_out) + shutil.move(os.path.join(html_path, "fimo.xml"), xml_out) + shutil.move(os.path.join(html_path, "fimo.html"), html_out) - out_file = open(interval_out, 'wb') - out_file.write("#%s\n" % "\t".join(("chr", "start", "end", "pattern name", "score", "strand", "matched sequence", "p-value", "q-value"))) + out_file = open(interval_out, "wb") + out_file.write( + "#%s\n" + % "\t".join( + ("chr", "start", "end", "pattern name", "score", "strand", "matched sequence", "p-value", "q-value") + ) + ) for line in open(txt_out): - if line.startswith('#'): + if line.startswith("#"): continue fields = line.rstrip("\n\r").split("\t") start, end = int(fields[2]), int(fields[3]) @@ -66,11 +71,16 @@ def main(): if start > end: start, end = end, start # flip start and end, and set strand strand = "-" - sequence = DNA_reverse_complement(sequence) # we want sequences relative to strand; FIMO always provides + stranded sequence + sequence = DNA_reverse_complement( + sequence + ) # we want sequences relative to strand; FIMO always provides + stranded sequence else: strand = "+" start -= 1 # make 0-based start position - out_file.write("%s\n" % "\t".join((fields[1], str(start), str(end), fields[0], fields[4], strand, sequence, fields[5], fields[6]))) + out_file.write( + "%s\n" + % "\t".join((fields[1], str(start), str(end), fields[0], fields[4], strand, sequence, fields[5], fields[6])) + ) out_file.close() diff --git a/tools/metag_tools/blat_wrapper.py b/tools/metag_tools/blat_wrapper.py index 74f5c747631..1b171bbb86c 100644 --- a/tools/metag_tools/blat_wrapper.py +++ b/tools/metag_tools/blat_wrapper.py @@ -14,15 +14,15 @@ def stop_err(msg): def check_nib_file(dbkey, GALAXY_DATA_INDEX_DIR): nib_file = "%s/alignseq.loc" % GALAXY_DATA_INDEX_DIR - nib_path = '' + nib_path = "" nibs = {} for line in open(nib_file): - line = line.rstrip('\r\n') + line = line.rstrip("\r\n") if line and not line.startswith("#"): - fields = line.split('\t') + fields = line.split("\t") if len(fields) < 3: continue - if fields[0] == 'seq': + if fields[0] == "seq": nibs[(fields[1])] = fields[2] if dbkey in nibs: nib_path = nibs[(dbkey)] @@ -31,12 +31,12 @@ def check_nib_file(dbkey, GALAXY_DATA_INDEX_DIR): def check_twobit_file(dbkey, GALAXY_DATA_INDEX_DIR): twobit_file = "%s/twobit.loc" % GALAXY_DATA_INDEX_DIR - twobit_path = '' + twobit_path = "" twobits = {} for line in open(twobit_file): - line = line.rstrip('\r\n') + line = line.rstrip("\r\n") if line and not line.startswith("#"): - fields = line.split('\t') + fields = line.split("\t") if len(fields) < 2: continue twobits[(fields[0])] = fields[1] @@ -47,7 +47,7 @@ def check_twobit_file(dbkey, GALAXY_DATA_INDEX_DIR): def __main__(): # I/O - source_format = sys.argv[1] # 0: dbkey; 1: upload file + source_format = sys.argv[1] # 0: dbkey; 1: upload file target_file = sys.argv[2] query_file = sys.argv[3] output_file = sys.argv[4] @@ -58,24 +58,24 @@ def __main__(): try: float(min_iden) except ValueError: - stop_err('Invalid value for minimal identity.') + stop_err("Invalid value for minimal identity.") try: test = int(tile_size) assert test >= 6 and test <= 18 except Exception: - stop_err('Invalid value for tile size. DNA word size must be between 6 and 18.') + stop_err("Invalid value for tile size. DNA word size must be between 6 and 18.") try: test = int(one_off) assert test >= 0 and test <= int(tile_size) except Exception: - stop_err('Invalid value for mismatch numbers in the word') + stop_err("Invalid value for mismatch numbers in the word") GALAXY_DATA_INDEX_DIR = sys.argv[8] all_files = [] - if source_format == '0': + if source_format == "0": # check target genome dbkey = target_file nib_path = check_nib_file(dbkey, GALAXY_DATA_INDEX_DIR) @@ -102,11 +102,18 @@ def __main__(): for detail_file_path in all_files: output_tempfile = tempfile.NamedTemporaryFile().name - command = "blat %s %s %s -oneOff=%s -tileSize=%s -minIdentity=%s -mask=lower -noHead -out=pslx 2>&1" % (detail_file_path, query_file, output_tempfile, one_off, tile_size, min_iden) + command = "blat %s %s %s -oneOff=%s -tileSize=%s -minIdentity=%s -mask=lower -noHead -out=pslx 2>&1" % ( + detail_file_path, + query_file, + output_tempfile, + one_off, + tile_size, + min_iden, + ) os.system(command) - os.system('cat %s >> %s' % (output_tempfile, output_file)) + os.system("cat %s >> %s" % (output_tempfile, output_file)) os.remove(output_tempfile) -if __name__ == '__main__': +if __name__ == "__main__": __main__() diff --git a/tools/metag_tools/shrimp_color_wrapper.py b/tools/metag_tools/shrimp_color_wrapper.py index a68bb99c9ce..f53db4928bf 100644 --- a/tools/metag_tools/shrimp_color_wrapper.py +++ b/tools/metag_tools/shrimp_color_wrapper.py @@ -19,7 +19,7 @@ def stop_err(msg): def __main__(): # SHRiMP path - shrimp = 'rmapper-cs' + shrimp = "rmapper-cs" # I/O input_target_file = sys.argv[1] # fasta @@ -27,23 +27,23 @@ def __main__(): shrimp_outfile = sys.argv[3] # shrimp output # SHRiMP parameters - spaced_seed = '1111001111' - seed_matches_per_window = '2' - seed_hit_taboo_length = '4' - seed_generation_taboo_length = '0' - seed_window_length = '115.0' - max_hits_per_read = '100' - max_read_length = '1000' - kmer = '-1' - sw_match_value = '100' - sw_mismatch_value = '-150' - sw_gap_open_ref = '-400' - sw_gap_open_query = '-400' - sw_gap_ext_ref = '-70' - sw_gap_ext_query = '-70' - sw_crossover_penalty = '-140' - sw_full_hit_threshold = '68.0' - sw_vector_hit_threshold = '60.0' + spaced_seed = "1111001111" + seed_matches_per_window = "2" + seed_hit_taboo_length = "4" + seed_generation_taboo_length = "0" + seed_window_length = "115.0" + max_hits_per_read = "100" + max_read_length = "1000" + kmer = "-1" + sw_match_value = "100" + sw_mismatch_value = "-150" + sw_gap_open_ref = "-400" + sw_gap_open_query = "-400" + sw_gap_ext_ref = "-70" + sw_gap_ext_query = "-70" + sw_crossover_penalty = "-140" + sw_full_hit_threshold = "68.0" + sw_vector_hit_threshold = "60.0" # TODO: put the threshold on each of these parameters if len(sys.argv) > 4: @@ -51,9 +51,9 @@ def __main__(): if sys.argv[4].isdigit(): spaced_seed = sys.argv[4] else: - stop_err('Error in assigning parameter: Spaced seed.') + stop_err("Error in assigning parameter: Spaced seed.") except Exception: - stop_err('Spaced seed must be a combination of 1s and 0s.') + stop_err("Spaced seed must be a combination of 1s and 0s.") seed_matches_per_window = sys.argv[5] seed_hit_taboo_length = sys.argv[6] @@ -76,7 +76,51 @@ def __main__(): shrimp_log = tempfile.NamedTemporaryFile().name # SHRiMP command - command = ' '.join((shrimp, '-s', spaced_seed, '-n', seed_matches_per_window, '-t', seed_hit_taboo_length, '-9', seed_generation_taboo_length, '-w', seed_window_length, '-o', max_hits_per_read, '-r', max_read_length, '-d', kmer, '-m', sw_match_value, '-i', sw_mismatch_value, '-g', sw_gap_open_ref, '-q', sw_gap_open_query, '-e', sw_gap_ext_ref, '-f', sw_gap_ext_query, '-x', sw_crossover_penalty, '-h', sw_full_hit_threshold, '-v', sw_vector_hit_threshold, input_query_file, input_target_file, '>', shrimp_outfile, '2>', shrimp_log)) + command = " ".join( + ( + shrimp, + "-s", + spaced_seed, + "-n", + seed_matches_per_window, + "-t", + seed_hit_taboo_length, + "-9", + seed_generation_taboo_length, + "-w", + seed_window_length, + "-o", + max_hits_per_read, + "-r", + max_read_length, + "-d", + kmer, + "-m", + sw_match_value, + "-i", + sw_mismatch_value, + "-g", + sw_gap_open_ref, + "-q", + sw_gap_open_query, + "-e", + sw_gap_ext_ref, + "-f", + sw_gap_ext_query, + "-x", + sw_crossover_penalty, + "-h", + sw_full_hit_threshold, + "-v", + sw_vector_hit_threshold, + input_query_file, + input_target_file, + ">", + shrimp_outfile, + "2>", + shrimp_log, + ) + ) try: os.system(command) @@ -87,8 +131,8 @@ def __main__(): num_hits = 0 if shrimp_outfile: for line in open(shrimp_outfile): - line = line.rstrip('\r\n') - if not line or line.startswith('#'): + line = line.rstrip("\r\n") + if not line or line.startswith("#"): continue try: line.split() @@ -96,21 +140,21 @@ def __main__(): except Exception as e: stop_err(str(e)) - if num_hits == 0: # no hits generated - err_msg = '' + if num_hits == 0: # no hits generated + err_msg = "" if shrimp_log: for line in open(shrimp_log): - if line.startswith('error'): # deal with memory error: - err_msg += line # error: realloc failed: Cannot allocate memory - if re.search('Reads Matched', line): # deal with zero hits + if line.startswith("error"): # deal with memory error: + err_msg += line # error: realloc failed: Cannot allocate memory + if re.search("Reads Matched", line): # deal with zero hits if int(line[8:].split()[2]) == 0: - err_msg = 'Zero hits found.\n' - stop_err('SHRiMP Failed due to:\n' + err_msg) + err_msg = "Zero hits found.\n" + stop_err("SHRiMP Failed due to:\n" + err_msg) # remove temp. files if os.path.exists(shrimp_log): os.remove(shrimp_log) -if __name__ == '__main__': +if __name__ == "__main__": __main__() diff --git a/tools/metag_tools/shrimp_wrapper.py b/tools/metag_tools/shrimp_wrapper.py index ea8cdcaf4a8..fc5ea7abf04 100644 --- a/tools/metag_tools/shrimp_wrapper.py +++ b/tools/metag_tools/shrimp_wrapper.py @@ -59,7 +59,20 @@ def stop_err(msg): def reverse_complement(s): - complement_dna = {"A": "T", "T": "A", "C": "G", "G": "C", "a": "t", "t": "a", "c": "g", "g": "c", "N": "N", "n": "n", ".": ".", "-": "-"} + complement_dna = { + "A": "T", + "T": "A", + "C": "G", + "G": "C", + "a": "t", + "t": "a", + "c": "g", + "g": "c", + "N": "N", + "n": "n", + ".": ".", + "-": "-", + } reversed_s = [] for i in s: reversed_s.append(complement_dna[i]) @@ -69,35 +82,35 @@ def reverse_complement(s): def generate_sub_table(result_file, ref_file, score_files, table_outfile, hit_per_read, insertion_size): invalid_editstring_char = 0 - all_score_file = score_files.split(',') + all_score_file = score_files.split(",") if len(all_score_file) != hit_per_read: - stop_err('One or more query files is missing. Please check your dataset.') + stop_err("One or more query files is missing. Please check your dataset.") temp_table_name = tempfile.NamedTemporaryFile().name - temp_table = open(temp_table_name, 'w') + temp_table = open(temp_table_name, "w") - outfile = open(table_outfile, 'w') + outfile = open(table_outfile, "w") # reference seq: not a single fasta seq refseq = {} chrom_cov = {} - seq = '' + seq = "" title = None for line in open(ref_file): line = line.rstrip() - if not line or line.startswith('#'): + if not line or line.startswith("#"): continue - if line.startswith('>'): + if line.startswith(">"): if seq: if title in refseq: pass else: refseq[title] = seq chrom_cov[title] = {} - seq = '' + seq = "" title = line[1:] else: seq += line @@ -110,11 +123,11 @@ def generate_sub_table(result_file, ref_file, score_files, table_outfile, hit_pe hits = {} for line in open(result_file): line = line.rstrip() - if not line or line.startswith('#'): + if not line or line.startswith("#"): continue # FORMAT: readname contigname strand contigstart contigend readstart readend readlength score editstring - fields = line.split('\t') + fields = line.split("\t") readname = fields[0][1:] chrom = fields[1] strand = fields[2] @@ -125,9 +138,9 @@ def generate_sub_table(result_file, ref_file, score_files, table_outfile, hit_pe editstring = fields[9] if hit_per_read == 1: - endindex = '1' + endindex = "1" else: - readname, endindex = readname.split('/') + readname, endindex = readname.split("/") if readname in hits: if endindex in hits[readname]: @@ -140,16 +153,16 @@ def generate_sub_table(result_file, ref_file, score_files, table_outfile, hit_pe # find score: one end and the other end hits_score = {} - readname = '' - score = '' + readname = "" + score = "" for num_score_file in range(len(all_score_file)): score_file = all_score_file[num_score_file] for line in open(score_file): line = line.rstrip() - if not line or line.startswith('#'): + if not line or line.startswith("#"): continue - if line.startswith('>'): + if line.startswith(">"): if score: if readname in hits: if len(hits[readname]) == hit_per_read: @@ -161,12 +174,12 @@ def generate_sub_table(result_file, ref_file, score_files, table_outfile, hit_pe else: hits_score[readname] = {} hits_score[readname][endindex] = score - score = '' + score = "" if hit_per_read == 1: readname = line[1:] - endindex = '1' + endindex = "1" else: - readname, endindex = line[1:].split('/') + readname, endindex = line[1:].split("/") else: score = line @@ -191,16 +204,16 @@ def generate_sub_table(result_file, ref_file, score_files, table_outfile, hit_pe match_count = 0 if hit_per_read == 1: - if len(hits[readkey]['1']) == 1: - matches = [hits[readkey]['1']] + if len(hits[readkey]["1"]) == 1: + matches = [hits[readkey]["1"]] match_count = 1 else: - end1_data = hits[readkey]['1'] - end2_data = hits[readkey]['2'] + end1_data = hits[readkey]["1"] + end2_data = hits[readkey]["2"] for end1_hit in end1_data: - crin_strand = {'+': False, '-': False} - crin_insertSize = {'+': False, '-': False} + crin_strand = {"+": False, "-": False} + crin_insertSize = {"+": False, "-": False} crin_strand[end1_hit[0]] = True crin_insertSize[end1_hit[0]] = int(end1_hit[2]) @@ -212,8 +225,8 @@ def generate_sub_table(result_file, ref_file, score_files, table_outfile, hit_pe if end1_hit[-1] != end2_hit[-1]: continue - if crin_strand['+'] and crin_strand['-']: - if (crin_insertSize['-'] - crin_insertSize['+']) <= insertion_size: + if crin_strand["+"] and crin_strand["-"]: + if (crin_insertSize["-"] - crin_insertSize["+"]) <= insertion_size: matches.append([end1_hit, end2_hit]) match_count += 1 @@ -222,7 +235,7 @@ def generate_sub_table(result_file, ref_file, score_files, table_outfile, hit_pe end_strand, end_editstring, end_chr_start, end_chr_end, end_read_start, end_chrom = end_data end_read_start = int(end_read_start) - 1 - if end_strand == '-': + if end_strand == "-": refsegment = reverse_complement(refseq[end_chrom][end_chr_start:end_chr_end]) else: refsegment = refseq[end_chrom][end_chr_start:end_chr_end] @@ -233,11 +246,11 @@ def generate_sub_table(result_file, ref_file, score_files, table_outfile, hit_pe while editindex < len(end_editstring): editchr = end_editstring[editindex] - chrA = '' - chrB = '' + chrA = "" + chrB = "" if editchr.isdigit(): - editcode = '' + editcode = "" while editchr.isdigit() and editindex < len(end_editstring): editcode += editchr @@ -251,7 +264,7 @@ def generate_sub_table(result_file, ref_file, score_files, table_outfile, hit_pe match_len += int(editcode) - elif editchr == 'x': + elif editchr == "x": # crossover: inserted between the appropriate two bases # Two sequencing errors: 4x15x6 (25 matches with 2 crossovers) # Treated as errors in the reads; Do nothing. @@ -264,7 +277,7 @@ def generate_sub_table(result_file, ref_file, score_files, table_outfile, hit_pe chrB = editcode match_len += len(editcode) - elif editchr == '-': + elif editchr == "-": editcode = editchr editindex += 1 chrA = refsegment[match_len] @@ -272,28 +285,28 @@ def generate_sub_table(result_file, ref_file, score_files, table_outfile, hit_pe match_len += len(editcode) gap_read += 1 - elif editchr == '(': - editcode = '' + elif editchr == "(": + editcode = "" - while editchr != ')' and editindex < len(end_editstring): + while editchr != ")" and editindex < len(end_editstring): if editindex < len(end_editstring): editchr = end_editstring[editindex] editcode += editchr editindex += 1 editcode = editcode[1:-1] - chrA = '-' * len(editcode) + chrA = "-" * len(editcode) chrB = editcode else: invalid_editstring_char += 1 - if end_strand == '-': + if end_strand == "-": chrA = reverse_complement(chrA) chrB = reverse_complement(chrB) - pos_line = '' - rev_line = '' + pos_line = "" + rev_line = "" for mappingIndex in range(len(chrA)): # reference @@ -301,36 +314,64 @@ def generate_sub_table(result_file, ref_file, score_files, table_outfile, hit_pe # read chrBx = chrB[mappingIndex] - if chrAx and chrBx and chrBx.upper() != 'N': - if end_strand == '+': + if chrAx and chrBx and chrBx.upper() != "N": + if end_strand == "+": chrom_loc = end_chr_start + match_len - len(chrA) + mappingIndex read_loc = end_read_start + match_len - len(chrA) + mappingIndex - gap_read - if chrAx == '-': + if chrAx == "-": chrom_loc -= 1 - if chrBx == '-': - scoreBx = '-1' + if chrBx == "-": + scoreBx = "-1" else: scoreBx = hits_score[readkey][str(x + 1)].split()[read_loc] # 1-based on chrom_loc and read_loc - pos_line = pos_line + '\t'.join((end_chrom, str(chrom_loc + 1), readkey + '/' + str(x + 1), str(read_loc + 1), chrAx, chrBx, scoreBx)) + '\n' + pos_line = ( + pos_line + + "\t".join( + ( + end_chrom, + str(chrom_loc + 1), + readkey + "/" + str(x + 1), + str(read_loc + 1), + chrAx, + chrBx, + scoreBx, + ) + ) + + "\n" + ) else: chrom_loc = end_chr_end - match_len + mappingIndex read_loc = end_read_start + match_len - 1 - mappingIndex - gap_read - if chrAx == '-': + if chrAx == "-": chrom_loc -= 1 - if chrBx == '-': - scoreBx = '-1' + if chrBx == "-": + scoreBx = "-1" else: scoreBx = hits_score[readkey][str(x + 1)].split()[read_loc] # 1-based on chrom_loc and read_loc - rev_line = '\t'.join((end_chrom, str(chrom_loc + 1), readkey + '/' + str(x + 1), str(read_loc + 1), chrAx, chrBx, scoreBx)) + '\n' + rev_line + rev_line = ( + "\t".join( + ( + end_chrom, + str(chrom_loc + 1), + readkey + "/" + str(x + 1), + str(read_loc + 1), + chrAx, + chrBx, + scoreBx, + ) + ) + + "\n" + + rev_line + ) if end_chrom in chrom_cov: if chrom_loc in chrom_cov[end_chrom]: @@ -343,16 +384,16 @@ def generate_sub_table(result_file, ref_file, score_files, table_outfile, hit_pe chrom_cov[end_chrom][chrom_loc] = 1 if pos_line: - temp_table.write('%s\n' % (pos_line.rstrip('\r\n'))) + temp_table.write("%s\n" % (pos_line.rstrip("\r\n"))) if rev_line: - temp_table.write('%s\n' % (rev_line.rstrip('\r\n'))) + temp_table.write("%s\n" % (rev_line.rstrip("\r\n"))) temp_table.close() # chrom-wide coverage for line in open(temp_table_name): line = line.rstrip() - if not line or line.startswith('#'): + if not line or line.startswith("#"): continue fields = line.split() @@ -361,12 +402,12 @@ def generate_sub_table(result_file, ref_file, score_files, table_outfile, hit_pe readname = fields[2] if hit_per_read == 1: - fields[2] = readname.split('/')[0] + fields[2] = readname.split("/")[0] if eachBp in chrom_cov[chrom]: - outfile.write('%s\t%d\n' % ('\t'.join(fields), chrom_cov[chrom][eachBp])) + outfile.write("%s\t%d\n" % ("\t".join(fields), chrom_cov[chrom][eachBp])) else: - outfile.write('%s\t%d\n' % ('\t'.join(fields), 0)) + outfile.write("%s\t%d\n" % ("\t".join(fields), 0)) outfile.close() @@ -374,24 +415,24 @@ def generate_sub_table(result_file, ref_file, score_files, table_outfile, hit_pe os.remove(temp_table_name) if invalid_editstring_char: - print('Skip ', invalid_editstring_char, ' invalid characters in editstrings') + print("Skip ", invalid_editstring_char, " invalid characters in editstrings") return True def convert_fastqsolexa_to_fasta_qual(infile_name, query_fasta, query_qual): - outfile_seq = open(query_fasta, 'w') - outfile_score = open(query_qual, 'w') + outfile_seq = open(query_fasta, "w") + outfile_score = open(query_qual, "w") - seq_title_startswith = '' - qual_title_startswith = '' + seq_title_startswith = "" + qual_title_startswith = "" default_coding_value = 64 # Solexa ascii-code fastq_block_lines = 0 for i, line in enumerate(open(infile_name)): line = line.rstrip() - if not line or line.startswith('#'): + if not line or line.startswith("#"): continue fastq_block_lines = (fastq_block_lines + 1) % 4 @@ -405,15 +446,15 @@ def convert_fastqsolexa_to_fasta_qual(infile_name, query_fasta, query_qual): if line_startswith != seq_title_startswith: outfile_seq.close() outfile_score.close() - stop_err('Invalid fastqsolexa format at line %d: %s.' % (i + 1, line)) + stop_err("Invalid fastqsolexa format at line %d: %s." % (i + 1, line)) read_title = line[1:] - outfile_seq.write('>%s\n' % line[1:]) + outfile_seq.write(">%s\n" % line[1:]) elif fastq_block_lines == 2: # second line is nucleotides read_length = len(line) - outfile_seq.write('%s\n' % line) + outfile_seq.write("%s\n" % line) elif fastq_block_lines == 3: # third line is +title_of_qualityscore ( might be skipped ) @@ -423,22 +464,25 @@ def convert_fastqsolexa_to_fasta_qual(infile_name, query_fasta, query_qual): if line_startswith != qual_title_startswith: outfile_seq.close() outfile_score.close() - stop_err('Invalid fastqsolexa format at line %d: %s.' % (i + 1, line)) + stop_err("Invalid fastqsolexa format at line %d: %s." % (i + 1, line)) quality_title = line[1:] if quality_title and read_title != quality_title: outfile_seq.close() outfile_score.close() - stop_err('Invalid fastqsolexa format at line %d: sequence title "%s" differes from score title "%s".' % (i + 1, read_title, quality_title)) + stop_err( + 'Invalid fastqsolexa format at line %d: sequence title "%s" differes from score title "%s".' + % (i + 1, read_title, quality_title) + ) if not quality_title: - outfile_score.write('>%s\n' % read_title) + outfile_score.write(">%s\n" % read_title) else: - outfile_score.write('>%s\n' % line[1:]) + outfile_score.write(">%s\n" % line[1:]) else: # fourth line is quality scores - qual = '' + qual = "" fastq_integer = True # peek: ascii or digits? val = line.split()[0] @@ -461,13 +505,16 @@ def convert_fastqsolexa_to_fasta_qual(infile_name, query_fasta, query_qual): elif quality_score_length == read_length: qual_score_startswith = default_coding_value else: - stop_err('Invalid fastqsolexa format at line %d: the number of quality scores ( %d ) is not the same as bases ( %d ).' % (i + 1, quality_score_length, read_length)) + stop_err( + "Invalid fastqsolexa format at line %d: the number of quality scores ( %d ) is not the same as bases ( %d )." + % (i + 1, quality_score_length, read_length) + ) for char in line: - score = ord(char) - qual_score_startswith # 64 + score = ord(char) - qual_score_startswith # 64 qual = "%s%s " % (qual, str(score)) - outfile_score.write('%s\n' % qual) + outfile_score.write("%s\n" % qual) outfile_seq.close() outfile_score.close() @@ -477,25 +524,25 @@ def convert_fastqsolexa_to_fasta_qual(infile_name, query_fasta, query_qual): def __main__(): # SHRiMP path - shrimp = 'rmapper-ls' + shrimp = "rmapper-ls" # I/O input_target_file = sys.argv[1] # fasta shrimp_outfile = sys.argv[2] # shrimp output table_outfile = sys.argv[3] # table output - single_or_paired = sys.argv[4].split(',') + single_or_paired = sys.argv[4].split(",") insertion_size = 600 - if len(single_or_paired) == 1: # single or paired - type_of_reads = 'single' + if len(single_or_paired) == 1: # single or paired + type_of_reads = "single" hit_per_read = 1 input_query = single_or_paired[0] query_fasta = tempfile.NamedTemporaryFile().name query_qual = tempfile.NamedTemporaryFile().name - else: # paired-end - type_of_reads = 'paired' + else: # paired-end + type_of_reads = "paired" hit_per_read = 2 input_query_end1 = single_or_paired[0] input_query_end2 = single_or_paired[1] @@ -506,21 +553,21 @@ def __main__(): query_qual_end2 = tempfile.NamedTemporaryFile().name # SHRiMP parameters: total = 15, default values - spaced_seed = '111111011111' - seed_matches_per_window = '2' - seed_hit_taboo_length = '4' - seed_generation_taboo_length = '0' - seed_window_length = '115.0' - max_hits_per_read = '100' - max_read_length = '1000' - kmer = '-1' - sw_match_value = '100' - sw_mismatch_value = '-150' - sw_gap_open_ref = '-400' - sw_gap_open_query = '-400' - sw_gap_ext_ref = '-70' - sw_gap_ext_query = '-70' - sw_hit_threshold = '68.0' + spaced_seed = "111111011111" + seed_matches_per_window = "2" + seed_hit_taboo_length = "4" + seed_generation_taboo_length = "0" + seed_window_length = "115.0" + max_hits_per_read = "100" + max_read_length = "1000" + kmer = "-1" + sw_match_value = "100" + sw_mismatch_value = "-150" + sw_gap_open_ref = "-400" + sw_gap_open_query = "-400" + sw_gap_ext_ref = "-70" + sw_gap_ext_query = "-70" + sw_hit_threshold = "68.0" # TODO: put the threshold on each of these parameters if len(sys.argv) > 5: @@ -528,9 +575,9 @@ def __main__(): if sys.argv[5].isdigit(): spaced_seed = sys.argv[5] else: - stop_err('Error in assigning parameter: Spaced seed.') + stop_err("Error in assigning parameter: Spaced seed.") except Exception: - stop_err('Spaced seed must be a combination of 1s and 0s.') + stop_err("Spaced seed must be a combination of 1s and 0s.") seed_matches_per_window = sys.argv[6] seed_hit_taboo_length = sys.argv[7] @@ -551,15 +598,55 @@ def __main__(): shrimp_log = tempfile.NamedTemporaryFile().name # convert fastq to fasta and quality score files - if type_of_reads == 'single': + if type_of_reads == "single": convert_fastqsolexa_to_fasta_qual(input_query, query_fasta, query_qual) else: convert_fastqsolexa_to_fasta_qual(input_query_end1, query_fasta_end1, query_qual_end1) convert_fastqsolexa_to_fasta_qual(input_query_end2, query_fasta_end2, query_qual_end2) # SHRiMP command - if type_of_reads == 'single': - command = ' '.join((shrimp, '-s', spaced_seed, '-n', seed_matches_per_window, '-t', seed_hit_taboo_length, '-9', seed_generation_taboo_length, '-w', seed_window_length, '-o', max_hits_per_read, '-r', max_read_length, '-d', kmer, '-m', sw_match_value, '-i', sw_mismatch_value, '-g', sw_gap_open_ref, '-q', sw_gap_open_query, '-e', sw_gap_ext_ref, '-f', sw_gap_ext_query, '-h', sw_hit_threshold, query_fasta, input_target_file, '>', shrimp_outfile, '2>', shrimp_log)) + if type_of_reads == "single": + command = " ".join( + ( + shrimp, + "-s", + spaced_seed, + "-n", + seed_matches_per_window, + "-t", + seed_hit_taboo_length, + "-9", + seed_generation_taboo_length, + "-w", + seed_window_length, + "-o", + max_hits_per_read, + "-r", + max_read_length, + "-d", + kmer, + "-m", + sw_match_value, + "-i", + sw_mismatch_value, + "-g", + sw_gap_open_ref, + "-q", + sw_gap_open_query, + "-e", + sw_gap_ext_ref, + "-f", + sw_gap_ext_query, + "-h", + sw_hit_threshold, + query_fasta, + input_target_file, + ">", + shrimp_outfile, + "2>", + shrimp_log, + ) + ) try: os.system(command) @@ -571,8 +658,88 @@ def __main__(): stop_err(str(e)) else: # paired - command_end1 = ' '.join((shrimp, '-s', spaced_seed, '-n', seed_matches_per_window, '-t', seed_hit_taboo_length, '-9', seed_generation_taboo_length, '-w', seed_window_length, '-o', max_hits_per_read, '-r', max_read_length, '-d', kmer, '-m', sw_match_value, '-i', sw_mismatch_value, '-g', sw_gap_open_ref, '-q', sw_gap_open_query, '-e', sw_gap_ext_ref, '-f', sw_gap_ext_query, '-h', sw_hit_threshold, query_fasta_end1, input_target_file, '>', shrimp_outfile, '2>', shrimp_log)) - command_end2 = ' '.join((shrimp, '-s', spaced_seed, '-n', seed_matches_per_window, '-t', seed_hit_taboo_length, '-9', seed_generation_taboo_length, '-w', seed_window_length, '-o', max_hits_per_read, '-r', max_read_length, '-d', kmer, '-m', sw_match_value, '-i', sw_mismatch_value, '-g', sw_gap_open_ref, '-q', sw_gap_open_query, '-e', sw_gap_ext_ref, '-f', sw_gap_ext_query, '-h', sw_hit_threshold, query_fasta_end2, input_target_file, '>>', shrimp_outfile, '2>>', shrimp_log)) + command_end1 = " ".join( + ( + shrimp, + "-s", + spaced_seed, + "-n", + seed_matches_per_window, + "-t", + seed_hit_taboo_length, + "-9", + seed_generation_taboo_length, + "-w", + seed_window_length, + "-o", + max_hits_per_read, + "-r", + max_read_length, + "-d", + kmer, + "-m", + sw_match_value, + "-i", + sw_mismatch_value, + "-g", + sw_gap_open_ref, + "-q", + sw_gap_open_query, + "-e", + sw_gap_ext_ref, + "-f", + sw_gap_ext_query, + "-h", + sw_hit_threshold, + query_fasta_end1, + input_target_file, + ">", + shrimp_outfile, + "2>", + shrimp_log, + ) + ) + command_end2 = " ".join( + ( + shrimp, + "-s", + spaced_seed, + "-n", + seed_matches_per_window, + "-t", + seed_hit_taboo_length, + "-9", + seed_generation_taboo_length, + "-w", + seed_window_length, + "-o", + max_hits_per_read, + "-r", + max_read_length, + "-d", + kmer, + "-m", + sw_match_value, + "-i", + sw_mismatch_value, + "-g", + sw_gap_open_ref, + "-q", + sw_gap_open_query, + "-e", + sw_gap_ext_ref, + "-f", + sw_gap_ext_query, + "-h", + sw_hit_threshold, + query_fasta_end2, + input_target_file, + ">>", + shrimp_outfile, + "2>>", + shrimp_log, + ) + ) try: os.system(command_end1) @@ -592,8 +759,8 @@ def __main__(): num_hits = 0 if shrimp_outfile: for line in open(shrimp_outfile): - line = line.rstrip('\r\n') - if not line or line.startswith('#'): + line = line.rstrip("\r\n") + if not line or line.startswith("#"): continue try: line.split() @@ -601,25 +768,32 @@ def __main__(): except Exception as e: stop_err(str(e)) - if num_hits == 0: # no hits generated - err_msg = '' + if num_hits == 0: # no hits generated + err_msg = "" if shrimp_log: for line in open(shrimp_log): - if line.startswith('error'): # deal with memory error: - err_msg += line # error: realloc failed: Cannot allocate memory - if re.search('Reads Matched', line): # deal with zero hits + if line.startswith("error"): # deal with memory error: + err_msg += line # error: realloc failed: Cannot allocate memory + if re.search("Reads Matched", line): # deal with zero hits if int(line[8:].split()[2]) == 0: - err_msg = 'Zero hits found.\n' - stop_err('SHRiMP Failed due to:\n' + err_msg) + err_msg = "Zero hits found.\n" + stop_err("SHRiMP Failed due to:\n" + err_msg) # convert to table - if type_of_reads == 'single': + if type_of_reads == "single": generate_sub_table(shrimp_outfile, input_target_file, query_qual, table_outfile, hit_per_read, insertion_size) else: - generate_sub_table(shrimp_outfile, input_target_file, query_qual_end1 + ',' + query_qual_end2, table_outfile, hit_per_read, insertion_size) + generate_sub_table( + shrimp_outfile, + input_target_file, + query_qual_end1 + "," + query_qual_end2, + table_outfile, + hit_per_read, + insertion_size, + ) # remove temp. files - if type_of_reads == 'single': + if type_of_reads == "single": if os.path.exists(query_fasta): os.remove(query_fasta) if os.path.exists(query_qual): @@ -638,5 +812,5 @@ def __main__(): os.remove(shrimp_log) -if __name__ == '__main__': +if __name__ == "__main__": __main__() diff --git a/tools/next_gen_conversion/fastq_conversions.py b/tools/next_gen_conversion/fastq_conversions.py index 5260d634b82..91c1abc1224 100644 --- a/tools/next_gen_conversion/fastq_conversions.py +++ b/tools/next_gen_conversion/fastq_conversions.py @@ -29,11 +29,11 @@ def __main__(): options, args = doc_optparse.parse(__doc__) cmd = "fq_all2std.pl %s %s > %s" - if options.command == 'sol2std': + if options.command == "sol2std": cmd = cmd % (options.command, options.input, options.outputFastqsanger) - elif options.command == 'std2sol': + elif options.command == "std2sol": cmd = cmd % (options.command, options.input, options.outputFastqsolexa) - elif options.command == 'fq2fa': + elif options.command == "fq2fa": cmd = cmd % (options.command, options.input, options.outputFasta) try: os.system(cmd) diff --git a/tools/next_gen_conversion/fastq_gen_conv.py b/tools/next_gen_conversion/fastq_gen_conv.py index aad04791c51..e55ac06db27 100644 --- a/tools/next_gen_conversion/fastq_gen_conv.py +++ b/tools/next_gen_conversion/fastq_gen_conv.py @@ -24,7 +24,7 @@ def stop_err(msg): def all_bases_valid(seq): """Confirm that the sequence contains only bases""" - valid_bases = ['a', 'A', 'c', 'C', 'g', 'G', 't', 'T', 'N'] + valid_bases = ["a", "A", "c", "C", "g", "G", "t", "T", "N"] for base in seq: if base not in valid_bases: return False @@ -35,12 +35,12 @@ def __main__(): # Parse Command Line options, args = doc_optparse.parse(__doc__) orig_type = options.origType - if orig_type == 'sanger' and options.allOrNot == 'not': + if orig_type == "sanger" and options.allOrNot == "not": max_blocks = int(options.blocks) else: max_blocks = -1 fin = open(options.input) - fout = open(options.output, 'w') + fout = open(options.output, "w") range_min = 1000 range_max = -5 block_num = 0 @@ -50,14 +50,14 @@ def __main__(): lines = [] line = fin.readline() while line: - if line.strip() and max_blocks >= 0 and block_num > 0 and orig_type == 'sanger' and block_num >= max_blocks: + if line.strip() and max_blocks >= 0 and block_num > 0 and orig_type == "sanger" and block_num >= max_blocks: fout.write(line) if line_count % 4 == 0: block_num += 1 line_count += 1 elif line.strip(): # the line that starts a block, with a name - if line_count % 4 == 0 and line.startswith('@'): + if line_count % 4 == 0 and line.startswith("@"): lines.append(line) else: # if we expect a sequence of bases @@ -65,7 +65,7 @@ def __main__(): lines.append(line) base_len = len(line.strip()) # if we expect the second name line - elif line_count % 4 == 2 and line.startswith('+'): + elif line_count % 4 == 2 and line.startswith("+"): lines.append(line) # if we expect a sequence of qualities and it's the expected length elif line_count % 4 == 3: @@ -88,7 +88,7 @@ def __main__(): for line_to_write in lines: fout.write(line_to_write) # print converted quality line - fout.write(''.join(phred_list)) + fout.write("".join(phred_list)) # reset lines = [] base_len = -1 @@ -101,7 +101,7 @@ def __main__(): elif len(split_line[0]) == base_len: qualities = [] # print converted quality line - if orig_type == 'illumina': + if orig_type == "illumina": for c in line.strip(): if ord(c) - 64 < range_min: range_min = ord(c) - 64 @@ -114,8 +114,8 @@ def __main__(): break else: qualities.append(chr(ord(c) - 31)) - quals = ''.join(qualities) - elif orig_type == 'solexa': + quals = "".join(qualities) + elif orig_type == "solexa": for c in line.strip(): if ord(c) - 64 < range_min: range_min = ord(c) - 64 @@ -129,7 +129,7 @@ def __main__(): else: p = 10.0 ** ((ord(c) - 64) / -10.0) / (1 + 10.0 ** ((ord(c) - 64) / -10.0)) qualities.append(chr(int(-10.0 * math.log10(p)) + 33)) - quals = ''.join(qualities) + quals = "".join(qualities) else: # 'sanger' for c in line.strip(): if ord(c) - 33 < range_min: @@ -143,14 +143,14 @@ def __main__(): break else: qualities.append(c) - quals = ''.join(qualities) + quals = "".join(qualities) # make sure we don't have bad qualities if len(quals) == base_len: # print first three lines for line_to_write in lines: fout.write(line_to_write) # print out quality line - fout.write(quals + '\n') + fout.write(quals + "\n") # reset lines = [] base_len = -1 @@ -165,11 +165,11 @@ def __main__(): fout.close() fin.close() if range_min != 1000 and range_min != -5: - outmsg = 'The range of quality values found were: %s to %s' % (range_min, range_max) + outmsg = "The range of quality values found were: %s to %s" % (range_min, range_max) else: - outmsg = '' + outmsg = "" if bad_blocks > 0: - outmsg += '\nThere were %s bad blocks skipped' % (bad_blocks) + outmsg += "\nThere were %s bad blocks skipped" % (bad_blocks) sys.stdout.write(outmsg) diff --git a/tools/next_gen_conversion/solid2fastq.py b/tools/next_gen_conversion/solid2fastq.py index a78ccdbfc3c..7a57ccd6162 100644 --- a/tools/next_gen_conversion/solid2fastq.py +++ b/tools/next_gen_conversion/solid2fastq.py @@ -22,7 +22,7 @@ def solid2sanger(quality_string, min_qual=0): for qv in quality_string.split(" "): try: if int(qv) < 0: - qv = '0' + qv = "0" if int(qv) < min_qual: return False break @@ -32,7 +32,7 @@ def solid2sanger(quality_string, min_qual=0): return sanger -def Translator(frm='', to='', delete=''): +def Translator(frm="", to="", delete=""): if len(to) == 1: to = to * len(frm) trans = maketrans(frm, to) @@ -43,7 +43,18 @@ def Translator(frm='', to='', delete=''): return callable -def merge_reads_qual(f_reads, f_qual, f_out, trim_name=False, out='fastq', double_encode=False, trim_first_base=False, pair_end_flag='', min_qual=0, table_name=None): +def merge_reads_qual( + f_reads, + f_qual, + f_out, + trim_name=False, + out="fastq", + double_encode=False, + trim_first_base=False, + pair_end_flag="", + min_qual=0, + table_name=None, +): # Reads from two files f_csfasta (reads) and f_qual (quality values) and produces output in three formats depending on out parameter, # which can have three values: fastq, txt, and db # fastq = fastq format @@ -52,7 +63,7 @@ def merge_reads_qual(f_reads, f_qual, f_out, trim_name=False, out='fastq', doubl # IMPORTNAT! If out = db two optins must be provided: # 1. f_out must be a db connection object initialized with sqlite3.connect() # 2. table_name must be provided - if out == 'db': + if out == "db": cursor = f_out.cursor() sql = "create table %s (name varchar(50) not null, read blob, qv blob)" % table_name cursor.execute(sql) @@ -61,20 +72,22 @@ def merge_reads_qual(f_reads, f_qual, f_out, trim_name=False, out='fastq', doubl line = " " while line: for f in [f_reads, f_qual]: - line = f.readline().rstrip('\n\r') - while line.startswith('#'): - line = f.readline().rstrip('\n\r') + line = f.readline().rstrip("\n\r") + while line.startswith("#"): + line = f.readline().rstrip("\n\r") lines.append(line) - if lines[0].startswith('>') and lines[1].startswith('>'): + if lines[0].startswith(">") and lines[1].startswith(">"): if lines[0] != lines[1]: - stop_err('Files reads and quality score files are out of sync and likely corrupted. Please, check your input data') + stop_err( + "Files reads and quality score files are out of sync and likely corrupted. Please, check your input data" + ) defline = lines[0][1:] - if trim_name and (defline[len(defline) - 3:] == "_F3" or defline[len(defline) - 3:] == "_R3"): - defline = defline[:len(defline) - 3] + if trim_name and (defline[len(defline) - 3 :] == "_F3" or defline[len(defline) - 3 :] == "_R3"): + defline = defline[: len(defline) - 3] - elif (not lines[0].startswith('>') and not lines[1].startswith('>') and len(lines[0]) > 0 and len(lines[1]) > 0): + elif not lines[0].startswith(">") and not lines[1].startswith(">") and len(lines[0]) > 0 and len(lines[1]) > 0: if trim_first_base: lines[0] = lines[0][1:] if double_encode: @@ -82,11 +95,11 @@ def merge_reads_qual(f_reads, f_qual, f_out, trim_name=False, out='fastq', doubl lines[0] = de(lines[0]) qual = solid2sanger(lines[1], int(min_qual)) if qual: - if out == 'fastq': + if out == "fastq": f_out.write("@%s%s\n%s\n+\n%s\n" % (defline, pair_end_flag, lines[0], qual)) - if out == 'txt': - f_out.write('%s %s %s\n' % (defline, lines[0], qual)) - if out == 'db': + if out == "txt": + f_out.write("%s %s %s\n" % (defline, lines[0], qual)) + if out == "db": cursor.execute('insert into %s values("%s","%s","%s")' % (table_name, defline, lines[0], qual)) lines = [] @@ -95,82 +108,84 @@ def main(): usage = "%prog --fr F3.csfasta --fq R3.csfasta --fout fastq_output_file [option]" parser = optparse.OptionParser(usage=usage) parser.add_option( - '--fr', '--f_reads', - metavar="F3_CSFASTA_FILE", - dest='fr', - help='Name of F3 file with color space reads') + "--fr", "--f_reads", metavar="F3_CSFASTA_FILE", dest="fr", help="Name of F3 file with color space reads" + ) parser.add_option( - '--fq', '--f_qual', - metavar="F3_QUAL_FILE", - dest='fq', - help='Name of F3 file with color quality values') + "--fq", "--f_qual", metavar="F3_QUAL_FILE", dest="fq", help="Name of F3 file with color quality values" + ) + parser.add_option("--fout", "--f3_fastq_output", metavar="F3_OUTPUT", dest="fout", help="Name for F3 output file") parser.add_option( - '--fout', '--f3_fastq_output', - metavar="F3_OUTPUT", - dest='fout', - help='Name for F3 output file') - parser.add_option( - '--rr', '--r_reads', + "--rr", + "--r_reads", metavar="R3_CSFASTA_FILE", - dest='rr', + dest="rr", default=False, - help='Name of R3 file with color space reads') + help="Name of R3 file with color space reads", + ) parser.add_option( - '--rq', '--r_qual', + "--rq", + "--r_qual", metavar="R3_QUAL_FILE", - dest='rq', + dest="rq", default=False, - help='Name of R3 file with color quality values') + help="Name of R3 file with color quality values", + ) + parser.add_option("--rout", metavar="R3_OUTPUT", dest="rout", help="Name for F3 output file") parser.add_option( - '--rout', - metavar="R3_OUTPUT", - dest='rout', - help='Name for F3 output file') + "-q", + "--min_qual", + dest="min_qual", + default="-1000", + help="Minimum quality threshold for printing reads. If a read contains a single call with QV lower than this value, it will not be reported. Default is -1000", + ) parser.add_option( - '-q', '--min_qual', - dest='min_qual', - default='-1000', - help='Minimum quality threshold for printing reads. If a read contains a single call with QV lower than this value, it will not be reported. Default is -1000') - parser.add_option( - '-t', '--trim_name', - dest='trim_name', - action='store_true', + "-t", + "--trim_name", + dest="trim_name", + action="store_true", default=False, - help='Trim _R3 and _F3 off read names. Default is False') + help="Trim _R3 and _F3 off read names. Default is False", + ) parser.add_option( - '-f', '--trim_first_base', - dest='trim_first_base', - action='store_true', + "-f", + "--trim_first_base", + dest="trim_first_base", + action="store_true", default=False, - help='Remove the first base of reads in color-space. Default is False') + help="Remove the first base of reads in color-space. Default is False", + ) parser.add_option( - '-d', '--double_encode', - dest='de', - action='store_true', + "-d", + "--double_encode", + dest="de", + action="store_true", default=False, - help='Double encode color calls as nucleotides: 0123. becomes ACGTN. Default is False') + help="Double encode color calls as nucleotides: 0123. becomes ACGTN. Default is False", + ) options, args = parser.parse_args() if not (options.fout and options.fr and options.fq): - parser.error(""" + parser.error( + """ One or more of the three required paremetrs is missing: (1) --fr F3.csfasta file (2) --fq F3.qual file (3) --fout name of output file Use --help for more info - """) + """ + ) fr = open(options.fr) fq = open(options.fq) - f_out = open(options.fout, 'w') + f_out = open(options.fout, "w") if options.rr and options.rq: rr = open(options.rr) rq = open(options.rq) if not options.rout: parser.error("Provide the name for f3 output using --rout option. Use --help for more info") - r_out = open(options.rout, 'w') + r_out = open(options.rout, "w") db = tempfile.NamedTemporaryFile() @@ -178,20 +193,49 @@ def main(): con = sqlite3.connect(db.name) cur = con.cursor() except Exception: - stop_err('Cannot connect to %s\n') % db.name + stop_err("Cannot connect to %s\n") % db.name - merge_reads_qual(fr, fq, con, trim_name=options.trim_name, out='db', double_encode=options.de, trim_first_base=options.trim_first_base, min_qual=options.min_qual, table_name="f3") - merge_reads_qual(rr, rq, con, trim_name=options.trim_name, out='db', double_encode=options.de, trim_first_base=options.trim_first_base, min_qual=options.min_qual, table_name="r3") - cur.execute('create index f3_name on f3( name )') - cur.execute('create index r3_name on r3( name )') + merge_reads_qual( + fr, + fq, + con, + trim_name=options.trim_name, + out="db", + double_encode=options.de, + trim_first_base=options.trim_first_base, + min_qual=options.min_qual, + table_name="f3", + ) + merge_reads_qual( + rr, + rq, + con, + trim_name=options.trim_name, + out="db", + double_encode=options.de, + trim_first_base=options.trim_first_base, + min_qual=options.min_qual, + table_name="r3", + ) + cur.execute("create index f3_name on f3( name )") + cur.execute("create index r3_name on r3( name )") - cur.execute('select * from f3,r3 where f3.name = r3.name') + cur.execute("select * from f3,r3 where f3.name = r3.name") for item in cur: f_out.write("@%s%s\n%s\n+\n%s\n" % (item[0], "/1", item[1], item[2])) r_out.write("@%s%s\n%s\n+\n%s\n" % (item[3], "/2", item[4], item[5])) else: - merge_reads_qual(fr, fq, f_out, trim_name=options.trim_name, out='fastq', double_encode=options.de, trim_first_base=options.trim_first_base, min_qual=options.min_qual) + merge_reads_qual( + fr, + fq, + f_out, + trim_name=options.trim_name, + out="fastq", + double_encode=options.de, + trim_first_base=options.trim_first_base, + min_qual=options.min_qual, + ) f_out.close() diff --git a/tools/next_gen_conversion/solid_to_fastq.py b/tools/next_gen_conversion/solid_to_fastq.py index a67a989f7a0..59d58b7689f 100644 --- a/tools/next_gen_conversion/solid_to_fastq.py +++ b/tools/next_gen_conversion/solid_to_fastq.py @@ -29,7 +29,7 @@ def stop_err(msg): def replaceNeg1(fin, fout): line = fin.readline() while line.strip(): - fout.write(line.replace('-1', '1')) + fout.write(line.replace("-1", "1")) line = fin.readline() fout.seek(0) return fout @@ -47,27 +47,43 @@ def __main__(): tmpr = tempfile.NamedTemporaryFile() # reverse reads # replace the -1 in the qualities file tmpqr = tempfile.NamedTemporaryFile() - tmpqr = replaceNeg1(open(options.input4, 'r'), tmpqr) - cmd1 = "%s/bwa_solid2fastq_modified.pl 'yes' %s %s %s %s %s %s 2>&1" % (os.path.split(sys.argv[0])[0], tmpf.name, tmpr.name, options.input1, tmpqf.name, options.input3, tmpqr.name) + tmpqr = replaceNeg1(open(options.input4, "r"), tmpqr) + cmd1 = "%s/bwa_solid2fastq_modified.pl 'yes' %s %s %s %s %s %s 2>&1" % ( + os.path.split(sys.argv[0])[0], + tmpf.name, + tmpr.name, + options.input1, + tmpqf.name, + options.input3, + tmpqr.name, + ) try: os.system(cmd1) - os.system('gunzip -c %s >> %s' % (tmpf.name, options.output1)) - os.system('gunzip -c %s >> %s' % (tmpr.name, options.output2)) + os.system("gunzip -c %s >> %s" % (tmpf.name, options.output1)) + os.system("gunzip -c %s >> %s" % (tmpr.name, options.output2)) except Exception as eq: stop_err("Error converting data to fastq format.\n" + str(eq)) tmpr.close() tmpqr.close() # if single-end data else: - cmd1 = "%s/bwa_solid2fastq_modified.pl 'no' %s %s %s %s %s %s 2>&1" % (os.path.split(sys.argv[0])[0], tmpf.name, None, options.input1, tmpqf.name, None, None) + cmd1 = "%s/bwa_solid2fastq_modified.pl 'no' %s %s %s %s %s %s 2>&1" % ( + os.path.split(sys.argv[0])[0], + tmpf.name, + None, + options.input1, + tmpqf.name, + None, + None, + ) try: os.system(cmd1) - os.system('gunzip -c %s >> %s' % (tmpf.name, options.output1)) + os.system("gunzip -c %s >> %s" % (tmpf.name, options.output1)) except Exception as eq: stop_err("Error converting data to fastq format.\n" + str(eq)) tmpqf.close() tmpf.close() - sys.stdout.write('converted SOLiD data') + sys.stdout.write("converted SOLiD data") if __name__ == "__main__": diff --git a/tools/ngs_simulation/ngs_simulation.py b/tools/ngs_simulation/ngs_simulation.py index 5a49caa1b02..fc6a226d603 100644 --- a/tools/ngs_simulation/ngs_simulation.py +++ b/tools/ngs_simulation/ngs_simulation.py @@ -32,7 +32,7 @@ from rpy import r def stop_err(msg): - sys.stderr.write('%s\n' % msg) + sys.stderr.write("%s\n" % msg) sys.exit() @@ -40,71 +40,71 @@ def __main__(): # Parse Command Line options, args = doc_optparse.parse(__doc__) # validate parameters - error = '' + error = "" try: read_len = int(options.read_len) if read_len <= 0: - raise Exception(' greater than 0') + raise Exception(" greater than 0") except TypeError as e: - error = ': %s' % str(e) + error = ": %s" % str(e) if error: - stop_err('Make sure your number of reads is an integer value%s' % error) - error = '' + stop_err("Make sure your number of reads is an integer value%s" % error) + error = "" try: avg_coverage = int(options.avg_coverage) if avg_coverage <= 0: - raise Exception(' greater than 0') + raise Exception(" greater than 0") except Exception as e: - error = ': %s' % str(e) + error = ": %s" % str(e) if error: - stop_err('Make sure your average coverage is an integer value%s' % error) - error = '' + stop_err("Make sure your average coverage is an integer value%s" % error) + error = "" try: error_rate = float(options.error_rate) if error_rate >= 1.0: error_rate = 10 ** (-error_rate / 10.0) elif error_rate < 0: - raise Exception(' between 0 and 1') + raise Exception(" between 0 and 1") except Exception as e: - error = ': %s' % str(e) + error = ": %s" % str(e) if error: - stop_err('Make sure the error rate is a decimal value%s or the quality score is at least 1' % error) + stop_err("Make sure the error rate is a decimal value%s or the quality score is at least 1" % error) try: num_sims = int(options.num_sims) except TypeError as e: - stop_err('Make sure the number of simulations is an integer value: %s' % str(e)) - if options.polymorphism != 'None': - polymorphisms = [float(p) for p in options.polymorphism.split(',')] + stop_err("Make sure the number of simulations is an integer value: %s" % str(e)) + if options.polymorphism != "None": + polymorphisms = [float(p) for p in options.polymorphism.split(",")] else: - stop_err('Select at least one polymorphism value to use') - if options.detection_thresh != 'None': - detection_threshes = [float(dt) for dt in options.detection_thresh.split(',')] + stop_err("Select at least one polymorphism value to use") + if options.detection_thresh != "None": + detection_threshes = [float(dt) for dt in options.detection_thresh.split(",")] else: - stop_err('Select at least one detection threshold to use') + stop_err("Select at least one detection threshold to use") # mutation dictionaries - hp_dict = {'A': 'G', 'G': 'A', 'C': 'T', 'T': 'C', 'N': 'N'} # heteroplasmy dictionary - mt_dict = {'A': 'C', 'C': 'A', 'G': 'T', 'T': 'G', 'N': 'N'} # misread dictionary + hp_dict = {"A": "G", "G": "A", "C": "T", "T": "C", "N": "N"} # heteroplasmy dictionary + mt_dict = {"A": "C", "C": "A", "G": "T", "T": "G", "N": "N"} # misread dictionary # read fasta file to seq string - all_lines = open(options.input, 'rb').readlines() - seq = '' + all_lines = open(options.input, "rb").readlines() + seq = "" for line in all_lines: line = line.rstrip() - if line.startswith('>'): + if line.startswith(">"): pass else: seq += line.upper() seq_len = len(seq) # output file name template -# removed output of all simulation results on request (not working) -# if options.sim_results == "true": -# out_name_template = os.path.join( options.new_file_path, 'primary_output%s_' + options.output + '_visible_tabular' ) -# else: -# out_name_template = tempfile.NamedTemporaryFile().name + '_%s' - out_name_template = tempfile.NamedTemporaryFile().name + '_%s' - print('out_name_template:', out_name_template) + # removed output of all simulation results on request (not working) + # if options.sim_results == "true": + # out_name_template = os.path.join( options.new_file_path, 'primary_output%s_' + options.output + '_visible_tabular' ) + # else: + # out_name_template = tempfile.NamedTemporaryFile().name + '_%s' + out_name_template = tempfile.NamedTemporaryFile().name + "_%s" + print("out_name_template:", out_name_template) # set up output files outputs = {} @@ -118,8 +118,8 @@ def __main__(): # run sims for polymorphism in polymorphisms: for detection_thresh in detection_threshes: - output = open(outputs[polymorphism][detection_thresh], 'wb') - output.write('FP\tFN\tGENOMESIZE=%s\n' % seq_len) + output = open(outputs[polymorphism][detection_thresh], "wb") + output.write("FP\tFN\tGENOMESIZE=%s\n" % seq_len) sim_count = 0 while sim_count < num_sims: # randomly pick heteroplasmic base index @@ -158,10 +158,10 @@ def __main__(): bases, fpos, fneg = {}, 0, 0 # last two will be outputted to summary file later for i, nuc in enumerate(seq): cov = len(qspec[i]) - bases['A'] = qspec[i].count('A') - bases['C'] = qspec[i].count('C') - bases['G'] = qspec[i].count('G') - bases['T'] = qspec[i].count('T') + bases["A"] = qspec[i].count("A") + bases["C"] = qspec[i].count("C") + bases["G"] = qspec[i].count("G") + bases["T"] = qspec[i].count("T") # calculate max NON-REF deviation del bases[nuc] maxdev = float(max(bases.values())) / cov @@ -172,108 +172,136 @@ def __main__(): # deal with het sites if i == hbase: hnuc = hp_dict[nuc] # let's recover het variant - if (float(bases[hnuc]) / cov) < detection_thresh: # less than detection threshold = false negative + if ( + float(bases[hnuc]) / cov + ) < detection_thresh: # less than detection threshold = false negative fneg += 1 del bases[hnuc] # ignore het variant maxdev = float(max(bases.values())) / cov # check other non-ref bases at het site if maxdev >= detection_thresh: # greater than detection threshold = false positive (possible) fpos += 1 # output error sums and genome size to summary file - output.write('%d\t%d\n' % (fpos, fneg)) + output.write("%d\t%d\n" % (fpos, fneg)) sim_count += 1 # close output up output.close() # Parameters (heteroplasmy, error threshold, colours) - r(''' + r( + """ het=c(%s) err=c(%s) grade = (0:32)/32 hues = rev(gray(grade)) - ''' % (','.join(str(p) for p in polymorphisms), ','.join(str(d) for d in detection_threshes))) + """ + % (",".join(str(p) for p in polymorphisms), ",".join(str(d) for d in detection_threshes)) + ) # Suppress warnings - r('options(warn=-1)') + r("options(warn=-1)") # Create allsum (for FP) and allneg (for FN) objects - r('allsum <- data.frame()') + r("allsum <- data.frame()") for polymorphism in polymorphisms: for detection_thresh in detection_threshes: output = outputs[polymorphism][detection_thresh] - cmd = ''' + cmd = """ ngsum = read.delim('%s', header=T) ngsum$fprate <- ngsum$FP/%s ngsum$hetcol <- %s ngsum$errcol <- %s allsum <- rbind(allsum, ngsum) - ''' % (output, seq_len, polymorphism, detection_thresh) + """ % ( + output, + seq_len, + polymorphism, + detection_thresh, + ) r(cmd) if os.path.getsize(output) == 0: for p in outputs.keys(): for d in outputs[p].keys(): - sys.stderr.write(outputs[p][d] + ' ' + str(os.path.getsize(outputs[p][d])) + '\n') + sys.stderr.write(outputs[p][d] + " " + str(os.path.getsize(outputs[p][d])) + "\n") if options.summary_out == "true": r('write.table(summary(ngsum), file="%s", quote=FALSE, sep="\t", row.names=FALSE)' % options.output_summary) # Summary objects (these could be printed) - r(''' + r( + """ tr_pos <- tapply(allsum$fprate,list(allsum$hetcol,allsum$errcol), mean) tr_neg <- tapply(allsum$FN,list(allsum$hetcol,allsum$errcol), mean) cat('\nFalse Positive Rate Summary\n\t', file='%s', append=T, sep='\t') write.table(format(tr_pos, digits=4), file='%s', append=T, quote=F, sep='\t') cat('\nFalse Negative Rate Summary\n\t', file='%s', append=T, sep='\t') write.table(format(tr_neg, digits=4), file='%s', append=T, quote=F, sep='\t') - ''' % tuple([options.output_summary] * 4)) + """ + % tuple([options.output_summary] * 4) + ) # Setup graphs - r(''' + r( + """ png('%s', width=800, height=500, units='px', res=250) layout(matrix(data=c(1,2,1,3,1,4), nrow=2, ncol=3), widths=c(4,6,2), heights=c(1,10,10)) - ''' % options.output_png) + """ + % options.output_png + ) # Main title - genome = '' + genome = "" if options.genome: - genome = '%s: ' % options.genome - r(''' + genome = "%s: " % options.genome + r( + """ par(mar=c(0,0,0,0)) plot(1, type='n', axes=F, xlab='', ylab='') text(1,1,paste('%sVariation in False Positives and Negatives (', %s, ' simulations, coverage ', %s,')', sep=''), font=2, family='sans', cex=0.7) - ''' % (genome, options.num_sims, options.avg_coverage)) + """ + % (genome, options.num_sims, options.avg_coverage) + ) # False positive boxplot - r(''' + r( + """ par(mar=c(5,4,2,2), las=1, cex=0.35) boxplot(allsum$fprate ~ allsum$errcol, horizontal=T, ylim=rev(range(allsum$fprate)), cex.axis=0.85) title(main='False Positives', xlab='false positive rate', ylab='') - ''') + """ + ) # False negative heatmap (note zlim command!) num_polys = len(polymorphisms) num_dets = len(detection_threshes) - r(''' + r( + """ par(mar=c(5,4,2,1), las=1, cex=0.35) image(1:%s, 1:%s, tr_neg, zlim=c(0,1), col=hues, xlab='', ylab='', axes=F, border=1) axis(1, at=1:%s, labels=rownames(tr_neg), lwd=1, cex.axis=0.85, axs='i') axis(2, at=1:%s, labels=colnames(tr_neg), lwd=1, cex.axis=0.85) title(main='False Negatives', xlab='minor allele frequency', ylab='detection threshold') - ''' % (num_polys, num_dets, num_polys, num_dets)) + """ + % (num_polys, num_dets, num_polys, num_dets) + ) # Scale alongside - r(''' + r( + """ par(mar=c(2,2,2,3), las=1) image(1, grade, matrix(grade, ncol=length(grade), nrow=1), col=hues, xlab='', ylab='', xaxt='n', las=1, cex.axis=0.85) title(main='Key', cex=0.35) mtext('false negative rate', side=1, cex=0.35) - ''') + """ + ) # Close graphics - r(''' + r( + """ layout(1) dev.off() - ''') + """ + ) if __name__ == "__main__": diff --git a/tools/phenotype_association/pagetag.py b/tools/phenotype_association/pagetag.py index 5490b6b63a7..7cd7e1fdcc1 100755 --- a/tools/phenotype_association/pagetag.py +++ b/tools/phenotype_association/pagetag.py @@ -61,15 +61,17 @@ HOMR = str(2) HETE = str(3) OTHER = str(4) -indexcalculator = {(HOMC, HOMC): 0, - (HOMC, HOMR): 1, - (HOMC, HETE): 2, - (HOMR, HOMC): 3, - (HOMR, HOMR): 4, - (HOMR, HETE): 5, - (HETE, HOMC): 6, - (HETE, HOMR): 7, - (HETE, HETE): 8} +indexcalculator = { + (HOMC, HOMC): 0, + (HOMC, HOMR): 1, + (HOMC, HETE): 2, + (HOMR, HOMC): 3, + (HOMR, HOMR): 4, + (HOMR, HETE): 5, + (HETE, HOMC): 6, + (HETE, HOMR): 7, + (HETE, HETE): 8, +} def read_inputfile(filename, samples): @@ -101,7 +103,7 @@ def annotate_locus(input, minorallelefrequency, snpsfile): genotypes = v.values() alleles = [y for x in genotypes for y in x] alleleset = list(set(alleles)) - alleleset = list(set(alleles) - {'N', 'X'}) + alleleset = list(set(alleles) - {"N", "X"}) if len(alleleset) == 2: genotypevec = "" @@ -141,7 +143,7 @@ def calculateLD(loci, rsqthreshold): rsquare = {} for index, loc1 in enumerate(snps): - for loc2 in snps[index + 1:]: + for loc2 in snps[index + 1 :]: matrix = [0] * 9 vec1 = loci[loc1][0] @@ -184,15 +186,14 @@ def calculateLD(loci, rsqthreshold): dvalue = p11 - (p * q) if dvalue != 0.0: - rsq = (dvalue ** 2) / (p * q * (1 - p) * (1 - q)) + rsq = (dvalue**2) / (p * q * (1 - p) * (1 - q)) if rsq >= rsqthreshold: rsquare["%s %s" % (loc1, loc2)] = rsq return rsquare -def main(inputfile, snpsfile, neigborhoodfile, - rsquare, minorallelefrequency, samples): +def main(inputfile, snpsfile, neigborhoodfile, rsquare, minorallelefrequency, samples): # read the input file input = read_inputfile(inputfile, samples) print("Read %d locations" % len(input), file=stderr) @@ -279,8 +280,7 @@ def usage(): if __name__ == "__main__": try: - opts, args = getopt(argv[1:], "hds:r:f:", - ["help", "debug", "rsquare=", "freq=", "sample="]) + opts, args = getopt(argv[1:], "hds:r:f:", ["help", "debug", "rsquare=", "freq=", "sample="]) except GetoptError as err: print(str(err)) usage() diff --git a/tools/phenotype_association/senatag.py b/tools/phenotype_association/senatag.py index 74f4b107f24..79a4847c96b 100755 --- a/tools/phenotype_association/senatag.py +++ b/tools/phenotype_association/senatag.py @@ -239,8 +239,7 @@ def usage(): if __name__ == "__main__": try: - opts, args = getopt(argv[1:], "hdr:e:", - ["help", "debug", "required=", "excluded="]) + opts, args = getopt(argv[1:], "hdr:e:", ["help", "debug", "required=", "excluded="]) except GetoptError as err: print(str(err)) usage() diff --git a/tools/plotting/bar_chart.py b/tools/plotting/bar_chart.py index c0e52d1d4da..8320c870b25 100644 --- a/tools/plotting/bar_chart.py +++ b/tools/plotting/bar_chart.py @@ -37,12 +37,12 @@ def stop_err(msg): def main(tmpFileName): skipped_lines_count = 0 skipped_lines_index = [] - gf = open(tmpFileName, 'w') + gf = open(tmpFileName, "w") try: in_file = open(sys.argv[1]) xtic = int(sys.argv[2]) - col_list = sys.argv[3].split(',') + col_list = sys.argv[3].split(",") title = 'set title "' + sys.argv[4] + '"' ylabel = 'set ylabel "' + sys.argv[5] + '"' ymin = sys.argv[6] @@ -55,15 +55,15 @@ def main(tmpFileName): try: int(col_list[0]) except Exception: - stop_err('You forgot to set columns for plotting\n') + stop_err("You forgot to set columns for plotting\n") for i, line in enumerate(in_file): valid = True - line = line.rstrip('\r\n') - if line and not line.startswith('#'): + line = line.rstrip("\r\n") + if line and not line.startswith("#"): row = [] try: - fields = line.split('\t') + fields = line.split("\t") for col in col_list: row.append(str(float(fields[int(col) - 1]))) except Exception: @@ -81,12 +81,12 @@ def main(tmpFileName): row.append(str(i)) if valid: - gf.write('\t'.join(row)) - gf.write('\n') + gf.write("\t".join(row)) + gf.write("\n") if skipped_lines_count < i: # Prepare 'using' clause of plot statement - g_plot_command = ' ' + g_plot_command = " " # Set the first column if xtic > 0: @@ -98,42 +98,45 @@ def main(tmpFileName): for i in range(1, len(col_list)): g_plot_command += "'%s' using %s t 'Column %s', " % (tmpFileName, str(i + 1), col_list[i]) - g_plot_command = g_plot_command.rstrip(', ') + g_plot_command = g_plot_command.rstrip(", ") - yrange = 'set yrange [' + ymin + ":" + ymax + ']' + yrange = "set yrange [" + ymin + ":" + ymax + "]" try: g = Gnuplot.Gnuplot() - g('reset') - g('set boxwidth 0.9 absolute') - g('set style fill solid 1.00 border -1') - g('set style histogram clustered gap 5 title offset character 0, 0, 0') - g('set xtics border in scale 1,0.5 nomirror rotate by 90 offset character 0, 0, 0') - g('set key invert reverse Left outside') + g("reset") + g("set boxwidth 0.9 absolute") + g("set style fill solid 1.00 border -1") + g("set style histogram clustered gap 5 title offset character 0, 0, 0") + g("set xtics border in scale 1,0.5 nomirror rotate by 90 offset character 0, 0, 0") + g("set key invert reverse Left outside") if xtic == 0: - g('unset xtics') + g("unset xtics") g(title) g(ylabel) - g_term = 'set terminal png tiny size ' + img_size + g_term = "set terminal png tiny size " + img_size g(g_term) g_out = 'set output "' + img_file + '"' if ymin != ymax: g(yrange) g(g_out) - g('set style data histograms') + g("set style data histograms") g.plot(g_plot_command) except Exception: stop_err("Gnuplot error: Data cannot be plotted") else: - sys.stderr.write('Column(s) %s of your dataset do not contain valid numeric data' % sys.argv[3]) + sys.stderr.write("Column(s) %s of your dataset do not contain valid numeric data" % sys.argv[3]) if skipped_lines_count > 0: - sys.stdout.write('\nWARNING. You dataset contain(s) %d invalid lines starting with line #%d. These lines were skipped while building the graph.\n' % (skipped_lines_count, skipped_lines_index[0] + 1)) + sys.stdout.write( + "\nWARNING. You dataset contain(s) %d invalid lines starting with line #%d. These lines were skipped while building the graph.\n" + % (skipped_lines_count, skipped_lines_index[0] + 1) + ) if __name__ == "__main__": # The tempfile initialization is here because while inside the main() it seems to create a condition # when the file is removed before gnuplot has a chance of accessing it - gp_data_file = tempfile.NamedTemporaryFile('w') - Gnuplot.gp.GnuplotOpts.default_term = 'png' + gp_data_file = tempfile.NamedTemporaryFile("w") + Gnuplot.gp.GnuplotOpts.default_term = "png" main(gp_data_file.name) diff --git a/tools/solid_tools/maq_cs_wrapper.py b/tools/solid_tools/maq_cs_wrapper.py index 5405bb84d32..75eab7e12ab 100644 --- a/tools/solid_tools/maq_cs_wrapper.py +++ b/tools/solid_tools/maq_cs_wrapper.py @@ -16,12 +16,12 @@ def stop_err(msg): def __main__(): out_fname = sys.argv[1].strip() - out_f2 = open(sys.argv[2].strip(), 'r+') + out_f2 = open(sys.argv[2].strip(), "r+") ref_fname = sys.argv[3].strip() f3_read_fname = sys.argv[4].strip() f3_qual_fname = sys.argv[5].strip() paired = sys.argv[6] - if paired == 'yes': + if paired == "yes": r3_read_fname = sys.argv[7].strip() r3_qual_fname = sys.argv[8].strip() min_mapqual = int(sys.argv[9].strip()) @@ -32,9 +32,9 @@ def __main__(): ref_csfa = tempfile.NamedTemporaryFile() ref_bfa = tempfile.NamedTemporaryFile() ref_csbfa = tempfile.NamedTemporaryFile() - cmd2_1 = 'maq fasta2csfa %s > %s 2>&1' % (ref_fname, ref_csfa.name) - cmd2_2 = 'maq fasta2bfa %s %s 2>&1' % (ref_csfa.name, ref_csbfa.name) - cmd2_3 = 'maq fasta2bfa %s %s 2>&1' % (ref_fname, ref_bfa.name) + cmd2_1 = "maq fasta2csfa %s > %s 2>&1" % (ref_fname, ref_csfa.name) + cmd2_2 = "maq fasta2bfa %s %s 2>&1" % (ref_csfa.name, ref_csbfa.name) + cmd2_3 = "maq fasta2bfa %s %s 2>&1" % (ref_fname, ref_bfa.name) try: os.system(cmd2_1) os.system(cmd2_2) @@ -42,7 +42,7 @@ def __main__(): except Exception as erf: stop_err(str(erf) + "Error processing reference sequence") - if paired == 'yes': # paired end reads + if paired == "yes": # paired end reads tmpf = tempfile.NamedTemporaryFile() # forward reads tmpr = tempfile.NamedTemporaryFile() # reverse reads tmps = tempfile.NamedTemporaryFile() # single reads @@ -50,12 +50,20 @@ def __main__(): tmprfastq = tempfile.NamedTemporaryFile() tmpsfastq = tempfile.NamedTemporaryFile() - cmd1 = "solid2fastq_modified.pl 'yes' %s %s %s %s %s %s %s 2>&1" % (tmpf.name, tmpr.name, tmps.name, f3_read_fname, f3_qual_fname, r3_read_fname, r3_qual_fname) + cmd1 = "solid2fastq_modified.pl 'yes' %s %s %s %s %s %s %s 2>&1" % ( + tmpf.name, + tmpr.name, + tmps.name, + f3_read_fname, + f3_qual_fname, + r3_read_fname, + r3_qual_fname, + ) try: os.system(cmd1) - os.system('gunzip -c %s >> %s' % (tmpf.name, tmpffastq.name)) - os.system('gunzip -c %s >> %s' % (tmpr.name, tmprfastq.name)) - os.system('gunzip -c %s >> %s' % (tmps.name, tmpsfastq.name)) + os.system("gunzip -c %s >> %s" % (tmpf.name, tmpffastq.name)) + os.system("gunzip -c %s >> %s" % (tmpr.name, tmprfastq.name)) + os.system("gunzip -c %s >> %s" % (tmps.name, tmpsfastq.name)) except Exception as eq: stop_err("Error converting data to fastq format." + str(eq)) @@ -65,8 +73,16 @@ def __main__(): split_dir = tempfile.mkdtemp() split_file_prefix_f = tempfile.mktemp(dir=split_dir) split_file_prefix_r = tempfile.mktemp(dir=split_dir) - splitcmd_f = 'split -a 2 -l %d %s %s' % (32000000, tmpffastq.name, split_file_prefix_f) # 32M lines correspond to 8M reads - splitcmd_r = 'split -a 2 -l %d %s %s' % (32000000, tmprfastq.name, split_file_prefix_r) # 32M lines correspond to 8M reads + splitcmd_f = "split -a 2 -l %d %s %s" % ( + 32000000, + tmpffastq.name, + split_file_prefix_f, + ) # 32M lines correspond to 8M reads + splitcmd_r = "split -a 2 -l %d %s %s" % ( + 32000000, + tmprfastq.name, + split_file_prefix_r, + ) # 32M lines correspond to 8M reads os.system(splitcmd_f) os.system(splitcmd_r) @@ -75,17 +91,33 @@ def __main__(): for fastq in os.listdir(split_dir): if not fastq.startswith(split_file_prefix_f.split("/")[-1]): continue - fastq_r = split_file_prefix_r + fastq.split(split_file_prefix_f.split("/")[-1])[1] # find the reverse strand fastq corresponding to forward strand fastq + fastq_r = ( + split_file_prefix_r + fastq.split(split_file_prefix_f.split("/")[-1])[1] + ) # find the reverse strand fastq corresponding to forward strand fastq tmpbfq_f = tempfile.NamedTemporaryFile() tmpbfq_r = tempfile.NamedTemporaryFile() - cmd3 = 'maq fastq2bfq %s %s 2>&1; maq fastq2bfq %s %s 2>&1; maq map -c %s.csmap %s %s %s 1>/dev/null 2>&1; maq mapview %s.csmap > %s.txt' % (fastq, tmpbfq_f.name, fastq_r, tmpbfq_r.name, fastq, ref_csbfa.name, tmpbfq_f.name, tmpbfq_r.name, fastq, fastq) - subprocess_dict['sp' + str(ii + 1)] = subprocess.Popen([cmd3], shell=True, stdout=subprocess.PIPE) + cmd3 = ( + "maq fastq2bfq %s %s 2>&1; maq fastq2bfq %s %s 2>&1; maq map -c %s.csmap %s %s %s 1>/dev/null 2>&1; maq mapview %s.csmap > %s.txt" + % ( + fastq, + tmpbfq_f.name, + fastq_r, + tmpbfq_r.name, + fastq, + ref_csbfa.name, + tmpbfq_f.name, + tmpbfq_r.name, + fastq, + fastq, + ) + ) + subprocess_dict["sp" + str(ii + 1)] = subprocess.Popen([cmd3], shell=True, stdout=subprocess.PIPE) ii += 1 while True: all_done = True for j in range(len(subprocess_dict)): - if subprocess_dict['sp' + str(j + 1)].wait() != 0: - err = subprocess_dict['sp' + str(j + 1)].communicate()[1] + if subprocess_dict["sp" + str(j + 1)].wait() != 0: + err = subprocess_dict["sp" + str(j + 1)].communicate()[1] if err is not None: stop_err("Mapping error: %s" % err) all_done = False @@ -99,7 +131,13 @@ def __main__(): os.system(cmd_cat_csmap) tmppileup = tempfile.NamedTemporaryFile() - cmdpileup = "maq pileup -m %s -q %s %s %s > %s" % (max_mismatch, min_mapqual, ref_bfa.name, tmpcsmap.name, tmppileup.name) + cmdpileup = "maq pileup -m %s -q %s %s %s > %s" % ( + max_mismatch, + min_mapqual, + ref_bfa.name, + tmpcsmap.name, + tmppileup.name, + ) os.system(cmdpileup) tmppileup.seek(0) print("#chr\tposition\tref_nt\tcoverage\tSNP_count\tA_count\tT_count\tG_count\tC_count", file=out_f2) @@ -112,13 +150,13 @@ def __main__(): ref_nt_count = 0 for ch in read_nt: ch = ch.capitalize() - if ch not in ['A', 'T', 'G', 'C', ',', '.']: + if ch not in ["A", "T", "G", "C", ",", "."]: continue - if ch in [',', '.']: + if ch in [",", "."]: ch = ref_nt ref_nt_count += 1 try: - nt_ind = ['A', 'T', 'G', 'C'].index(ch) + nt_ind = ["A", "T", "G", "C"].index(ch) if nt_ind == 0: a += 1 elif nt_ind == 1: @@ -129,17 +167,27 @@ def __main__(): c += 1 except ValueError as we: print(we, file=sys.stderr) - print("%s\t%s\t%s\t%s\t%s\t%s" % ("\t".join(elems[:4]), coverage - ref_nt_count, a, t, g, c), file=out_f2) + print( + "%s\t%s\t%s\t%s\t%s\t%s" % ("\t".join(elems[:4]), coverage - ref_nt_count, a, t, g, c), file=out_f2 + ) except Exception as er2: stop_err("Encountered error while mapping: %s" % (str(er2))) else: # single end reads tmpf = tempfile.NamedTemporaryFile() tmpfastq = tempfile.NamedTemporaryFile() - cmd1 = "solid2fastq_modified.pl 'no' %s %s %s %s %s %s %s 2>&1" % (tmpf.name, None, None, f3_read_fname, f3_qual_fname, None, None) + cmd1 = "solid2fastq_modified.pl 'no' %s %s %s %s %s %s %s 2>&1" % ( + tmpf.name, + None, + None, + f3_read_fname, + f3_qual_fname, + None, + None, + ) try: os.system(cmd1) - os.system('gunzip -c %s >> %s' % (tmpf.name, tmpfastq.name)) + os.system("gunzip -c %s >> %s" % (tmpf.name, tmpfastq.name)) tmpf.close() except Exception: stop_err("Error converting data to fastq format.") @@ -148,19 +196,26 @@ def __main__(): try: split_dir = tempfile.mkdtemp() split_file_prefix = tempfile.mktemp(dir=split_dir) - splitcmd = 'split -a 2 -l %d %s %s' % (32000000, tmpfastq.name, split_file_prefix) # 32M lines correspond to 8M reads + splitcmd = "split -a 2 -l %d %s %s" % ( + 32000000, + tmpfastq.name, + split_file_prefix, + ) # 32M lines correspond to 8M reads os.system(splitcmd) os.chdir(split_dir) for i, fastq in enumerate(os.listdir(split_dir)): tmpbfq = tempfile.NamedTemporaryFile() - cmd3 = 'maq fastq2bfq %s %s 2>&1; maq map -c %s.csmap %s %s 1>/dev/null 2>&1; maq mapview %s.csmap > %s.txt' % (fastq, tmpbfq.name, fastq, ref_csbfa.name, tmpbfq.name, fastq, fastq) - subprocess_dict['sp' + str(i + 1)] = subprocess.Popen([cmd3], shell=True, stdout=subprocess.PIPE) + cmd3 = ( + "maq fastq2bfq %s %s 2>&1; maq map -c %s.csmap %s %s 1>/dev/null 2>&1; maq mapview %s.csmap > %s.txt" + % (fastq, tmpbfq.name, fastq, ref_csbfa.name, tmpbfq.name, fastq, fastq) + ) + subprocess_dict["sp" + str(i + 1)] = subprocess.Popen([cmd3], shell=True, stdout=subprocess.PIPE) while True: all_done = True for j in range(len(subprocess_dict)): - if subprocess_dict['sp' + str(j + 1)].wait() != 0: - err = subprocess_dict['sp' + str(j + 1)].communicate()[1] + if subprocess_dict["sp" + str(j + 1)].wait() != 0: + err = subprocess_dict["sp" + str(j + 1)].communicate()[1] if err is not None: stop_err("Mapping error: %s" % err) all_done = False @@ -175,7 +230,13 @@ def __main__(): os.system(cmd_cat_csmap) tmppileup = tempfile.NamedTemporaryFile() - cmdpileup = "maq pileup -m %s -q %s %s %s > %s" % (max_mismatch, min_mapqual, ref_bfa.name, tmpcsmap.name, tmppileup.name) + cmdpileup = "maq pileup -m %s -q %s %s %s > %s" % ( + max_mismatch, + min_mapqual, + ref_bfa.name, + tmpcsmap.name, + tmppileup.name, + ) os.system(cmdpileup) tmppileup.seek(0) print("#chr\tposition\tref_nt\tcoverage\tSNP_count\tA_count\tT_count\tG_count\tC_count", file=out_f2) @@ -188,13 +249,13 @@ def __main__(): ref_nt_count = 0 for ch in read_nt: ch = ch.capitalize() - if ch not in ['A', 'T', 'G', 'C', ',', '.']: + if ch not in ["A", "T", "G", "C", ",", "."]: continue - if ch in [',', '.']: + if ch in [",", "."]: ch = ref_nt ref_nt_count += 1 try: - nt_ind = ['A', 'T', 'G', 'C'].index(ch) + nt_ind = ["A", "T", "G", "C"].index(ch) if nt_ind == 0: a += 1 elif nt_ind == 1: @@ -205,7 +266,9 @@ def __main__(): c += 1 except Exception: pass - print("%s\t%s\t%s\t%s\t%s\t%s" % ("\t".join(elems[:4]), coverage - ref_nt_count, a, t, g, c), file=out_f2) + print( + "%s\t%s\t%s\t%s\t%s\t%s" % ("\t".join(elems[:4]), coverage - ref_nt_count, a, t, g, c), file=out_f2 + ) except Exception as er2: stop_err("Encountered error while mapping: %s" % (str(er2))) @@ -217,11 +280,21 @@ def __main__(): fout_t = tempfile.NamedTemporaryFile() fout_g = tempfile.NamedTemporaryFile() fout_c = tempfile.NamedTemporaryFile() - fcov.write('''track type=wiggle_0 name="Coverage track" description="Coverage track (from Galaxy)" color=0,0,0 visibility=2\n''') - fout_a.write('''track type=wiggle_0 name="Track A" description="Track A (from Galaxy)" color=255,0,0 visibility=2\n''') - fout_t.write('''track type=wiggle_0 name="Track T" description="Track T (from Galaxy)" color=0,255,0 visibility=2\n''') - fout_g.write('''track type=wiggle_0 name="Track G" description="Track G (from Galaxy)" color=0,0,255 visibility=2\n''') - fout_c.write('''track type=wiggle_0 name="Track C" description="Track C (from Galaxy)" color=255,0,255 visibility=2\n''') + fcov.write( + """track type=wiggle_0 name="Coverage track" description="Coverage track (from Galaxy)" color=0,0,0 visibility=2\n""" + ) + fout_a.write( + """track type=wiggle_0 name="Track A" description="Track A (from Galaxy)" color=255,0,0 visibility=2\n""" + ) + fout_t.write( + """track type=wiggle_0 name="Track T" description="Track T (from Galaxy)" color=0,255,0 visibility=2\n""" + ) + fout_g.write( + """track type=wiggle_0 name="Track G" description="Track G (from Galaxy)" color=0,0,255 visibility=2\n""" + ) + fout_c.write( + """track type=wiggle_0 name="Track C" description="Track C (from Galaxy)" color=255,0,255 visibility=2\n""" + ) for line in out_f2: if line.startswith("#"): @@ -231,8 +304,8 @@ def __main__(): if chr not in chr_list: chr_list.append(chr) - if not (chr.startswith('chr') or chr.startswith('scaffold')): - chr = 'chr' + if not (chr.startswith("chr") or chr.startswith("scaffold")): + chr = "chr" header = "variableStep chrom=%s" % (chr) fcov.write("%s\n" % (header)) fout_a.write("%s\n" % (header)) @@ -250,10 +323,10 @@ def __main__(): continue fcov.write("%s\t%s\n" % (pos, cov)) try: - a_freq = a * 100. / cov - t_freq = t * 100. / cov - g_freq = g * 100. / cov - c_freq = c * 100. / cov + a_freq = a * 100.0 / cov + t_freq = t * 100.0 / cov + g_freq = g * 100.0 / cov + c_freq = c * 100.0 / cov except ZeroDivisionError: a_freq = t_freq = g_freq = c_freq = 0 fout_a.write("%s\t%s\n" % (pos, a_freq)) @@ -266,7 +339,9 @@ def __main__(): fout_g.seek(0) fout_t.seek(0) fout_c.seek(0) - os.system("cat %s %s %s %s %s | cat > %s" % (fcov.name, fout_a.name, fout_t.name, fout_g.name, fout_c.name, out_f3name)) + os.system( + "cat %s %s %s %s %s | cat > %s" % (fcov.name, fout_a.name, fout_t.name, fout_g.name, fout_c.name, out_f3name) + ) if __name__ == "__main__": diff --git a/tools/solid_tools/maq_cs_wrapper_code.py b/tools/solid_tools/maq_cs_wrapper_code.py index 7a0a7e7f108..6354ee3065d 100644 --- a/tools/solid_tools/maq_cs_wrapper_code.py +++ b/tools/solid_tools/maq_cs_wrapper_code.py @@ -1,4 +1,4 @@ def exec_before_job(app, inp_data, out_data, param_dict, tool): - out_data['output1'].name = out_data['output1'].name + " [ ALIGNMENT INFO ]" - out_data['output2'].name = out_data['output2'].name + " [ PILEUP ]" - out_data['output3'].name = out_data['output3'].name + " [ CUSTOM TRACK ]" + out_data["output1"].name = out_data["output1"].name + " [ ALIGNMENT INFO ]" + out_data["output2"].name = out_data["output2"].name + " [ PILEUP ]" + out_data["output3"].name = out_data["output3"].name + " [ CUSTOM TRACK ]" diff --git a/tools/solid_tools/solid_qual_stats.py b/tools/solid_tools/solid_qual_stats.py index 30239ccb979..1504e9c75c6 100644 --- a/tools/solid_tools/solid_qual_stats.py +++ b/tools/solid_tools/solid_qual_stats.py @@ -16,17 +16,17 @@ def stop_err(msg): def unzip(filename): - zip_file = zipfile.ZipFile(filename, 'r') + zip_file = zipfile.ZipFile(filename, "r") tmpfilename = tempfile.NamedTemporaryFile().name for name in zip_file.namelist(): - open(tmpfilename, 'a').write(zip_file.read(name)) + open(tmpfilename, "a").write(zip_file.read(name)) zip_file.close() return tmpfilename def __main__(): infile_score_name = sys.argv[1].strip() - with open(sys.argv[2].strip(), 'w') as fout: + with open(sys.argv[2].strip(), "w") as fout: if zipfile.is_zipfile(infile_score_name): infile_name = unzip(infile_score_name) @@ -39,7 +39,7 @@ def __main__(): with open(infile_name) as fin: for line in fin: line = line.strip() - if not(line) or line.startswith("#") or line.startswith(">"): + if not (line) or line.startswith("#") or line.startswith(">"): continue elems = line.split() try: @@ -59,7 +59,7 @@ def __main__(): print("column\tcount\tmin\tmax\tsum\tmean\tQ1\tmed\tQ3\tIQR\tlW\trW", file=fout) for line in open(infile_name): line = line.strip() - if not(line) or line.startswith("#") or line.startswith(">"): + if not (line) or line.startswith("#") or line.startswith(">"): continue elems = line.split() if position_dict == {}: @@ -80,12 +80,12 @@ def __main__(): carr = position_dict[pos] # count array for position pos total = sum(carr) # number of bases found in this column. med_elem = int(round(total / 2.0)) - lowest = None # Lowest quality score value found in this column. + lowest = None # Lowest quality score value found in this column. highest = None # Highest quality score value found in this column. - median = None # Median quality score value found in this column. - qsum = 0.0 # Sum of quality score values for this column. - q1 = None # 1st quartile quality score. - q3 = None # 3rd quartile quality score. + median = None # Median quality score value found in this column. + qsum = 0.0 # Sum of quality score values for this column. + q1 = None # 1st quartile quality score. + q3 = None # 3rd quartile quality score. q1_elem = int(round((total + 1) / 4.0)) q3_elem = int(round((total + 1) * 3 / 4.0)) @@ -100,24 +100,24 @@ def __main__(): lowest = ind if q1 is None: - if sum(carr[:ind + 1]) >= q1_elem: + if sum(carr[: ind + 1]) >= q1_elem: q1 = ind if median is None: - if sum(carr[:ind + 1]) < med_elem: + if sum(carr[: ind + 1]) < med_elem: continue median = ind if total % 2 == 0: # even number of elements median2 = median - if sum(carr[:ind + 1]) < med_elem + 1: - for ind2, elem in enumerate(carr[ind + 1:]): + if sum(carr[: ind + 1]) < med_elem + 1: + for ind2, elem in enumerate(carr[ind + 1 :]): if elem != 0: median2 = ind + ind2 + 1 break median = (median + median2) / 2.0 if q3 is None: - if sum(carr[:ind + 1]) >= q3_elem: + if sum(carr[: ind + 1]) >= q3_elem: q3 = ind mean = qsum / total # Mean quality score value for this column. @@ -125,11 +125,28 @@ def __main__(): left_whisker = max(q1 - 1.5 * iqr, lowest) right_whisker = min(q3 + 1.5 * iqr, highest) - print("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" % (pos + 1, total, lowest, highest, qsum, mean, q1, median, q3, iqr, left_whisker, right_whisker), file=fout) + print( + "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" + % ( + pos + 1, + total, + lowest, + highest, + qsum, + mean, + q1, + median, + q3, + iqr, + left_whisker, + right_whisker, + ), + file=fout, + ) except Exception: invalid_positions += 1 - nullvals = ['NA'] * 11 - print("%s\t%s" % (pos + 1, '\t'.join(nullvals)), file=fout) + nullvals = ["NA"] * 11 + print("%s\t%s" % (pos + 1, "\t".join(nullvals)), file=fout) if invalid_lines: print("Skipped %d reads as invalid." % invalid_lines) diff --git a/tools/sr_assembly/velvetg_wrapper.py b/tools/sr_assembly/velvetg_wrapper.py index 7f85d51c842..a6ef93e4116 100644 --- a/tools/sr_assembly/velvetg_wrapper.py +++ b/tools/sr_assembly/velvetg_wrapper.py @@ -15,16 +15,16 @@ assert sys.version_info[:2] >= (2, 6) def __main__(): # Parse Command Line working_dir = sys.argv[1] - inputs = ' '.join(sys.argv[2:]) - for _ in ('Roadmaps', 'Sequences'): + inputs = " ".join(sys.argv[2:]) + for _ in ("Roadmaps", "Sequences"): os.symlink(os.path.join(working_dir, _), _) - cmdline = 'velvetg . %s' % (inputs) + cmdline = "velvetg . %s" % (inputs) print("Command to be executed: %s" % cmdline) try: proc = subprocess.Popen(args=cmdline, shell=True, stderr=subprocess.PIPE) returncode = proc.wait() # get stderr, allowing for case where it's very large - stderr = b'' + stderr = b"" buffsize = 1048576 try: while True: @@ -36,7 +36,7 @@ def __main__(): if returncode != 0: raise Exception(stderr) except Exception as e: - sys.exit('Error running velvetg ' + str(e)) + sys.exit("Error running velvetg " + str(e)) if __name__ == "__main__": diff --git a/tools/sr_assembly/velveth_wrapper.py b/tools/sr_assembly/velveth_wrapper.py index 22257a47b59..42d49fce862 100644 --- a/tools/sr_assembly/velveth_wrapper.py +++ b/tools/sr_assembly/velveth_wrapper.py @@ -17,16 +17,16 @@ def __main__(): try: # for test - needs this done os.makedirs(working_dir) except Exception as e: - sys.exit('Error running velveth ' + str(e)) + sys.exit("Error running velveth " + str(e)) hash_length = sys.argv[3] - inputs = ' '.join(sys.argv[4:]) - cmdline = 'velveth %s %s %s > /dev/null' % (working_dir, hash_length, inputs) + inputs = " ".join(sys.argv[4:]) + cmdline = "velveth %s %s %s > /dev/null" % (working_dir, hash_length, inputs) try: proc = subprocess.Popen(args=cmdline, shell=True, stderr=subprocess.PIPE) returncode = proc.wait() # get stderr, allowing for case where it's very large - stderr = b'' + stderr = b"" buffsize = 1048576 try: while True: @@ -38,19 +38,19 @@ def __main__(): if returncode != 0: raise Exception(stderr) except Exception as e: - sys.exit('Error running velveth ' + str(e)) + sys.exit("Error running velveth " + str(e)) - sequences_path = os.path.join(working_dir, 'Sequences') - roadmaps_path = os.path.join(working_dir, 'Roadmaps') - rval = ['Velvet Galaxy Composite Dataset

'] - rval.append('

%s

' % (cmdline)) - rval.append('
This composite dataset is composed of the following files:

    ') - rval.append('
  • %s %s
    • ' % (sequences_path, 'Sequences', 'Sequences')) - rval.append('
  • %s %s
    • ' % (roadmaps_path, 'Roadmaps', 'Roadmaps')) - rval.append('
') - with open(html_file, 'w') as f: + sequences_path = os.path.join(working_dir, "Sequences") + roadmaps_path = os.path.join(working_dir, "Roadmaps") + rval = ["Velvet Galaxy Composite Dataset

"] + rval.append("

%s

" % (cmdline)) + rval.append("
This composite dataset is composed of the following files:

    ") + rval.append('
  • %s %s
    • ' % (sequences_path, "Sequences", "Sequences")) + rval.append('
  • %s %s
    • ' % (roadmaps_path, "Roadmaps", "Roadmaps")) + rval.append("
") + with open(html_file, "w") as f: f.write("\n".join(rval)) - f.write('\n') + f.write("\n") if __name__ == "__main__": diff --git a/tools/sr_mapping/bfast_wrapper.py b/tools/sr_mapping/bfast_wrapper.py index b8a5e9e4dd4..7685652e975 100644 --- a/tools/sr_mapping/bfast_wrapper.py +++ b/tools/sr_mapping/bfast_wrapper.py @@ -48,69 +48,196 @@ import tempfile def stop_err(msg): - sys.stderr.write('%s\n' % msg) + sys.stderr.write("%s\n" % msg) sys.exit() def __main__(): parser = optparse.OptionParser() - parser.add_option('-r', '--ref', dest='ref', help='The reference genome to index and use') - parser.add_option('-f', '--fastq', dest='fastq', help='The fastq file to use for the mapping') - parser.add_option('-F', '--output', dest='output', help='The file to save the output (SAM format)') - parser.add_option('-A', '--space', dest='space', type="choice", default='0', choices=('0', '1'), help='The encoding space (0: base 1: color)') - parser.add_option('-H', '--suppressHeader', action="store_true", dest='suppressHeader', default=False, help='Suppress header') - parser.add_option('-n', '--numThreads', dest='numThreads', type="int", default="1", help='The number of threads to use') - parser.add_option('-t', '--timing', action="store_true", default=False, dest='timing', help='output timming information to stderr') - parser.add_option('-l', '--loadAllIndexes', action="store_true", default=False, dest='loadAllIndexes', help='Load all indexes into memory') - parser.add_option('-m', '--indexMask', dest='indexMask', help='String containing info on how to build custom indexes') - parser.add_option("-b", "--buildIndex", action="store_true", dest="buildIndex", default=False, help='String containing info on how to build custom indexes') - parser.add_option("--indexRepeatMasker", action="store_true", dest="indexRepeatMasker", default=False, help='Do not index lower case sequences. Such as those created by RepeatMasker') - parser.add_option('--indexContigOptions', dest='indexContigOptions', default="", help='The contig range options to use for the indexing') - parser.add_option('--indexExonsFileName', dest='indexExonsFileName', default="", help='The exons file to use for the indexing') + parser.add_option("-r", "--ref", dest="ref", help="The reference genome to index and use") + parser.add_option("-f", "--fastq", dest="fastq", help="The fastq file to use for the mapping") + parser.add_option("-F", "--output", dest="output", help="The file to save the output (SAM format)") + parser.add_option( + "-A", + "--space", + dest="space", + type="choice", + default="0", + choices=("0", "1"), + help="The encoding space (0: base 1: color)", + ) + parser.add_option( + "-H", "--suppressHeader", action="store_true", dest="suppressHeader", default=False, help="Suppress header" + ) + parser.add_option( + "-n", "--numThreads", dest="numThreads", type="int", default="1", help="The number of threads to use" + ) + parser.add_option( + "-t", "--timing", action="store_true", default=False, dest="timing", help="output timming information to stderr" + ) + parser.add_option( + "-l", + "--loadAllIndexes", + action="store_true", + default=False, + dest="loadAllIndexes", + help="Load all indexes into memory", + ) + parser.add_option( + "-m", "--indexMask", dest="indexMask", help="String containing info on how to build custom indexes" + ) + parser.add_option( + "-b", + "--buildIndex", + action="store_true", + dest="buildIndex", + default=False, + help="String containing info on how to build custom indexes", + ) + parser.add_option( + "--indexRepeatMasker", + action="store_true", + dest="indexRepeatMasker", + default=False, + help="Do not index lower case sequences. Such as those created by RepeatMasker", + ) + parser.add_option( + "--indexContigOptions", + dest="indexContigOptions", + default="", + help="The contig range options to use for the indexing", + ) + parser.add_option( + "--indexExonsFileName", dest="indexExonsFileName", default="", help="The exons file to use for the indexing" + ) - parser.add_option('-o', '--offsets', dest='offsets', default="", help='The offsets for \'match\'') - parser.add_option('-k', '--keySize', dest='keySize', type="int", default="-1", help='truncate key size in \'match\'') - parser.add_option('-K', '--maxKeyMatches', dest='maxKeyMatches', type="int", default="-1", help='the maximum number of matches to allow before a key is ignored') - parser.add_option('-M', '--maxNumMatches', dest='maxNumMatches', type="int", default="-1", help='the maximum number of matches to allow bfore the read is discarded') - parser.add_option('-w', '--whichStrand', dest='whichStrand', type="choice", default='0', choices=('0', '1', '2'), help='the strands to consider (0: both 1: forward 2: reverse)') + parser.add_option("-o", "--offsets", dest="offsets", default="", help="The offsets for 'match'") + parser.add_option("-k", "--keySize", dest="keySize", type="int", default="-1", help="truncate key size in 'match'") + parser.add_option( + "-K", + "--maxKeyMatches", + dest="maxKeyMatches", + type="int", + default="-1", + help="the maximum number of matches to allow before a key is ignored", + ) + parser.add_option( + "-M", + "--maxNumMatches", + dest="maxNumMatches", + type="int", + default="-1", + help="the maximum number of matches to allow bfore the read is discarded", + ) + parser.add_option( + "-w", + "--whichStrand", + dest="whichStrand", + type="choice", + default="0", + choices=("0", "1", "2"), + help="the strands to consider (0: both 1: forward 2: reverse)", + ) - parser.add_option('--scoringMatrixFileName', dest='scoringMatrixFileName', help='Scoring Matrix file used to score the alignments') - parser.add_option('-u', '--ungapped', dest='ungapped', action="store_true", default=False, help='performed ungapped local alignment') - parser.add_option('-U', '--unconstrained', dest='unconstrained', action="store_true", default=False, help='performed local alignment without mask constraints') - parser.add_option('-O', '--offset', dest='offset', type="int", default="0", help='the number of bases before and after each hit to consider in local alignment') - parser.add_option('-q', '--avgMismatchQuality', type="int", default="-1", dest='avgMismatchQuality', help='average mismatch quality') + parser.add_option( + "--scoringMatrixFileName", dest="scoringMatrixFileName", help="Scoring Matrix file used to score the alignments" + ) + parser.add_option( + "-u", + "--ungapped", + dest="ungapped", + action="store_true", + default=False, + help="performed ungapped local alignment", + ) + parser.add_option( + "-U", + "--unconstrained", + dest="unconstrained", + action="store_true", + default=False, + help="performed local alignment without mask constraints", + ) + parser.add_option( + "-O", + "--offset", + dest="offset", + type="int", + default="0", + help="the number of bases before and after each hit to consider in local alignment", + ) + parser.add_option( + "-q", + "--avgMismatchQuality", + type="int", + default="-1", + dest="avgMismatchQuality", + help="average mismatch quality", + ) - parser.add_option('-a', '--algorithm', dest='algorithm', default='0', type="choice", choices=('0', '1', '2', '3', '4'), help='post processing algorithm (0: no filtering, 1: all passing filters, 2: unique, 3: best scoring unique, 4: best score all') - parser.add_option('--unpaired', dest='unpaired', action="store_true", default=False, help='do not choose alignments based on pairing') - parser.add_option('--reverseStrand', dest='reverseStrand', action="store_true", default=False, help='paired end reads are given on reverse strands') - parser.add_option('--pairedEndInfer', dest='pairedEndInfer', action="store_true", default=False, help='break ties when one end of a paired end read by estimating the insert size distribution') - parser.add_option('--randomBest', dest='randomBest', action="store_true", default=False, help='output a random best scoring alignment') + parser.add_option( + "-a", + "--algorithm", + dest="algorithm", + default="0", + type="choice", + choices=("0", "1", "2", "3", "4"), + help="post processing algorithm (0: no filtering, 1: all passing filters, 2: unique, 3: best scoring unique, 4: best score all", + ) + parser.add_option( + "--unpaired", + dest="unpaired", + action="store_true", + default=False, + help="do not choose alignments based on pairing", + ) + parser.add_option( + "--reverseStrand", + dest="reverseStrand", + action="store_true", + default=False, + help="paired end reads are given on reverse strands", + ) + parser.add_option( + "--pairedEndInfer", + dest="pairedEndInfer", + action="store_true", + default=False, + help="break ties when one end of a paired end read by estimating the insert size distribution", + ) + parser.add_option( + "--randomBest", + dest="randomBest", + action="store_true", + default=False, + help="output a random best scoring alignment", + ) (options, args) = parser.parse_args() # output version # of tool try: tmp = tempfile.NamedTemporaryFile().name - tmp_stdout = open(tmp, 'wb') - proc = subprocess.Popen(args='bfast 2>&1', shell=True, stdout=tmp_stdout) + tmp_stdout = open(tmp, "wb") + proc = subprocess.Popen(args="bfast 2>&1", shell=True, stdout=tmp_stdout) tmp_stdout.close() returncode = proc.wait() stdout = None - for line in open(tmp_stdout.name, 'rb'): - if line.lower().find('version') >= 0: + for line in open(tmp_stdout.name, "rb"): + if line.lower().find("version") >= 0: stdout = line.strip() break if stdout: - sys.stdout.write('%s\n' % stdout) + sys.stdout.write("%s\n" % stdout) else: raise Exception except Exception: - sys.stdout.write('Could not determine BFAST version\n') + sys.stdout.write("Could not determine BFAST version\n") buffsize = 1048576 # make temp directory for bfast, requires trailing slash - tmp_dir = '%s/' % tempfile.mkdtemp() + tmp_dir = "%s/" % tempfile.mkdtemp() # 'generic' options used in all bfast commands here if options.timing: @@ -120,7 +247,7 @@ def __main__(): try: if options.buildIndex: - reference_filepath = tempfile.NamedTemporaryFile(dir=tmp_dir, suffix='.fa').name + reference_filepath = tempfile.NamedTemporaryFile(dir=tmp_dir, suffix=".fa").name # build bfast indexes os.symlink(options.ref, reference_filepath) @@ -133,13 +260,13 @@ def __main__(): for space in nuc_space: cmd = 'bfast fasta2brg -f "%s" -A "%s" %s' % (reference_filepath, space, all_cmd_options) tmp = tempfile.NamedTemporaryFile(dir=tmp_dir).name - tmp_stderr = open(tmp, 'wb') + tmp_stderr = open(tmp, "wb") proc = subprocess.Popen(args=cmd, shell=True, cwd=tmp_dir, stderr=tmp_stderr.fileno()) returncode = proc.wait() tmp_stderr.close() # get stderr, allowing for case where it's very large - tmp_stderr = open(tmp, 'rb') - stderr = '' + tmp_stderr = open(tmp, "rb") + stderr = "" try: while True: stderr += tmp_stderr.read(buffsize) @@ -151,17 +278,22 @@ def __main__(): if returncode != 0: raise Exception(stderr) except Exception as e: - raise Exception('Error in \'bfast fasta2brg\'.\n' + str(e)) + raise Exception("Error in 'bfast fasta2brg'.\n" + str(e)) # bfast index try: - all_index_cmds = 'bfast index %s -f "%s" -A "%s" -n "%s"' % (all_cmd_options, reference_filepath, options.space, options.numThreads) + all_index_cmds = 'bfast index %s -f "%s" -A "%s" -n "%s"' % ( + all_cmd_options, + reference_filepath, + options.space, + options.numThreads, + ) if options.indexRepeatMasker: all_index_cmds += " -R" if options.indexContigOptions: - index_contig_options = [int(_) for _ in options.indexContigOptions.split(',')] + index_contig_options = [int(_) for _ in options.indexContigOptions.split(",")] if index_contig_options[0] >= 0: all_index_cmds += ' -s "%s"' % index_contig_options[0] if index_contig_options[1] >= 0: @@ -174,16 +306,16 @@ def __main__(): all_index_cmds += ' -x "%s"' % options.indexExonsFileName index_count = 1 - for mask, hash_width in [mask.split(':') for mask in options.indexMask.split(',')]: + for mask, hash_width in [mask.split(":") for mask in options.indexMask.split(",")]: cmd = '%s -m "%s" -w "%s" -i "%i"' % (all_index_cmds, mask, hash_width, index_count) tmp = tempfile.NamedTemporaryFile(dir=tmp_dir).name - tmp_stderr = open(tmp, 'wb') + tmp_stderr = open(tmp, "wb") proc = subprocess.Popen(args=cmd, shell=True, cwd=tmp_dir, stderr=tmp_stderr.fileno()) returncode = proc.wait() tmp_stderr.close() # get stderr, allowing for case where it's very large - tmp_stderr = open(tmp, 'rb') - stderr = '' + tmp_stderr = open(tmp, "rb") + stderr = "" try: while True: stderr += tmp_stderr.read(buffsize) @@ -196,11 +328,11 @@ def __main__(): raise Exception(stderr) index_count += 1 except Exception as e: - raise Exception('Error in \'bfast index\'.\n' + str(e)) + raise Exception("Error in 'bfast index'.\n" + str(e)) else: reference_filepath = options.ref - assert reference_filepath and os.path.exists(reference_filepath), 'A valid genome reference was not provided.' + assert reference_filepath and os.path.exists(reference_filepath), "A valid genome reference was not provided." # set up aligning and generate aligning command options # set up temp output files @@ -211,9 +343,31 @@ def __main__(): tmp_baf_name = tmp_baf.name tmp_baf.close() - bfast_match_cmd = 'bfast match -f "%s" -r "%s" -n "%s" -A "%s" -T "%s" -w "%s" %s' % (reference_filepath, options.fastq, options.numThreads, options.space, tmp_dir, options.whichStrand, all_cmd_options) - bfast_localalign_cmd = 'bfast localalign -f "%s" -m "%s" -n "%s" -A "%s" -o "%s" %s' % (reference_filepath, tmp_bmf_name, options.numThreads, options.space, options.offset, all_cmd_options) - bfast_postprocess_cmd = 'bfast postprocess -O 1 -f "%s" -i "%s" -n "%s" -A "%s" -a "%s" %s' % (reference_filepath, tmp_baf_name, options.numThreads, options.space, options.algorithm, all_cmd_options) + bfast_match_cmd = 'bfast match -f "%s" -r "%s" -n "%s" -A "%s" -T "%s" -w "%s" %s' % ( + reference_filepath, + options.fastq, + options.numThreads, + options.space, + tmp_dir, + options.whichStrand, + all_cmd_options, + ) + bfast_localalign_cmd = 'bfast localalign -f "%s" -m "%s" -n "%s" -A "%s" -o "%s" %s' % ( + reference_filepath, + tmp_bmf_name, + options.numThreads, + options.space, + options.offset, + all_cmd_options, + ) + bfast_postprocess_cmd = 'bfast postprocess -O 1 -f "%s" -i "%s" -n "%s" -A "%s" -a "%s" %s' % ( + reference_filepath, + tmp_baf_name, + options.numThreads, + options.space, + options.algorithm, + all_cmd_options, + ) if options.offsets: bfast_match_cmd += ' -o "%s"' % options.offsets @@ -228,21 +382,21 @@ def __main__(): bfast_localalign_cmd += ' -x "%s"' % options.scoringMatrixFileName bfast_postprocess_cmd += ' -x "%s"' % options.scoringMatrixFileName if options.ungapped: - bfast_localalign_cmd += ' -u' + bfast_localalign_cmd += " -u" if options.unconstrained: - bfast_localalign_cmd += ' -U' + bfast_localalign_cmd += " -U" if options.avgMismatchQuality >= 0: bfast_localalign_cmd += ' -q "%s"' % options.avgMismatchQuality bfast_postprocess_cmd += ' -q "%s"' % options.avgMismatchQuality if options.algorithm == 3: if options.pairedEndInfer: - bfast_postprocess_cmd += ' -P' + bfast_postprocess_cmd += " -P" if options.randomBest: - bfast_postprocess_cmd += ' -z' + bfast_postprocess_cmd += " -z" if options.unpaired: - bfast_postprocess_cmd += ' -U' + bfast_postprocess_cmd += " -U" if options.reverseStrand: - bfast_postprocess_cmd += ' -R' + bfast_postprocess_cmd += " -R" # instead of using temp files, should we stream through pipes? bfast_match_cmd += " > %s" % tmp_bmf_name @@ -254,13 +408,13 @@ def __main__(): # bfast 'match' try: tmp = tempfile.NamedTemporaryFile(dir=tmp_dir).name - tmp_stderr = open(tmp, 'wb') + tmp_stderr = open(tmp, "wb") proc = subprocess.Popen(args=bfast_match_cmd, shell=True, cwd=tmp_dir, stderr=tmp_stderr.fileno()) returncode = proc.wait() tmp_stderr.close() # get stderr, allowing for case where it's very large - tmp_stderr = open(tmp, 'rb') - stderr = '' + tmp_stderr = open(tmp, "rb") + stderr = "" try: while True: stderr += tmp_stderr.read(buffsize) @@ -272,17 +426,17 @@ def __main__(): if returncode != 0: raise Exception(stderr) except Exception as e: - raise Exception('Error in \'bfast match\'. \n' + str(e)) + raise Exception("Error in 'bfast match'. \n" + str(e)) # bfast 'localalign' try: tmp = tempfile.NamedTemporaryFile(dir=tmp_dir).name - tmp_stderr = open(tmp, 'wb') + tmp_stderr = open(tmp, "wb") proc = subprocess.Popen(args=bfast_localalign_cmd, shell=True, cwd=tmp_dir, stderr=tmp_stderr.fileno()) returncode = proc.wait() tmp_stderr.close() # get stderr, allowing for case where it's very large - tmp_stderr = open(tmp, 'rb') - stderr = '' + tmp_stderr = open(tmp, "rb") + stderr = "" try: while True: stderr += tmp_stderr.read(buffsize) @@ -294,17 +448,17 @@ def __main__(): if returncode != 0: raise Exception(stderr) except Exception as e: - raise Exception('Error in \'bfast localalign\'. \n' + str(e)) + raise Exception("Error in 'bfast localalign'. \n" + str(e)) # bfast 'postprocess' try: tmp = tempfile.NamedTemporaryFile(dir=tmp_dir).name - tmp_stderr = open(tmp, 'wb') + tmp_stderr = open(tmp, "wb") proc = subprocess.Popen(args=bfast_postprocess_cmd, shell=True, cwd=tmp_dir, stderr=tmp_stderr.fileno()) returncode = proc.wait() tmp_stderr.close() # get stderr, allowing for case where it's very large - tmp_stderr = open(tmp, 'rb') - stderr = '' + tmp_stderr = open(tmp, "rb") + stderr = "" try: while True: stderr += tmp_stderr.read(buffsize) @@ -316,7 +470,7 @@ def __main__(): if returncode != 0: raise Exception(stderr) except Exception as e: - raise Exception('Error in \'bfast postprocess\'. \n' + str(e)) + raise Exception("Error in 'bfast postprocess'. \n" + str(e)) # remove header if necessary if options.suppressHeader: tmp_out = tempfile.NamedTemporaryFile(dir=tmp_dir) @@ -325,22 +479,24 @@ def __main__(): try: shutil.move(options.output, tmp_out_name) except Exception as e: - raise Exception('Error moving output file before removing headers. \n' + str(e)) - fout = open(options.output, 'w') + raise Exception("Error moving output file before removing headers. \n" + str(e)) + fout = open(options.output, "w") for line in open(tmp_out.name): - if len(line) < 3 or line[0:3] not in ['@HD', '@SQ', '@RG', '@PG', '@CO']: + if len(line) < 3 or line[0:3] not in ["@HD", "@SQ", "@RG", "@PG", "@CO"]: fout.write(line) fout.close() # check that there are results in the output file if os.path.getsize(options.output) > 0: if "0" == options.space: - sys.stdout.write('BFAST run on Base Space data') + sys.stdout.write("BFAST run on Base Space data") else: - sys.stdout.write('BFAST run on Color Space data') + sys.stdout.write("BFAST run on Color Space data") else: - raise Exception('The output file is empty. You may simply have no matches, or there may be an error with your input file or settings.') + raise Exception( + "The output file is empty. You may simply have no matches, or there may be an error with your input file or settings." + ) except Exception as e: - stop_err('The alignment failed.\n' + str(e)) + stop_err("The alignment failed.\n" + str(e)) finally: # clean up temp dir if os.path.exists(tmp_dir): diff --git a/tools/sr_mapping/srma_wrapper.py b/tools/sr_mapping/srma_wrapper.py index ae3c9ca7f11..0d52547f5a1 100644 --- a/tools/sr_mapping/srma_wrapper.py +++ b/tools/sr_mapping/srma_wrapper.py @@ -17,14 +17,14 @@ import tempfile def stop_err(msg): - sys.stderr.write('%s\n' % msg) + sys.stderr.write("%s\n" % msg) sys.exit() def parseRefLoc(refLoc, refUID): for line in open(refLoc): - if not line.startswith('#'): - fields = line.strip().split('\t') + if not line.startswith("#"): + fields = line.strip().split("\t") if len(fields) >= 3: if fields[0] == refUID: return fields[1] @@ -33,23 +33,40 @@ def parseRefLoc(refLoc, refUID): def __main__(): parser = optparse.OptionParser() - parser.add_option('-r', '--ref', dest='ref', help='The reference genome to index and use') - parser.add_option('-u', '--refUID', dest='refUID', help='The pre-index reference genome unique Identifier') - parser.add_option('-i', '--input', dest='input', help='The SAM/BAM input file') - parser.add_option('-I', '--inputIndex', dest='inputIndex', help='The SAM/BAM input index file') - parser.add_option('-o', '--output', dest='output', help='The SAM/BAM output file') - parser.add_option('-O', '--offset', dest='offset', help='The alignment offset') - parser.add_option('-Q', '--minMappingQuality', dest='minMappingQuality', help='The minimum mapping quality') - parser.add_option('-P', '--minAlleleProbability', dest='minAlleleProbability', help='The minimum allele probability conditioned on coverage (for the binomial quantile).') - parser.add_option('-C', '--minAlleleCoverage', dest='minAlleleCoverage', help='The minimum haploid coverage for the consensus') - parser.add_option('-R', '--range', dest='range', help='A range to examine') - parser.add_option('-c', '--correctBases', dest='correctBases', help='Correct bases ') - parser.add_option('-q', '--useSequenceQualities', dest='useSequenceQualities', help='Use sequence qualities ') - parser.add_option('-M', '--maxHeapSize', dest='maxHeapSize', help='The maximum number of nodes on the heap before re-alignment is ignored') - parser.add_option('-s', '--fileSource', dest='fileSource', help='Whether to use a previously indexed reference sequence or one from history (indexed or history)') - parser.add_option('-p', '--params', dest='params', help='Parameter setting to use (pre_set or full)') - parser.add_option('-j', '--jarBin', dest='jarBin', default='', help='The path to where jars are stored') - parser.add_option('-f', '--jarFile', dest='jarFile', help='The file name of the jar file to use') + parser.add_option("-r", "--ref", dest="ref", help="The reference genome to index and use") + parser.add_option("-u", "--refUID", dest="refUID", help="The pre-index reference genome unique Identifier") + parser.add_option("-i", "--input", dest="input", help="The SAM/BAM input file") + parser.add_option("-I", "--inputIndex", dest="inputIndex", help="The SAM/BAM input index file") + parser.add_option("-o", "--output", dest="output", help="The SAM/BAM output file") + parser.add_option("-O", "--offset", dest="offset", help="The alignment offset") + parser.add_option("-Q", "--minMappingQuality", dest="minMappingQuality", help="The minimum mapping quality") + parser.add_option( + "-P", + "--minAlleleProbability", + dest="minAlleleProbability", + help="The minimum allele probability conditioned on coverage (for the binomial quantile).", + ) + parser.add_option( + "-C", "--minAlleleCoverage", dest="minAlleleCoverage", help="The minimum haploid coverage for the consensus" + ) + parser.add_option("-R", "--range", dest="range", help="A range to examine") + parser.add_option("-c", "--correctBases", dest="correctBases", help="Correct bases ") + parser.add_option("-q", "--useSequenceQualities", dest="useSequenceQualities", help="Use sequence qualities ") + parser.add_option( + "-M", + "--maxHeapSize", + dest="maxHeapSize", + help="The maximum number of nodes on the heap before re-alignment is ignored", + ) + parser.add_option( + "-s", + "--fileSource", + dest="fileSource", + help="Whether to use a previously indexed reference sequence or one from history (indexed or history)", + ) + parser.add_option("-p", "--params", dest="params", help="Parameter setting to use (pre_set or full)") + parser.add_option("-j", "--jarBin", dest="jarBin", default="", help="The path to where jars are stored") + parser.add_option("-f", "--jarFile", dest="jarFile", help="The file name of the jar file to use") (options, args) = parser.parse_args() # make temp directory for srma @@ -59,24 +76,24 @@ def __main__(): # set up reference filenames reference_filepath_name = None # need to create SRMA dict and Samtools fai files for custom genome - if options.fileSource == 'history': + if options.fileSource == "history": try: - reference_filepath = tempfile.NamedTemporaryFile(dir=tmp_dir, suffix='.fa') + reference_filepath = tempfile.NamedTemporaryFile(dir=tmp_dir, suffix=".fa") reference_filepath_name = reference_filepath.name reference_filepath.close() - dict_filepath_name = reference_filepath_name.replace('.fa', '.dict') + dict_filepath_name = reference_filepath_name.replace(".fa", ".dict") os.symlink(options.ref, reference_filepath_name) # create fai file using Samtools - index_fai_cmd = 'samtools faidx %s' % reference_filepath_name + index_fai_cmd = "samtools faidx %s" % reference_filepath_name try: tmp = tempfile.NamedTemporaryFile(dir=tmp_dir).name - tmp_stderr = open(tmp, 'wb') + tmp_stderr = open(tmp, "wb") proc = subprocess.Popen(args=index_fai_cmd, shell=True, cwd=tmp_dir, stderr=tmp_stderr.fileno()) returncode = proc.wait() tmp_stderr.close() # get stderr, allowing for case where it's very large - tmp_stderr = open(tmp, 'rb') - stderr = '' + tmp_stderr = open(tmp, "rb") + stderr = "" try: while True: stderr += tmp_stderr.read(buffsize) @@ -91,18 +108,22 @@ def __main__(): # clean up temp dir if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir) - stop_err('Error creating Samtools index for custom genome file: %s\n' % str(e)) + stop_err("Error creating Samtools index for custom genome file: %s\n" % str(e)) # create dict file using SRMA - dict_cmd = 'java -cp "%s" net.sf.picard.sam.CreateSequenceDictionary R=%s O=%s' % (os.path.join(options.jarBin, options.jarFile), reference_filepath_name, dict_filepath_name) + dict_cmd = 'java -cp "%s" net.sf.picard.sam.CreateSequenceDictionary R=%s O=%s' % ( + os.path.join(options.jarBin, options.jarFile), + reference_filepath_name, + dict_filepath_name, + ) try: tmp = tempfile.NamedTemporaryFile(dir=tmp_dir).name - tmp_stderr = open(tmp, 'wb') + tmp_stderr = open(tmp, "wb") proc = subprocess.Popen(args=dict_cmd, shell=True, cwd=tmp_dir, stderr=tmp_stderr.fileno()) returncode = proc.wait() tmp_stderr.close() # get stderr, allowing for case where it's very large - tmp_stderr = open(tmp, 'rb') - stderr = '' + tmp_stderr = open(tmp, "rb") + stderr = "" try: while True: stderr += tmp_stderr.read(buffsize) @@ -117,12 +138,12 @@ def __main__(): # clean up temp dir if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir) - stop_err('Error creating index for custom genome file: %s\n' % str(e)) + stop_err("Error creating index for custom genome file: %s\n" % str(e)) except Exception as e: # clean up temp dir if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir) - stop_err('Problem handling SRMA index (dict file) for custom genome file: %s\n' % str(e)) + stop_err("Problem handling SRMA index (dict file) for custom genome file: %s\n" % str(e)) # using built-in dict/index files else: if options.ref: @@ -130,18 +151,31 @@ def __main__(): else: reference_filepath_name = parseRefLoc(options.refLocation, options.refUID) if reference_filepath_name is None: - raise ValueError('A valid genome reference was not provided.') + raise ValueError("A valid genome reference was not provided.") # set up aligning and generate aligning command options - if options.params == 'pre_set': - srma_cmds = '' + if options.params == "pre_set": + srma_cmds = "" else: - ranges = 'null' - if options.range == 'None': - range = 'null' + ranges = "null" + if options.range == "None": + range = "null" else: range = options.range - srma_cmds = "OFFSET=%s MIN_MAPQ=%s MINIMUM_ALLELE_PROBABILITY=%s MINIMUM_ALLELE_COVERAGE=%s RANGES=%s RANGE=%s CORRECT_BASES=%s USE_SEQUENCE_QUALITIES=%s MAX_HEAP_SIZE=%s" % (options.offset, options.minMappingQuality, options.minAlleleProbability, options.minAlleleCoverage, ranges, range, options.correctBases, options.useSequenceQualities, options.maxHeapSize) + srma_cmds = ( + "OFFSET=%s MIN_MAPQ=%s MINIMUM_ALLELE_PROBABILITY=%s MINIMUM_ALLELE_COVERAGE=%s RANGES=%s RANGE=%s CORRECT_BASES=%s USE_SEQUENCE_QUALITIES=%s MAX_HEAP_SIZE=%s" + % ( + options.offset, + options.minMappingQuality, + options.minAlleleProbability, + options.minAlleleCoverage, + ranges, + range, + options.correctBases, + options.useSequenceQualities, + options.maxHeapSize, + ) + ) srma_cmds = "%s VALIDATION_STRINGENCY=LENIENT" % srma_cmds @@ -149,7 +183,7 @@ def __main__(): buffsize = 1048576 try: # symlink input bam and index files due to the naming conventions required by srma here - input_bam_filename = os.path.join(tmp_dir, '%s.bam' % os.path.split(options.input)[-1]) + input_bam_filename = os.path.join(tmp_dir, "%s.bam" % os.path.split(options.input)[-1]) os.symlink(options.input, input_bam_filename) input_bai_filename = "%s.bai" % os.path.splitext(input_bam_filename)[0] os.symlink(options.inputIndex, input_bai_filename) @@ -157,21 +191,28 @@ def __main__(): # create a temp output name, ending in .bam due to required naming conventions? unkown if required output_bam_filename = os.path.join(tmp_dir, "%s.bam" % os.path.split(options.output)[-1]) # generate commandline - java_opts = '' - if '_JAVA_OPTIONS' not in os.environ: - java_opts = '-Xmx2048m' - cmd = 'java %s -jar %s I=%s O=%s R=%s %s' % (java_opts, os.path.join(options.jarBin, options.jarFile), input_bam_filename, output_bam_filename, reference_filepath_name, srma_cmds) + java_opts = "" + if "_JAVA_OPTIONS" not in os.environ: + java_opts = "-Xmx2048m" + cmd = "java %s -jar %s I=%s O=%s R=%s %s" % ( + java_opts, + os.path.join(options.jarBin, options.jarFile), + input_bam_filename, + output_bam_filename, + reference_filepath_name, + srma_cmds, + ) # need to nest try-except in try-finally to handle 2.4 try: try: tmp = tempfile.NamedTemporaryFile(dir=tmp_dir).name - tmp_stderr = open(tmp, 'wb') + tmp_stderr = open(tmp, "wb") proc = subprocess.Popen(args=cmd, shell=True, cwd=tmp_dir, stderr=tmp_stderr.fileno()) returncode = proc.wait() tmp_stderr.close() # get stderr, allowing for case where it's very large - tmp_stderr = open(tmp, 'rb') - stderr = '' + tmp_stderr = open(tmp, "rb") + stderr = "" try: while True: stderr += tmp_stderr.read(buffsize) @@ -183,14 +224,16 @@ def __main__(): if returncode != 0: raise Exception(stderr) except Exception as e: - raise Exception('Error executing SRMA. ' + str(e)) + raise Exception("Error executing SRMA. " + str(e)) # move file from temp location (with .bam name) to provided path shutil.move(output_bam_filename, options.output) # check that there are results in the output file if os.path.getsize(options.output) <= 0: - raise Exception('The output file is empty. You may simply have no matches, or there may be an error with your input file or settings.') + raise Exception( + "The output file is empty. You may simply have no matches, or there may be an error with your input file or settings." + ) except Exception as e: - stop_err('The re-alignment failed.\n' + str(e)) + stop_err("The re-alignment failed.\n" + str(e)) finally: # clean up temp dir if os.path.exists(tmp_dir): diff --git a/tools/stats/aggregate_scores_in_intervals.py b/tools/stats/aggregate_scores_in_intervals.py index bca1a79c7eb..3b6d3c2529f 100755 --- a/tools/stats/aggregate_scores_in_intervals.py +++ b/tools/stats/aggregate_scores_in_intervals.py @@ -35,12 +35,12 @@ from galaxy.util.ucsc import ( class PositionalScoresOnDisk(object): - fmt = 'f' + fmt = "f" fmt_size = struct.calcsize(fmt) - default_value = float('nan') + default_value = float("nan") def __init__(self): - self.file = tempfile.TemporaryFile('w+b') + self.file = tempfile.TemporaryFile("w+b") self.length = 0 def __getitem__(self, i): @@ -58,7 +58,7 @@ class PositionalScoresOnDisk(object): if i < 0: i = self.length + i if i < 0: - raise IndexError('Negative assignment index out of range') + raise IndexError("Negative assignment index out of range") if i >= self.length: self.file.seek(self.length * self.fmt_size) self.file.write(struct.pack(self.fmt, self.default_value) * (i - self.length)) @@ -82,6 +82,7 @@ class FileBinnedArrayDir(Mapping): Adapter that makes a directory of FileBinnedArray files look like a regular dict of BinnedArray objects. """ + def __init__(self, dir): self.dir = dir self.cache = dict() @@ -128,7 +129,9 @@ def load_scores_wiggle(fname, chrom_buffer_size=3): scores_by_chrom[chrom][pos] = val except UCSCLimitException: # Wiggle data was truncated, at the very least need to warn the user. - print('Encountered message from UCSC: "Reached output limit of 100000 data values", so be aware your data was truncated.') + print( + 'Encountered message from UCSC: "Reached output limit of 100000 data values", so be aware your data was truncated.' + ) except IndexError: stop_err('Data error: one or more column data values is missing in "%s"' % fname) except ValueError: @@ -156,7 +159,7 @@ def main(): start_col = args[3] stop_col = args[4] if len(args) > 5: - out_file = open(args[5], 'w') + out_file = open(args[5], "w") else: out_file = sys.stdout binned = bool(options.binned) @@ -164,18 +167,24 @@ def main(): except Exception: doc_optparse.exit() - if score_fname == 'None': - stop_err('This tool works with data from genome builds hg16, hg17 or hg18. Click the pencil icon in your history item to set the genome build if appropriate.') + if score_fname == "None": + stop_err( + "This tool works with data from genome builds hg16, hg17 or hg18. Click the pencil icon in your history item to set the genome build if appropriate." + ) try: chrom_col = int(chrom_col) - 1 start_col = int(start_col) - 1 stop_col = int(stop_col) - 1 except Exception: - stop_err('Chrom, start & end column not properly set, click the pencil icon in your history item to set these values.') + stop_err( + "Chrom, start & end column not properly set, click the pencil icon in your history item to set these values." + ) if chrom_col < 0 or start_col < 0 or stop_col < 0: - stop_err('Chrom, start & end column not properly set, click the pencil icon in your history item to set these values.') + stop_err( + "Chrom, start & end column not properly set, click the pencil icon in your history item to set these values." + ) if binned: scores_by_chrom = load_scores_ba_dir(score_fname) @@ -193,12 +202,12 @@ def main(): skipped_lines = 0 first_invalid_line = 0 - invalid_line = '' + invalid_line = "" for i, line in enumerate(open(interval_fname)): valid = True - line = line.rstrip('\r\n') - if line and not line.startswith('#'): + line = line.rstrip("\r\n") + if line and not line.startswith("#"): fields = line.split() try: @@ -251,16 +260,21 @@ def main(): if not invalid_line: first_invalid_line = i + 1 invalid_line = line - elif line.startswith('#'): + elif line.startswith("#"): # We'll save the original comments print(line, file=out_file) out_file.close() if skipped_lines > 0: - print('Data issue: skipped %d invalid lines starting at line #%d which is "%s"' % (skipped_lines, first_invalid_line, invalid_line)) + print( + 'Data issue: skipped %d invalid lines starting at line #%d which is "%s"' + % (skipped_lines, first_invalid_line, invalid_line) + ) if skipped_lines == i: - print('Consider changing the metadata for the input dataset by clicking on the pencil icon in the history item.') + print( + "Consider changing the metadata for the input dataset by clicking on the pencil icon in the history item." + ) if __name__ == "__main__": diff --git a/tools/stats/filtering.py b/tools/stats/filtering.py index 4bcae7c9b99..840070a1da4 100644 --- a/tools/stats/filtering.py +++ b/tools/stats/filtering.py @@ -16,24 +16,62 @@ from ast import ( ) AST_NODE_TYPE_WHITELIST = [ - 'Expr', 'Load', 'Str', 'Num', 'BoolOp', 'Compare', 'And', 'Eq', 'NotEq', - 'Or', 'GtE', 'LtE', 'Lt', 'Gt', 'BinOp', 'Add', 'Div', 'Sub', 'Mult', 'Mod', - 'Pow', 'LShift', 'GShift', 'BitAnd', 'BitOr', 'BitXor', 'UnaryOp', 'Invert', - 'Not', 'UAdd', 'USub', 'NotIn', 'In', 'Is', 'IsNot', 'List', 'Index', - 'Subscript', 'Constant', + "Expr", + "Load", + "Str", + "Num", + "BoolOp", + "Compare", + "And", + "Eq", + "NotEq", + "Or", + "GtE", + "LtE", + "Lt", + "Gt", + "BinOp", + "Add", + "Div", + "Sub", + "Mult", + "Mod", + "Pow", + "LShift", + "GShift", + "BitAnd", + "BitOr", + "BitXor", + "UnaryOp", + "Invert", + "Not", + "UAdd", + "USub", + "NotIn", + "In", + "Is", + "IsNot", + "List", + "Index", + "Subscript", + "Constant", # Further checks - 'Name', 'Call', 'Attribute', + "Name", + "Call", + "Attribute", ] -BUILTIN_AND_MATH_FUNCTIONS = 'abs|all|any|bin|chr|cmp|complex|divmod|float|hex|int|len|long|max|min|oct|ord|pow|range|reversed|round|sorted|str|sum|type|unichr|unicode|log|exp|sqrt|ceil|floor'.split('|') -STRING_AND_LIST_METHODS = [name for name in dir('') + dir([]) if not name.startswith('_')] +BUILTIN_AND_MATH_FUNCTIONS = "abs|all|any|bin|chr|cmp|complex|divmod|float|hex|int|len|long|max|min|oct|ord|pow|range|reversed|round|sorted|str|sum|type|unichr|unicode|log|exp|sqrt|ceil|floor".split( + "|" +) +STRING_AND_LIST_METHODS = [name for name in dir("") + dir([]) if not name.startswith("_")] VALID_FUNCTIONS = BUILTIN_AND_MATH_FUNCTIONS + STRING_AND_LIST_METHODS def __check_name(ast_node): name = ast_node.id - if re.match(r'^c\d+$', name): + if re.match(r"^c\d+$", name): return True return name in VALID_FUNCTIONS @@ -50,10 +88,10 @@ def __check_call(ast_node): # string or list function. ast_func = ast_node.func ast_func_class = ast_func.__class__.__name__ - if ast_func_class == 'Name': + if ast_func_class == "Name": if ast_func.id not in BUILTIN_AND_MATH_FUNCTIONS: return False - elif ast_func_class == 'Attribute': + elif ast_func_class == "Attribute": if not __check_attribute(ast_func): return False else: @@ -107,7 +145,7 @@ def check_expression(text): if not len(statements) == 1: return False expression = statements[0] - if expression.__class__.__name__ != 'Expr': + if expression.__class__.__name__ != "Expr": return False for ast_node in walk(expression): @@ -119,17 +157,17 @@ def check_expression(text): return False # White-list more potentially dangerous types AST elements. - if ast_node_class == 'Name': + if ast_node_class == "Name": # In order to prevent loading 'exec', 'eval', etc... # put string restriction on names allowed. if not __check_name(ast_node): return False # Check only valid, white-listed functions are called. - elif ast_node_class == 'Call': + elif ast_node_class == "Call": if not __check_call(ast_node): return False # Check only valid, white-listed attributes are accessed - elif ast_node_class == 'Attribute': + elif ast_node_class == "Attribute": if not __check_attribute(ast_node): return False @@ -138,11 +176,39 @@ def check_expression(text): def get_operands(filter_condition): # Note that the order of all_operators is important - items_to_strip = ['+', '-', '**', '*', '//', '/', '%', '<<', '>>', '&', '|', '^', '~', '<=', '<', '>=', '>', '==', '!=', '<>', ' and ', ' or ', ' not ', ' is ', ' is not ', ' in ', ' not in '] + items_to_strip = [ + "+", + "-", + "**", + "*", + "//", + "/", + "%", + "<<", + ">>", + "&", + "|", + "^", + "~", + "<=", + "<", + ">=", + ">", + "==", + "!=", + "<>", + " and ", + " or ", + " not ", + " is ", + " is not ", + " in ", + " not in ", + ] for item in items_to_strip: if filter_condition.find(item) >= 0: - filter_condition = filter_condition.replace(item, ' ') - operands = set(filter_condition.split(' ')) + filter_condition = filter_condition.replace(item, " ") + operands = set(filter_condition.split(" ")) return operands @@ -159,23 +225,23 @@ cond_text = inputs["cond"] try: in_columns = int(sys.argv[4]) assert sys.argv[5] # check to see that the column types variable isn't null - in_column_types = sys.argv[5].split(',') + in_column_types = sys.argv[5].split(",") except Exception: stop_err("Data does not appear to be tabular. This tool can only be used with tab-delimited data.") num_header_lines = int(sys.argv[6]) # Unescape if input has been escaped mapped_str = { - '__lt__': '<', - '__le__': '<=', - '__eq__': '==', - '__ne__': '!=', - '__gt__': '>', - '__ge__': '>=', - '__sq__': '\'', - '__dq__': '"', - '__ob__': '[', - '__cb__': ']', + "__lt__": "<", + "__le__": "<=", + "__eq__": "==", + "__ne__": "!=", + "__gt__": ">", + "__ge__": ">=", + "__sq__": "'", + "__dq__": '"', + "__ob__": "[", + "__cb__": "]", } for key, value in mapped_str.items(): cond_text = cond_text.replace(key, value) @@ -194,8 +260,7 @@ if not check_expression(cond_text): stop_err("Illegal/invalid in condition '%s'" % (cond_text)) # Work out which columns are used in the filter (save using 1 based counting) -used_cols = sorted({int(match.group()[1:]) - for match in re.finditer(r'c(\d)+', cond_text)}) +used_cols = sorted({int(match.group()[1:]) for match in re.finditer(r"c(\d)+", cond_text)}) largest_col_index = max(used_cols) # Prepare the column variable names and wrappers for column data types. Only @@ -213,8 +278,8 @@ for col in range(1, largest_col_index + 1): type_cast = col_name type_casts.append(type_cast) -col_str = ', '.join(cols) # 'c1, c2, c3, c4' -type_cast_str = ', '.join(type_casts) # 'str(c1), int(c2), int(c3), str(c4)' +col_str = ", ".join(cols) # 'c1, c2, c3, c4' +type_cast_str = ", ".join(type_casts) # 'str(c1), int(c2), int(c3), str(c4)' assign = "%s, = line.split( '\\t' )[:%i]" % (col_str, largest_col_index) wrap = "%s = %s" % (col_str, type_cast_str) skipped_lines = 0 @@ -223,10 +288,10 @@ first_invalid_line = 0 invalid_line = None lines_kept = 0 total_lines = 0 -out = open(out_fname, 'wt') +out = open(out_fname, "wt") # Read and filter input file, skipping invalid lines -code = ''' +code = """ for i, line in enumerate( open( in_fname ) ): total_lines += 1 line = line.rstrip( '\\r\\n' ) @@ -250,13 +315,17 @@ for i, line in enumerate( open( in_fname ) ): if not invalid_line: first_invalid_line = i + 1 invalid_line = line -''' % (assign, wrap, cond_text) +""" % ( + assign, + wrap, + cond_text, +) valid_filter = True try: exec(code) except Exception as e: out.close() - if str(e).startswith('invalid syntax'): + if str(e).startswith("invalid syntax"): valid_filter = False stop_err('Filter condition "%s" likely invalid. See tool tips, syntax and examples.' % cond_text) else: @@ -265,12 +334,20 @@ except Exception as e: if valid_filter: out.close() valid_lines = total_lines - skipped_lines - print('Filtering with %s, ' % cond_text) + print("Filtering with %s, " % cond_text) if valid_lines > 0: - print('kept %4.2f%% of %d valid lines (%d total lines).' % (100.0 * lines_kept / valid_lines, valid_lines, total_lines)) + print( + "kept %4.2f%% of %d valid lines (%d total lines)." + % (100.0 * lines_kept / valid_lines, valid_lines, total_lines) + ) else: - print('Possible invalid filter condition "%s" or non-existent column referenced. See tool tips, syntax and examples.' % cond_text) + print( + 'Possible invalid filter condition "%s" or non-existent column referenced. See tool tips, syntax and examples.' + % cond_text + ) if invalid_lines: - print('Skipped %d invalid line(s) starting at line #%d: "%s"' % (invalid_lines, first_invalid_line, invalid_line)) + print( + 'Skipped %d invalid line(s) starting at line #%d: "%s"' % (invalid_lines, first_invalid_line, invalid_line) + ) if skipped_lines: - print('Skipped %i comment (starting with #) or blank line(s)' % skipped_lines) + print("Skipped %i comment (starting with #) or blank line(s)" % skipped_lines) diff --git a/tools/stats/grouping.py b/tools/stats/grouping.py index 95c8816626f..86ca7bff1e1 100644 --- a/tools/stats/grouping.py +++ b/tools/stats/grouping.py @@ -52,7 +52,7 @@ def mode(data): for x in counts: if counts[x] == maxcount: modelist.append(str(x)) - return ','.join(modelist) + return ",".join(modelist) def main(): @@ -68,8 +68,8 @@ def main(): asciitodelete = sys.argv[5] if asciitodelete: newinputfile = "input_cleaned.tsv" - with open(inputfile) as oldfile, open(newinputfile, 'w') as newfile: - asciitodelete = {chr(int(_)) for _ in asciitodelete.split(',')} + with open(inputfile) as oldfile, open(newinputfile, "w") as newfile: + asciitodelete = {chr(int(_)) for _ in asciitodelete.split(",")} for line in oldfile: if line[0] not in asciitodelete: newfile.write(line) @@ -77,11 +77,11 @@ def main(): # get operations and options in separate arrays for var in sys.argv[6:]: - op, col, do_round, default = var.split(',') + op, col, do_round, default = var.split(",") ops.append(op) cols.append(col) round_val.append(do_round) - default_val.append(float(default) if default != '' else None) + default_val.append(float(default) if default != "" else None) # At this point, ops, cols and rounds will look something like this: # ops: ['mean', 'min', 'c'] @@ -95,7 +95,7 @@ def main(): stop_err("Group column not specified.") # sort file into a temporary file - tmpfile = tempfile.NamedTemporaryFile(mode='r') + tmpfile = tempfile.NamedTemporaryFile(mode="r") try: """ The -k option for the Posix sort command is as follows: @@ -108,9 +108,9 @@ def main(): group_col_str = str(group_col + 1) command_line = ["sort", "-t", "\t", "-k%s,%s" % (group_col_str, group_col_str), "-o", tmpfile.name, inputfile] if ignorecase == 1: - command_line.append('-f') + command_line.append("-f") except Exception as exc: - stop_err('Initialization error -> %s' % str(exc)) + stop_err("Initialization error -> %s" % str(exc)) try: subprocess.check_output(command_line, stderr=subprocess.STDOUT) @@ -140,7 +140,10 @@ def main(): val = fields[col].strip() op_vals[i].append(val) except IndexError: - sys.stderr.write('Could not access the value for column %s on line: "%s". Make sure file is tab-delimited.\n' % (col + 1, line)) + sys.stderr.write( + 'Could not access the value for column %s on line: "%s". Make sure file is tab-delimited.\n' + % (col + 1, line) + ) sys.exit(1) # Generate string for each op for this group @@ -153,10 +156,10 @@ def main(): rval = len(data) elif op == "random": rval = random.choice(data) - elif op in ['cat', 'cat_uniq']: - if op == 'cat_uniq': + elif op in ["cat", "cat_uniq"]: + if op == "cat_uniq": data = numpy.unique(data) - rval = ','.join(data) + rval = ",".join(data) elif op == "unique": rval = len(numpy.unique(data)) else: @@ -167,10 +170,10 @@ def main(): sys.stderr.write("Operation %s expected number values but got %s instead.\n" % (op, data)) sys.exit(1) rval = getattr(numpy, op)(data) - if round_val[i] == 'yes': + if round_val[i] == "yes": rval = int(round(rval)) else: - rval = '%g' % rval + rval = "%g" % rval out_str += "\t%s" % rval fout.write(out_str + "\n") @@ -180,16 +183,16 @@ def main(): # Generate a useful info message. msg = "--Group by c%d: " % (group_col + 1) for i, op in enumerate(ops): - if op == 'cat': - op = 'concat' - elif op == 'cat_uniq': - op = 'concat_distinct' - elif op == 'length': - op = 'count' - elif op == 'unique': - op = 'count_distinct' - elif op == 'random': - op = 'randomly_pick' + if op == "cat": + op = "concat" + elif op == "cat_uniq": + op = "concat_distinct" + elif op == "length": + op = "count" + elif op == "unique": + op = "count_distinct" + elif op == "random": + op = "randomly_pick" msg += op + "[c" + cols[i] + "] " diff --git a/tools/stats/gsummary.py b/tools/stats/gsummary.py index 077d9dd1671..868b5d49cb2 100755 --- a/tools/stats/gsummary.py +++ b/tools/stats/gsummary.py @@ -30,13 +30,64 @@ def stop_err(msg): def S3_METHODS(all="key"): - Group_Math = ["abs", "sign", "sqrt", "floor", "ceiling", "trunc", "round", "signif", - "exp", "log", "cos", "sin", "tan", "acos", "asin", "atan", "cosh", "sinh", "tanh", - "acosh", "asinh", "atanh", "lgamma", "gamma", "gammaCody", "digamma", "trigamma", - "cumsum", "cumprod", "cummax", "cummin", "c"] - Group_Ops = ["+", "-", "*", "/", "^", "%%", "%/%", "&", "|", "!", "==", "!=", "<", "<=", ">=", ">", "(", ")", "~", ","] + Group_Math = [ + "abs", + "sign", + "sqrt", + "floor", + "ceiling", + "trunc", + "round", + "signif", + "exp", + "log", + "cos", + "sin", + "tan", + "acos", + "asin", + "atan", + "cosh", + "sinh", + "tanh", + "acosh", + "asinh", + "atanh", + "lgamma", + "gamma", + "gammaCody", + "digamma", + "trigamma", + "cumsum", + "cumprod", + "cummax", + "cummin", + "c", + ] + Group_Ops = [ + "+", + "-", + "*", + "/", + "^", + "%%", + "%/%", + "&", + "|", + "!", + "==", + "!=", + "<", + "<=", + ">=", + ">", + "(", + ")", + "~", + ",", + ] if all == "key": - return {'Math': Group_Math, 'Ops': Group_Ops} + return {"Math": Group_Math, "Ops": Group_Ops} def main(): @@ -45,34 +96,34 @@ def main(): outfile_name = sys.argv[2] expression = sys.argv[3] except Exception: - stop_err('Usage: python gsummary.py input_file ouput_file expression') + stop_err("Usage: python gsummary.py input_file ouput_file expression") - math_allowed = S3_METHODS()['Math'] - ops_allowed = S3_METHODS()['Ops'] + math_allowed = S3_METHODS()["Math"] + ops_allowed = S3_METHODS()["Ops"] # Check for invalid expressions - for word in re.compile('[a-zA-Z]+').findall(expression): + for word in re.compile("[a-zA-Z]+").findall(expression): if word and word not in math_allowed: stop_err("Invalid expression '%s': term '%s' is not recognized or allowed" % (expression, word)) symbols = set() - for symbol in re.compile(r'[^a-z0-9\s]+').findall(expression): + for symbol in re.compile(r"[^a-z0-9\s]+").findall(expression): if symbol and symbol not in ops_allowed: stop_err("Invalid expression '%s': operator '%s' is not recognized or allowed" % (expression, symbol)) else: symbols.add(symbol) - if len(symbols) == 1 and ',' in symbols: + if len(symbols) == 1 and "," in symbols: # User may have entered a comma-separated list r_data_frame columns stop_err("Invalid columns '%s': this tool requires a single column or expression" % expression) # Find all column references in the expression cols = [] - for col in re.compile('c[0-9]+').findall(expression): + for col in re.compile("c[0-9]+").findall(expression): try: cols.append(int(col[1:]) - 1) except Exception: pass - tmp_file = tempfile.NamedTemporaryFile('w+') + tmp_file = tempfile.NamedTemporaryFile("w+") # Write the R header row to the temporary file hdr_str = "\t".join("c%s" % str(col + 1) for col in cols) tmp_file.write("%s\n" % hdr_str) @@ -80,10 +131,10 @@ def main(): first_invalid_line = 0 i = 0 for i, line in enumerate(open(datafile)): - line = line.rstrip('\r\n') - if line and not line.startswith('#'): + line = line.rstrip("\r\n") + if line and not line.startswith("#"): valid = True - fields = line.split('\t') + fields = line.split("\t") # Write the R data row to the temporary file for col in cols: try: @@ -100,19 +151,23 @@ def main(): tmp_file.flush() if skipped_lines == i + 1: - stop_err("Invalid column or column data values invalid for computation. See tool tips and syntax for data requirements.") + stop_err( + "Invalid column or column data values invalid for computation. See tool tips and syntax for data requirements." + ) else: # summary function and return labels set_default_mode(NO_CONVERSION) - summary_func = r("function( x ) { c( sum=sum( as.numeric( x ), na.rm=T ), mean=mean( as.numeric( x ), na.rm=T ), stdev=sd( as.numeric( x ), na.rm=T ), quantile( as.numeric( x ), na.rm=TRUE ) ) }") - headings = ['sum', 'mean', 'stdev', '0%', '25%', '50%', '75%', '100%'] + summary_func = r( + "function( x ) { c( sum=sum( as.numeric( x ), na.rm=T ), mean=mean( as.numeric( x ), na.rm=T ), stdev=sd( as.numeric( x ), na.rm=T ), quantile( as.numeric( x ), na.rm=TRUE ) ) }" + ) + headings = ["sum", "mean", "stdev", "0%", "25%", "50%", "75%", "100%"] headings_str = "\t".join(headings) r_data_frame = r.read_table(tmp_file.name, header=True, sep="\t") - outfile = open(outfile_name, 'w') + outfile = open(outfile_name, "w") - for col in re.compile('c[0-9]+').findall(expression): + for col in re.compile("c[0-9]+").findall(expression): r.assign(col, r["$"](r_data_frame, col)) try: summary = summary_func(r(expression)) @@ -130,7 +185,10 @@ def main(): outfile.close() if skipped_lines: - print("Skipped %d invalid lines beginning with line #%d. See tool tips for data requirements." % (skipped_lines, first_invalid_line)) + print( + "Skipped %d invalid lines beginning with line #%d. See tool tips for data requirements." + % (skipped_lines, first_invalid_line) + ) if __name__ == "__main__": diff --git a/tools/visualization/LAJ_code.py b/tools/visualization/LAJ_code.py index 7633c121774..64d69e73552 100644 --- a/tools/visualization/LAJ_code.py +++ b/tools/visualization/LAJ_code.py @@ -12,7 +12,16 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr "alignfile1": "display?id=%s" % primary_data.id, "buttonlabel": "Launch LAJ", "title": "LAJ in Galaxy", - "posturl": "history_add_to?%s" % urlencode({'history_id': primary_data.history_id, 'ext': 'lav', 'name': 'LAJ Output', 'info': 'Added by LAJ', 'dbkey': primary_data.dbkey}) + "posturl": "history_add_to?%s" + % urlencode( + { + "history_id": primary_data.history_id, + "ext": "lav", + "name": "LAJ Output", + "info": "Added by LAJ", + "dbkey": primary_data.dbkey, + } + ), } for name, data in inp_data.items(): if name == "maf_input":