opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-21 05:45:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

Apply black to tools as well

Browse Source
This commit is contained in:
Dannon Baker
2022-02-03 09:12:13 -05:00
parent f74ae05cf1
commit 03351d2a9f
81 changed files with 3119 additions and 1798 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/data_source/data_source.py
+45 -31
View File
@@ -21,9 +21,9 @@ from galaxy.util import (
get_charset_from_http_headers,
)
GALAXY_PARAM_PREFIX = 'GALAXY'
GALAXY_PARAM_PREFIX = "GALAXY"
GALAXY_ROOT_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
GALAXY_DATATYPES_CONF_FILE = os.path.join(GALAXY_ROOT_DIR, 'datatypes_conf.xml')
GALAXY_DATATYPES_CONF_FILE = os.path.join(GALAXY_ROOT_DIR, "datatypes_conf.xml")
def stop_err(msg):
@@ -35,18 +35,18 @@ def load_input_parameters(filename, erase_file=True):
datasource_params = {}
try:
json_params = loads(open(filename).read())
datasource_params = json_params.get('param_dict')
datasource_params = json_params.get("param_dict")
except Exception:
json_params = None
for line in open(filename):
try:
line = line.strip()
fields = line.split('\t')
fields = line.split("\t")
datasource_params[fields[0]] = fields[1]
except Exception:
continue
if erase_file:
open(filename, 'w').close() # open file for writing, then close, removes params from file
open(filename, "w").close() # open file for writing, then close, removes params from file
return json_params, datasource_params
@@ -61,54 +61,68 @@ def __main__():
if job_params is None: # using an older tabular file
enhanced_handling = False
job_params = dict(param_dict=params)
job_params['output_data'] = [dict(out_data_name='output',
ext='data',
file_name=filename,
extra_files_path=None)]
job_params['job_config'] = dict(GALAXY_ROOT_DIR=GALAXY_ROOT_DIR, GALAXY_DATATYPES_CONF_FILE=GALAXY_DATATYPES_CONF_FILE, TOOL_PROVIDED_JOB_METADATA_FILE=TOOL_PROVIDED_JOB_METADATA_FILE)
job_params["output_data"] = [
dict(out_data_name="output", ext="data", file_name=filename, extra_files_path=None)
]
job_params["job_config"] = dict(
GALAXY_ROOT_DIR=GALAXY_ROOT_DIR,
GALAXY_DATATYPES_CONF_FILE=GALAXY_DATATYPES_CONF_FILE,
TOOL_PROVIDED_JOB_METADATA_FILE=TOOL_PROVIDED_JOB_METADATA_FILE,
)
else:
enhanced_handling = True
json_file = open(job_params['job_config']['TOOL_PROVIDED_JOB_METADATA_FILE'], 'w') # specially named file for output junk to pass onto set metadata
json_file = open(
job_params["job_config"]["TOOL_PROVIDED_JOB_METADATA_FILE"], "w"
) # specially named file for output junk to pass onto set metadata
datatypes_registry = Registry()
datatypes_registry.load_datatypes(root_dir=job_params['job_config']['GALAXY_ROOT_DIR'], config=job_params['job_config']['GALAXY_DATATYPES_CONF_FILE'])
datatypes_registry.load_datatypes(
root_dir=job_params["job_config"]["GALAXY_ROOT_DIR"],
config=job_params["job_config"]["GALAXY_DATATYPES_CONF_FILE"],
)
URL = params.get('URL', None) # using exactly URL indicates that only one dataset is being downloaded
URL_method = params.get('URL_method', None)
URL = params.get("URL", None) # using exactly URL indicates that only one dataset is being downloaded
URL_method = params.get("URL_method", None)
for data_dict in job_params['output_data']:
cur_filename = data_dict.get('file_name', filename)
cur_URL = params.get('%s|%s|URL' % (GALAXY_PARAM_PREFIX, data_dict['out_data_name']), URL)
if not cur_URL or urlparse(cur_URL).scheme not in ('http', 'https', 'ftp'):
open(cur_filename, 'w').write("")
stop_err('The remote data source application has not sent back a URL parameter in the request.')
for data_dict in job_params["output_data"]:
cur_filename = data_dict.get("file_name", filename)
cur_URL = params.get("%s|%s|URL" % (GALAXY_PARAM_PREFIX, data_dict["out_data_name"]), URL)
if not cur_URL or urlparse(cur_URL).scheme not in ("http", "https", "ftp"):
open(cur_filename, "w").write("")
stop_err("The remote data source application has not sent back a URL parameter in the request.")
# The following calls to urlopen() will use the above default timeout
try:
if not URL_method or URL_method == 'get':
if not URL_method or URL_method == "get":
page = urlopen(cur_URL, timeout=DEFAULT_SOCKET_TIMEOUT)
elif URL_method == 'post':
elif URL_method == "post":
page = urlopen(cur_URL, urlencode(params).encode("utf-8"), timeout=DEFAULT_SOCKET_TIMEOUT)
except Exception as e:
stop_err('The remote data source application may be off line, please try again later. Error: %s' % str(e))
stop_err("The remote data source application may be off line, please try again later. Error: %s" % str(e))
if max_file_size:
file_size = int(page.info().get('Content-Length', 0))
file_size = int(page.info().get("Content-Length", 0))
if file_size > max_file_size:
stop_err('The size of the data (%d bytes) you have requested exceeds the maximum allowed (%d bytes) on this server.' % (file_size, max_file_size))
stop_err(
"The size of the data (%d bytes) you have requested exceeds the maximum allowed (%d bytes) on this server."
% (file_size, max_file_size)
)
try:
cur_filename = sniff.stream_to_open_named_file(page, os.open(cur_filename, os.O_WRONLY | os.O_CREAT), cur_filename, source_encoding=get_charset_from_http_headers(page.headers))
cur_filename = sniff.stream_to_open_named_file(
page,
os.open(cur_filename, os.O_WRONLY | os.O_CREAT),
cur_filename,
source_encoding=get_charset_from_http_headers(page.headers),
)
except Exception as e:
stop_err('Unable to fetch %s:\n%s' % (cur_URL, e))
stop_err("Unable to fetch %s:\n%s" % (cur_URL, e))
# here import checks that upload tool performs
if enhanced_handling:
try:
ext = sniff.handle_uploaded_dataset_file(filename, datatypes_registry, ext=data_dict['ext'])
ext = sniff.handle_uploaded_dataset_file(filename, datatypes_registry, ext=data_dict["ext"])
except Exception as e:
stop_err(str(e))
info = dict(type='dataset',
dataset_id=data_dict['dataset_id'],
ext=ext)
info = dict(type="dataset", dataset_id=data_dict["dataset_id"], ext=ext)
json_file.write("%s\n" % dumps(info))
tools/data_source/genbank.py
+11 -11
View File
@@ -10,21 +10,21 @@ assert sys.version_info[:2] >= (2, 6)
def make_fasta(rec):
'''Creates fasta format from a record'''
gi = rec.annotations.get('gi', '')
org = rec.annotations.get('organism', '')
date = rec.annotations.get('date', '')
head = '>gi:%s, id:%s, org:%s, date:%s\n' % (gi, rec.id, org, date)
body = '\n'.join(textwrap.wrap(rec.seq.data, width=80))
"""Creates fasta format from a record"""
gi = rec.annotations.get("gi", "")
org = rec.annotations.get("organism", "")
date = rec.annotations.get("date", "")
head = ">gi:%s, id:%s, org:%s, date:%s\n" % (gi, rec.id, org, date)
body = "\n".join(textwrap.wrap(rec.seq.data, width=80))
return head, body
if __name__ == '__main__':
if __name__ == "__main__":
mode = sys.argv[1]
text = sys.argv[2]
output_file = sys.argv[3]
print('Searching for %s <br>' % text)
print("Searching for %s <br>" % text)
# check if inputs are all numbers
try:
@@ -33,12 +33,12 @@ if __name__ == '__main__':
except ValueError:
gi_list = GenBank.search_for(text, max_ids=10)
fp = open(output_file, 'wt')
fp = open(output_file, "wt")
record_parser = GenBank.FeatureParser()
ncbi_dict = GenBank.NCBIDictionary(mode, 'genbank', parser=record_parser)
ncbi_dict = GenBank.NCBIDictionary(mode, "genbank", parser=record_parser)
for gid in gi_list:
res = ncbi_dict[gid]
head, body = make_fasta(res)
fp.write(head + body + '\n')
fp.write(head + body + "\n")
print(head)
fp.close()
tools/data_source/hbvar_filter.py
+15 -15
View File
@@ -10,10 +10,10 @@ from galaxy.util import DEFAULT_SOCKET_TIMEOUT
def exec_before_job(app, inp_data, out_data, param_dict, tool=None):
"""Sets the name of the data"""
data_name = param_dict.get('name', 'HbVar query')
data_type = param_dict.get('type', 'txt')
if data_type == 'txt':
data_type = 'interval' # All data is TSV, assume interval
data_name = param_dict.get("name", "HbVar query")
data_type = param_dict.get("type", "txt")
if data_type == "txt":
data_type = "interval" # All data is TSV, assume interval
name, data = next(iter(out_data.items()))
data = app.datatypes_registry.change_datatype(data, data_type)
data.name = data_name
@@ -23,10 +23,10 @@ def exec_before_job(app, inp_data, out_data, param_dict, tool=None):
def exec_after_process(app, inp_data, out_data, param_dict, tool=None, stdout=None, stderr=None):
"""Verifies the data after the run"""
URL = param_dict.get('URL', None)
URL = URL + '&_export=1&GALAXY_URL=0'
URL = param_dict.get("URL", None)
URL = URL + "&_export=1&GALAXY_URL=0"
if not URL:
raise Exception('Datasource has not sent back a URL parameter')
raise Exception("Datasource has not sent back a URL parameter")
CHUNK_SIZE = 2**20 # 1Mb
MAX_SIZE = CHUNK_SIZE * 100
@@ -34,18 +34,18 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool=None, stdout=No
try:
page = urlopen(URL, timeout=DEFAULT_SOCKET_TIMEOUT)
except Exception as exc:
raise Exception('Problems connecting to %s (%s)' % (URL, exc))
raise Exception("Problems connecting to %s (%s)" % (URL, exc))
data = next(iter(out_data.values()))
fp = open(data.file_name, 'wb')
fp = open(data.file_name, "wb")
size = 0
while 1:
chunk = page.read(CHUNK_SIZE)
if not chunk:
break
if size > MAX_SIZE:
raise Exception('----- maximum datasize exceeded ---')
raise Exception("----- maximum datasize exceeded ---")
size += len(chunk)
fp.write(chunk)
@@ -56,10 +56,10 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool=None, stdout=No
# check for missing meta data, if all there, comment first line and process file
if not data.missing_meta():
line_ctr = -1
temp = tempfile.NamedTemporaryFile('w')
temp = tempfile.NamedTemporaryFile("w")
temp_filename = temp.name
temp.close()
temp = open(temp_filename, 'w')
temp = open(temp_filename, "w")
int(data.metadata.chromCol)
int(data.metadata.startCol)
int(data.metadata.strandCol)
@@ -67,15 +67,15 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool=None, stdout=No
for line in open(data.file_name):
line_ctr += 1
fields = line.strip().split('\t')
fields = line.strip().split("\t")
temp.write("%s\n" % '\t'.join(fields))
temp.write("%s\n" % "\t".join(fields))
temp.close()
shutil.move(temp_filename, data.file_name)
else:
data = app.datatypes_registry.change_datatype(data, 'tabular')
data = app.datatypes_registry.change_datatype(data, "tabular")
data.set_size()
data.set_peek()
app.model.context.add(data)
tools/data_source/import.py
+21 -21
View File
@@ -17,40 +17,40 @@ out_name = sys.argv[2]
id2name = {
'eryth': 'ErythPreCRMmm3_cusTrk.txt',
'cishg16': 'ReglRegHBBhg16CusTrk.txt',
'cishg17': 'ReglRegHBBhg17CusTrk.txt',
'exons': 'ExonsKnownGenes_mm3.txt',
'krhg16': 'known_regulatory_hg16.bed',
'krhg17': 'known_regulatory_hg17.bed',
'tARhg16mmc': 'hg16.mouse.t_AR.cold.bed',
'tARhg16mmm': 'hg16.mouse.t_AR.medium.bed',
'tARhg16mmh': 'hg16.mouse.t_AR.hot.bed',
'tARhg16rnc': 'hg16.rat.t_AR.cold.bed',
'tARhg16rnm': 'hg16.rat.t_AR.medium.bed',
'tARhg16rnh': 'hg16.rat.t_AR.hot.bed',
'phastConsHg16': 'phastConsMost_hg16.bed',
'omimhg16': 'omimDisorders_hg16.tab',
'omimhg17': 'omimDisorders_hg17.tab',
"eryth": "ErythPreCRMmm3_cusTrk.txt",
"cishg16": "ReglRegHBBhg16CusTrk.txt",
"cishg17": "ReglRegHBBhg17CusTrk.txt",
"exons": "ExonsKnownGenes_mm3.txt",
"krhg16": "known_regulatory_hg16.bed",
"krhg17": "known_regulatory_hg17.bed",
"tARhg16mmc": "hg16.mouse.t_AR.cold.bed",
"tARhg16mmm": "hg16.mouse.t_AR.medium.bed",
"tARhg16mmh": "hg16.mouse.t_AR.hot.bed",
"tARhg16rnc": "hg16.rat.t_AR.cold.bed",
"tARhg16rnm": "hg16.rat.t_AR.medium.bed",
"tARhg16rnh": "hg16.rat.t_AR.hot.bed",
"phastConsHg16": "phastConsMost_hg16.bed",
"omimhg16": "omimDisorders_hg16.tab",
"omimhg17": "omimDisorders_hg17.tab",
}
fname = id2name.get(dataid, '')
fname = id2name.get(dataid, "")
if not fname:
print('Importing invalid data %s' % dataid)
print("Importing invalid data %s" % dataid)
sys.exit()
else:
print('Imported %s' % fname)
print("Imported %s" % fname)
# this path is hardcoded
inp_name = os.path.join('database', 'import', fname)
inp_name = os.path.join("database", "import", fname)
try:
inp = open(inp_name)
except Exception:
print('Could not find file %s' % inp_name)
print("Could not find file %s" % inp_name)
sys.exit()
out = open(out_name, 'wt')
out = open(out_name, "wt")
while 1:
data = inp.read(BUFFER)
tools/data_source/microbial_import.py
+2 -2
View File
@@ -19,7 +19,7 @@ out_file1 = sys.argv[2]
have_none = True
while have_none:
try:
uids.remove('None')
uids.remove("None")
except ValueError:
have_none = False
@@ -31,7 +31,7 @@ try:
for line in open(filename):
if not line or line[0:1] == "#":
continue
fields = line.split('\t')
fields = line.split("\t")
try:
info_type = fields.pop(0)
tools/data_source/microbial_import_code.py
+54 -34
View File
@@ -3,7 +3,7 @@ from __future__ import print_function
from shutil import copyfile
def load_microbial_data(GALAXY_DATA_INDEX_DIR, sep='\t'):
def load_microbial_data(GALAXY_DATA_INDEX_DIR, sep="\t"):
# FIXME: this function is duplicated in the DynamicOptions class. It is used here only to
# set data.name in exec_after_process().
microbe_info = {}
@@ -11,8 +11,8 @@ def load_microbial_data(GALAXY_DATA_INDEX_DIR, sep='\t'):
filename = "%s/microbial_data.loc" % GALAXY_DATA_INDEX_DIR
for line in open(filename):
line = line.rstrip('\r\n')
if line and not line.startswith('#'):
line = line.rstrip("\r\n")
if line and not line.startswith("#"):
fields = line.split(sep)
# read each line, if not enough fields, go to next line
try:
@@ -28,13 +28,13 @@ def load_microbial_data(GALAXY_DATA_INDEX_DIR, sep='\t'):
link_site = fields.pop(0)
if org_num not in orgs:
orgs[org_num] = {}
orgs[org_num]['chrs'] = {}
orgs[org_num]['name'] = name
orgs[org_num]['kingdom'] = kingdom
orgs[org_num]['group'] = group
orgs[org_num]['chromosomes'] = chromosomes
orgs[org_num]['info_url'] = info_url
orgs[org_num]['link_site'] = link_site
orgs[org_num]["chrs"] = {}
orgs[org_num]["name"] = name
orgs[org_num]["kingdom"] = kingdom
orgs[org_num]["group"] = group
orgs[org_num]["chromosomes"] = chromosomes
orgs[org_num]["info_url"] = info_url
orgs[org_num]["link_site"] = link_site
elif info_type.upper() == "CHR":
# CHR 12521 CP000315 Clostridium perfringens phage phiSM101, complete genome 38092 110684521 CP000315.1
org_num = fields.pop(0)
@@ -45,15 +45,15 @@ def load_microbial_data(GALAXY_DATA_INDEX_DIR, sep='\t'):
gb = fields.pop(0)
info_url = fields.pop(0)
chr = {}
chr['name'] = name
chr['length'] = length
chr['gi'] = gi
chr['gb'] = gb
chr['info_url'] = info_url
chr["name"] = name
chr["length"] = length
chr["gi"] = gi
chr["gb"] = gb
chr["info_url"] = info_url
if org_num not in orgs:
orgs[org_num] = {}
orgs[org_num]['chrs'] = {}
orgs[org_num]['chrs'][chr_acc] = chr
orgs[org_num]["chrs"] = {}
orgs[org_num]["chrs"][chr_acc] = chr
elif info_type.upper() == "DATA":
# DATA 12521_12521_CDS 12521 CP000315 CDS bed /home/djb396/alignments/playground/bacteria/12521/CP000315.CDS.bed
uid = fields.pop(0)
@@ -63,26 +63,26 @@ def load_microbial_data(GALAXY_DATA_INDEX_DIR, sep='\t'):
filetype = fields.pop(0)
path = fields.pop(0)
data = {}
data['filetype'] = filetype
data['path'] = path
data['feature'] = feature
data["filetype"] = filetype
data["path"] = path
data["feature"] = feature
if org_num not in orgs:
orgs[org_num] = {}
orgs[org_num]['chrs'] = {}
if 'data' not in orgs[org_num]['chrs'][chr_acc]:
orgs[org_num]['chrs'][chr_acc]['data'] = {}
orgs[org_num]['chrs'][chr_acc]['data'][uid] = data
orgs[org_num]["chrs"] = {}
if "data" not in orgs[org_num]["chrs"][chr_acc]:
orgs[org_num]["chrs"][chr_acc]["data"] = {}
orgs[org_num]["chrs"][chr_acc]["data"][uid] = data
else:
continue
except Exception:
continue
for org_num in orgs:
org = orgs[org_num]
if org['kingdom'] not in microbe_info:
microbe_info[org['kingdom']] = {}
if org_num not in microbe_info[org['kingdom']]:
microbe_info[org['kingdom']][org_num] = org
if org["kingdom"] not in microbe_info:
microbe_info[org["kingdom"]] = {}
if org_num not in microbe_info[org["kingdom"]]:
microbe_info[org["kingdom"]][org_num] = org
return microbe_info
@@ -93,15 +93,15 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr
if history is None:
print("unknown history!")
return
kingdom = param_dict.get('kingdom', None)
org = param_dict.get('org', None)
kingdom = param_dict.get("kingdom", None)
org = param_dict.get("org", None)
# if not (kingdom or group or org):
if not (kingdom or org):
print("Parameters are not available.")
GALAXY_DATA_INDEX_DIR = app.config.tool_data_path
microbe_info = load_microbial_data(GALAXY_DATA_INDEX_DIR, sep='\t')
microbe_info = load_microbial_data(GALAXY_DATA_INDEX_DIR, sep="\t")
split_stdout = stdout.split("\n")
basic_name = ""
for line in split_stdout:
@@ -114,7 +114,16 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr
data = next(iter(out_data.values()))
data.set_size()
basic_name = data.name
data.name = data.name + " (" + microbe_info[kingdom][org]['chrs'][chr]['data'][description]['feature'] + " for " + microbe_info[kingdom][org]['name'] + ":" + chr + ")"
data.name = (
data.name
+ " ("
+ microbe_info[kingdom][org]["chrs"][chr]["data"][description]["feature"]
+ " for "
+ microbe_info[kingdom][org]["name"]
+ ":"
+ chr
+ ")"
)
data.dbkey = dbkey
data.info = data.name
data = app.datatypes_registry.change_datatype(data, file_type)
@@ -128,10 +137,21 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr
dbkey = fields[3]
filepath = fields[4]
file_type = fields[5]
newdata = app.model.HistoryDatasetAssociation(create_dataset=True, sa_session=app.model.context) # This import should become a library
newdata = app.model.HistoryDatasetAssociation(
create_dataset=True, sa_session=app.model.context
) # This import should become a library
newdata.set_size()
newdata.extension = file_type
newdata.name = basic_name + " (" + microbe_info[kingdom][org]['chrs'][chr]['data'][description]['feature'] + " for " + microbe_info[kingdom][org]['name'] + ":" + chr + ")"
newdata.name = (
basic_name
+ " ("
+ microbe_info[kingdom][org]["chrs"][chr]["data"][description]["feature"]
+ " for "
+ microbe_info[kingdom][org]["name"]
+ ":"
+ chr
+ ")"
)
app.model.context.add(newdata)
app.model.context.flush()
app.security_agent.copy_dataset_permissions(base_dataset.dataset, newdata.dataset)
tools/data_source/upload.py
+54 -50
View File
@@ -40,6 +40,7 @@ def get_file_sources():
global _file_sources
if _file_sources is None:
from galaxy.files import ConfiguredFileSources
file_sources = None
if os.path.exists("file_sources.json"):
file_sources_as_dict = None
@@ -55,16 +56,12 @@ def get_file_sources():
def file_err(msg, dataset):
# never remove a server-side upload
if dataset.type not in ('server_dir', 'path_paste'):
if dataset.type not in ("server_dir", "path_paste"):
try:
os.remove(dataset.path)
except Exception:
pass
return dict(type='dataset',
ext='data',
dataset_id=dataset.dataset_id,
stderr=msg,
failed=True)
return dict(type="dataset", ext="data", dataset_id=dataset.dataset_id, stderr=msg, failed=True)
def safe_dict(d):
@@ -80,7 +77,7 @@ def safe_dict(d):
def parse_outputs(args):
rval = {}
for arg in args:
id, files_path, path = arg.split(':', 2)
id, files_path, path = arg.split(":", 2)
rval[int(id)] = (path, files_path)
return rval
@@ -89,16 +86,18 @@ def add_file(dataset, registry, output_path: str) -> Dict[str, str]:
ext = None
compression_type = None
line_count = None
link_data_only_str = dataset.get('link_data_only', 'copy_files')
if link_data_only_str not in ['link_to_files', 'copy_files']:
raise UploadProblemException("Invalid setting '%s' for option link_data_only - upload request misconfigured" % link_data_only_str)
link_data_only = link_data_only_str == 'link_to_files'
link_data_only_str = dataset.get("link_data_only", "copy_files")
if link_data_only_str not in ["link_to_files", "copy_files"]:
raise UploadProblemException(
"Invalid setting '%s' for option link_data_only - upload request misconfigured" % link_data_only_str
)
link_data_only = link_data_only_str == "link_to_files"
# run_as_real_user is estimated from galaxy config (external chmod indicated of inputs executed)
# If this is True we always purge supplied upload inputs so they are cleaned up and we reuse their
# paths during data conversions since this user already owns that path.
# Older in_place check for upload jobs created before 18.01, TODO remove in 19.XX. xref #5206
run_as_real_user = dataset.get('run_as_real_user', False) or dataset.get("in_place", False)
run_as_real_user = dataset.get("run_as_real_user", False) or dataset.get("in_place", False)
# purge_source defaults to True unless this is an FTP import and
# ftp_upload_purge has been overridden to False in Galaxy's config.
@@ -106,43 +105,45 @@ def add_file(dataset, registry, output_path: str) -> Dict[str, str]:
# - the job does not have write access to the file, e.g. when running as the
# real user
# - the files are uploaded from external paths.
purge_source = dataset.get('purge_source', True) and not run_as_real_user and dataset.type not in ('server_dir', 'path_paste')
purge_source = (
dataset.get("purge_source", True) and not run_as_real_user and dataset.type not in ("server_dir", "path_paste")
)
# in_place is True unless we are running as a real user or importing external paths (i.e.
# this is a real upload and not a path paste or ftp import).
# in_place should always be False if running as real user because the uploaded file will
# be owned by Galaxy and not the user and it should be False for external paths so Galaxy doesn't
# modify files not controlled by Galaxy.
in_place = not run_as_real_user and dataset.type not in ('server_dir', 'path_paste', 'ftp_import')
in_place = not run_as_real_user and dataset.type not in ("server_dir", "path_paste", "ftp_import")
# Base on the check_upload_content Galaxy config option and on by default, this enables some
# security related checks on the uploaded content, but can prevent uploads from working in some cases.
check_content = dataset.get('check_content', True)
check_content = dataset.get("check_content", True)
# auto_decompress is a request flag that can be swapped off to prevent Galaxy from automatically
# decompressing archive files before sniffing.
auto_decompress = dataset.get('auto_decompress', True)
auto_decompress = dataset.get("auto_decompress", True)
try:
dataset.file_type
except AttributeError:
raise UploadProblemException('Unable to process uploaded file, missing file_type parameter.')
raise UploadProblemException("Unable to process uploaded file, missing file_type parameter.")
if dataset.type == 'url':
if dataset.type == "url":
try:
dataset.path = sniff.stream_url_to_file(dataset.path, file_sources=get_file_sources())
except Exception as e:
raise UploadProblemException('Unable to fetch %s\n%s' % (dataset.path, unicodify(e)))
raise UploadProblemException("Unable to fetch %s\n%s" % (dataset.path, unicodify(e)))
# See if we have an empty file
if not os.path.exists(dataset.path):
raise UploadProblemException('Uploaded temporary file (%s) does not exist.' % dataset.path)
raise UploadProblemException("Uploaded temporary file (%s) does not exist." % dataset.path)
stdout, ext, datatype, is_binary, converted_path, _, _ = handle_upload(
registry=registry,
path=dataset.path,
requested_ext=dataset.file_type,
name=dataset.name,
tmp_prefix='data_id_%s_upload_' % dataset.dataset_id,
tmp_prefix="data_id_%s_upload_" % dataset.dataset_id,
tmp_dir=output_adjacent_tmpdir(output_path),
check_content=check_content,
link_data_only=link_data_only,
@@ -153,15 +154,21 @@ def add_file(dataset, registry, output_path: str) -> Dict[str, str]:
)
# Strip compression extension from name
if compression_type and not getattr(datatype, 'compressed', False) and dataset.name.endswith('.' + compression_type):
dataset.name = dataset.name[:-len('.' + compression_type)]
if (
compression_type
and not getattr(datatype, "compressed", False)
and dataset.name.endswith("." + compression_type)
):
dataset.name = dataset.name[: -len("." + compression_type)]
# Move dataset
if link_data_only:
# Never alter a file that will not be copied to Galaxy's local file store.
if datatype.dataset_content_needs_grooming(dataset.path):
err_msg = 'The uploaded files need grooming, so change your <b>Copy data into Galaxy?</b> selection to be ' + \
'<b>Copy files into Galaxy</b> instead of <b>Link to files without copying into Galaxy</b> so grooming can be performed.'
err_msg = (
"The uploaded files need grooming, so change your <b>Copy data into Galaxy?</b> selection to be "
+ "<b>Copy files into Galaxy</b> instead of <b>Link to files without copying into Galaxy</b> so grooming can be performed."
)
raise UploadProblemException(err_msg)
if not link_data_only:
# Move the dataset to its "real" path. converted_path is a tempfile so we move it even if purge_source is False.
@@ -181,15 +188,12 @@ def add_file(dataset, registry, output_path: str) -> Dict[str, str]:
shutil.copy(dataset.path, output_path)
# Write the job info
stdout = stdout or 'uploaded %s file' % ext
info = dict(type='dataset',
dataset_id=dataset.dataset_id,
ext=ext,
stdout=stdout,
name=dataset.name,
line_count=line_count)
if dataset.get('uuid', None) is not None:
info['uuid'] = dataset.get('uuid')
stdout = stdout or "uploaded %s file" % ext
info = dict(
type="dataset", dataset_id=dataset.dataset_id, ext=ext, stdout=stdout, name=dataset.name, line_count=line_count
)
if dataset.get("uuid", None) is not None:
info["uuid"] = dataset.get("uuid")
# FIXME: does this belong here? also not output-adjacent-tmpdir aware =/
if not link_data_only and datatype and datatype.dataset_content_needs_grooming(output_path):
# Groom the dataset content if necessary
@@ -211,7 +215,7 @@ def add_composite_file(dataset, registry, output_path, files_path):
try:
temp_name = sniff.stream_url_to_file(path_or_url, file_sources=file_sources)
except Exception as e:
raise UploadProblemException('Unable to fetch %s\n%s' % (path_or_url, unicodify(e)))
raise UploadProblemException("Unable to fetch %s\n%s" % (path_or_url, unicodify(e)))
return temp_name, isa_url
@@ -221,13 +225,13 @@ def add_composite_file(dataset, registry, output_path, files_path):
safe_makedirs(files_path)
def stage_file(name, composite_file_path, is_binary=False):
dp = composite_file_path['path']
dp = composite_file_path["path"]
path, isa_url = to_path(dp)
if isa_url:
dataset.path = path
dp = path
auto_decompress = composite_file_path.get('auto_decompress', True)
auto_decompress = composite_file_path.get("auto_decompress", True)
if auto_decompress and not datatype.composite_type and CompressedFile.can_decompress(dp):
# It isn't an explicitly composite datatype, so these are just extra files to attach
# as composite data. It'd be better if Galaxy was communicating this to the tool
@@ -237,7 +241,7 @@ def add_composite_file(dataset, registry, output_path, files_path):
CompressedFile(dp).extract(files_path)
else:
tmpdir = output_adjacent_tmpdir(output_path)
tmp_prefix = 'data_id_%s_convert_' % dataset.dataset_id
tmp_prefix = "data_id_%s_convert_" % dataset.dataset_id
sniff.handle_composite_file(
datatype,
dp,
@@ -255,9 +259,11 @@ def add_composite_file(dataset, registry, output_path, files_path):
for name, value in dataset.composite_files.items():
value = bunch.Bunch(**value)
if value.name not in dataset.composite_file_paths:
raise UploadProblemException("Failed to find file_path %s in %s" % (value.name, dataset.composite_file_paths))
raise UploadProblemException(
"Failed to find file_path %s in %s" % (value.name, dataset.composite_file_paths)
)
if dataset.composite_file_paths[value.name] is None and not value.optional:
raise UploadProblemException('A required composite data file was not provided (%s)' % name)
raise UploadProblemException("A required composite data file was not provided (%s)" % name)
elif dataset.composite_file_paths[value.name] is not None:
composite_file_path = dataset.composite_file_paths[value.name]
stage_file(name, composite_file_path, value.is_binary)
@@ -273,9 +279,7 @@ def add_composite_file(dataset, registry, output_path, files_path):
shutil.move(primary_file_path, output_path)
# Write the job info
return dict(type='dataset',
dataset_id=dataset.dataset_id,
stdout='uploaded %s file' % dataset.file_type)
return dict(type="dataset", dataset_id=dataset.dataset_id, stdout="uploaded %s file" % dataset.file_type)
def __read_paramfile(path):
@@ -296,14 +300,14 @@ def __read_old_paramfile(path):
def __write_job_metadata(metadata):
# TODO: make upload/set_metadata compatible with https://github.com/galaxyproject/galaxy/pull/4437
with open('galaxy.json', 'w') as fh:
with open("galaxy.json", "w") as fh:
for meta in metadata:
dump(meta, fh)
fh.write('\n')
fh.write("\n")
def output_adjacent_tmpdir(output_path):
""" For temp files that will ultimately be moved to output_path anyway
"""For temp files that will ultimately be moved to output_path anyway
just create the file directly in output_path's directory so shutil.move
will work optimally.
"""
@@ -313,7 +317,7 @@ def output_adjacent_tmpdir(output_path):
def __main__():
if len(sys.argv) < 4:
print('usage: upload.py <root> <datatypes_conf> <json paramfile> <output spec> ...', file=sys.stderr)
print("usage: upload.py <root> <datatypes_conf> <json paramfile> <output spec> ...", file=sys.stderr)
sys.exit(1)
output_paths = parse_outputs(sys.argv[4:])
@@ -332,10 +336,10 @@ def __main__():
try:
output_path = output_paths[int(dataset.dataset_id)][0]
except Exception:
print('Output path for dataset %s not found on command line' % dataset.dataset_id, file=sys.stderr)
print("Output path for dataset %s not found on command line" % dataset.dataset_id, file=sys.stderr)
sys.exit(1)
try:
if dataset.type == 'composite':
if dataset.type == "composite":
files_path = output_paths[int(dataset.dataset_id)][1]
metadata.append(add_composite_file(dataset, registry, output_path, files_path))
else:
@@ -345,5 +349,5 @@ def __main__():
__write_job_metadata(metadata)
if __name__ == '__main__':
if __name__ == "__main__":
__main__()