opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-21 05:45:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

Apply black formatting.

Browse Source 
Apply isort.
This commit is contained in:
Dannon Baker
2022-02-03 07:42:20 -05:00
parent ff2f3600f4
commit 623a5e165a
1589 changed files with 91081 additions and 66217 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/api/common.py
+40 -40
View File
@@ -20,12 +20,12 @@ def make_url(api_key, url, args=None):
"""
if args is None:
args = []
argsep = '&'
if '?' not in url:
argsep = '?'
if '?key=' not in url and '&key=' not in url:
args.insert(0, ('key', api_key))
return url + argsep + '&'.join('='.join(t) for t in args)
argsep = "&"
if "?" not in url:
argsep = "?"
if "?key=" not in url and "&key=" not in url:
args.insert(0, ("key", api_key))
return url + argsep + "&".join("=".join(t) for t in args)
def get(api_key, url):
@@ -45,7 +45,7 @@ def post(api_key, url, data):
Do the actual POST.
"""
url = make_url(api_key, url)
req = Request(url, headers={'Content-Type': 'application/json'}, data=json.dumps(data))
req = Request(url, headers={"Content-Type": "application/json"}, data=json.dumps(data))
return json.loads(urlopen(req).read())
@@ -54,8 +54,8 @@ def put(api_key, url, data):
Do the actual PUT
"""
url = make_url(api_key, url)
req = Request(url, headers={'Content-Type': 'application/json'}, data=json.dumps(data))
req.get_method = lambda: 'PUT'
req = Request(url, headers={"Content-Type": "application/json"}, data=json.dumps(data))
req.get_method = lambda: "PUT"
return json.loads(urlopen(req).read())
@@ -64,8 +64,8 @@ def __del(api_key, url, data):
Do the actual DELETE
"""
url = make_url(api_key, url)
req = Request(url, headers={'Content-Type': 'application/json'}, data=json.dumps(data))
req.get_method = lambda: 'DELETE'
req = Request(url, headers={"Content-Type": "application/json"}, data=json.dumps(data))
req.get_method = lambda: "DELETE"
return json.loads(urlopen(req).read())
@@ -83,36 +83,36 @@ def display(api_key, url, return_formatted=True):
return r
elif type(r) == list:
# Response is a collection as defined in the REST style.
print('Collection Members')
print('------------------')
print("Collection Members")
print("------------------")
for n, i in enumerate(r):
if isinstance(i, str):
print(' %s' % i)
print(" %s" % i)
else:
# All collection members should have a name in the response.
# url is optional
if 'url' in i:
print('#%d: %s' % (n + 1, i.pop('url')))
if 'name' in i:
print(' name: %s' % i.pop('name'))
if "url" in i:
print("#%d: %s" % (n + 1, i.pop("url")))
if "name" in i:
print(" name: %s" % i.pop("name"))
try:
for k, v in i.items():
print(f' {k}: {v}')
print(f" {k}: {v}")
except AttributeError:
for item in i:
print(item)
print('')
print('%d element(s) in collection' % len(r))
print("")
print("%d element(s) in collection" % len(r))
elif type(r) == dict:
# Response is an element as defined in the REST style.
print('Member Information')
print('------------------')
print("Member Information")
print("------------------")
for k, v in r.items():
print(f'{k}: {v}')
print(f"{k}: {v}")
elif type(r) == str:
print(r)
else:
print('response is unknown type: %s' % type(r))
print("response is unknown type: %s" % type(r))
def submit(api_key, url, data, return_formatted=True):
@@ -128,24 +128,24 @@ def submit(api_key, url, data, return_formatted=True):
print(e.read(1024))
sys.exit(1)
else:
return 'Error. ' + str(e.read(1024))
return "Error. " + str(e.read(1024))
if not return_formatted:
return r
print('Response')
print('--------')
print("Response")
print("--------")
if type(r) == list:
# Currently the only implemented responses are lists of dicts, because
# submission creates some number of collection elements.
for i in r:
if type(i) == dict:
if 'url' in i:
print(i.pop('url'))
if "url" in i:
print(i.pop("url"))
else:
print('----')
if 'name' in i:
print(' name: %s' % i.pop('name'))
print("----")
if "name" in i:
print(" name: %s" % i.pop("name"))
for k, v in i.items():
print(f' {k}: {v}')
print(f" {k}: {v}")
else:
print(i)
else:
@@ -165,11 +165,11 @@ def update(api_key, url, data, return_formatted=True):
print(e.read(1024))
sys.exit(1)
else:
return 'Error. ' + str(e.read(1024))
return "Error. " + str(e.read(1024))
if not return_formatted:
return r
print('Response')
print('--------')
print("Response")
print("--------")
print(r)
@@ -186,9 +186,9 @@ def delete(api_key, url, data, return_formatted=True):
print(e.read(1024))
sys.exit(1)
else:
return 'Error. ' + str(e.read(1024))
return "Error. " + str(e.read(1024))
if not return_formatted:
return r
print('Response')
print('--------')
print("Response")
print("--------")
print(r)
scripts/api/copy_hda_to_library_folder.py
+7 -7
View File
@@ -7,12 +7,12 @@ from common import submit
usage = "USAGE: copy_hda_to_library_folder.py <base url> <api key> <hda id> <library id> <folder id> [ message ]"
def copy_hda_to_library_folder(base_url, key, hda_id, library_id, folder_id, message=''):
url = f'http://{base_url}/api/libraries/{library_id}/contents'
def copy_hda_to_library_folder(base_url, key, hda_id, library_id, folder_id, message=""):
url = f"http://{base_url}/api/libraries/{library_id}/contents"
payload = {
'folder_id': folder_id,
'create_type': 'file',
'from_hda_id': hda_id,
"folder_id": folder_id,
"create_type": "file",
"from_hda_id": hda_id,
}
if message:
payload.update(dict(ldda_message=message))
@@ -20,7 +20,7 @@ def copy_hda_to_library_folder(base_url, key, hda_id, library_id, folder_id, mes
return submit(key, url, payload)
if __name__ == '__main__':
if __name__ == "__main__":
num_args = len(sys.argv)
if num_args < 6:
print(usage, file=sys.stderr)
@@ -28,7 +28,7 @@ if __name__ == '__main__':
(base_url, key, hda_id, library_id, folder_id) = sys.argv[1:6]
message = ''
message = ""
if num_args >= 7:
message = sys.argv[6]
scripts/api/create.py
+1 -1
View File
@@ -9,7 +9,7 @@ import sys
from common import submit
data = {}
for k, v in [kwarg.split('=', 1) for kwarg in sys.argv[3:]]:
for k, v in [kwarg.split("=", 1) for kwarg in sys.argv[3:]]:
data[k] = v
submit(sys.argv[1], sys.argv[2], data)
scripts/api/data_manager_example_execute.py
+77 -31
View File
@@ -8,30 +8,35 @@ import optparse
import time
from urllib.parse import urljoin
from common import get, post # noqa: I100,I202
from common import ( # noqa: I100,I202
get,
post,
)
DEFAULT_SLEEP_TIME = 3
FETCH_GENOME_TOOL_ID = 'testtoolshed.g2.bx.psu.edu/repos/blankenberg/data_manager_fetch_genome_all_fasta/data_manager_fetch_genome_all_fasta/0.0.1'
BUILD_INDEX_TOOLS_ID = ['testtoolshed.g2.bx.psu.edu/repos/blankenberg/data_manager_bwa_index_builder/bwa_index_builder_data_manager/0.0.1',
'testtoolshed.g2.bx.psu.edu/repos/blankenberg/data_manager_bwa_index_builder/bwa_color_space_index_builder_data_manager/0.0.1']
FETCH_GENOME_TOOL_ID = "testtoolshed.g2.bx.psu.edu/repos/blankenberg/data_manager_fetch_genome_all_fasta/data_manager_fetch_genome_all_fasta/0.0.1"
BUILD_INDEX_TOOLS_ID = [
"testtoolshed.g2.bx.psu.edu/repos/blankenberg/data_manager_bwa_index_builder/bwa_index_builder_data_manager/0.0.1",
"testtoolshed.g2.bx.psu.edu/repos/blankenberg/data_manager_bwa_index_builder/bwa_color_space_index_builder_data_manager/0.0.1",
]
def run_tool(tool_id, history_id, params, api_key, galaxy_url, wait=True, sleep_time=None, **kwargs):
sleep_time = sleep_time or DEFAULT_SLEEP_TIME
tools_url = urljoin(galaxy_url, 'api/tools')
tools_url = urljoin(galaxy_url, "api/tools")
payload = {
'tool_id': tool_id,
"tool_id": tool_id,
}
if history_id:
payload['history_id'] = history_id
payload['inputs'] = params
payload["history_id"] = history_id
payload["inputs"] = params
rval = post(api_key, tools_url, payload)
if wait:
outputs = list(rval['outputs'])
outputs = list(rval["outputs"])
while outputs:
finished_datasets = []
for i, dataset_dict in enumerate(outputs):
if dataset_is_terminal(dataset_dict['id'], api_key=api_key, galaxy_url=galaxy_url):
if dataset_is_terminal(dataset_dict["id"], api_key=api_key, galaxy_url=galaxy_url):
finished_datasets.append(i)
for _ in reversed(finished_datasets):
outputs.pop(0)
@@ -42,64 +47,105 @@ def run_tool(tool_id, history_id, params, api_key, galaxy_url, wait=True, sleep_
def get_dataset_state(hda_id, api_key, galaxy_url):
datasets_url = urljoin(galaxy_url, 'api/datasets/%s' % hda_id)
datasets_url = urljoin(galaxy_url, "api/datasets/%s" % hda_id)
dataset_info = get(api_key, datasets_url)
return dataset_info['state']
return dataset_info["state"]
def dataset_is_terminal(hda_id, api_key, galaxy_url):
dataset_state = get_dataset_state(hda_id, api_key, galaxy_url)
return dataset_state in ['ok', 'error']
return dataset_state in ["ok", "error"]
if __name__ == '__main__':
if __name__ == "__main__":
parser = optparse.OptionParser()
parser.add_option('-k', '--key', dest='api_key', action='store', type="string", default=None, help='API Key.')
parser.add_option('-u', '--url', dest='base_url', action='store', type="string", default='http://localhost:8080', help='Base URL of Galaxy Server')
parser.add_option('-d', '--dbkey', dest='dbkeys', action='append', type="string", default=[], help='List of dbkeys to download and Index')
parser.add_option('-s', '--sleep_time', dest='sleep_time', action='store', type="int", default=DEFAULT_SLEEP_TIME, help='How long to sleep between check loops')
parser.add_option("-k", "--key", dest="api_key", action="store", type="string", default=None, help="API Key.")
parser.add_option(
"-u",
"--url",
dest="base_url",
action="store",
type="string",
default="http://localhost:8080",
help="Base URL of Galaxy Server",
)
parser.add_option(
"-d",
"--dbkey",
dest="dbkeys",
action="append",
type="string",
default=[],
help="List of dbkeys to download and Index",
)
parser.add_option(
"-s",
"--sleep_time",
dest="sleep_time",
action="store",
type="int",
default=DEFAULT_SLEEP_TIME,
help="How long to sleep between check loops",
)
(options, args) = parser.parse_args()
# check options
assert options.api_key is not None, ValueError('You must specify an API key.')
assert options.dbkeys, ValueError('You must specify at least one dbkey to use.')
assert options.api_key is not None, ValueError("You must specify an API key.")
assert options.dbkeys, ValueError("You must specify at least one dbkey to use.")
# check user is admin
configuration_options = get(options.api_key, urljoin(options.base_url, 'api/configuration'))
if 'library_import_dir' not in configuration_options: # hack to check if is admin user
print("Warning: Data Managers are only available to admin users. The API Key provided does not appear to belong to an admin user. Will attempt to run anyway.")
configuration_options = get(options.api_key, urljoin(options.base_url, "api/configuration"))
if "library_import_dir" not in configuration_options: # hack to check if is admin user
print(
"Warning: Data Managers are only available to admin users. The API Key provided does not appear to belong to an admin user. Will attempt to run anyway."
)
# Fetch Genomes
dbkeys = {}
for dbkey in options.dbkeys:
if dbkey not in dbkeys:
dbkeys[dbkey] = run_tool(FETCH_GENOME_TOOL_ID, None, {'dbkey': dbkey, 'reference_source|reference_source_selector': 'ucsc', 'reference_source|requested_dbkey': dbkey}, options.api_key, options.base_url, wait=False)
dbkeys[dbkey] = run_tool(
FETCH_GENOME_TOOL_ID,
None,
{
"dbkey": dbkey,
"reference_source|reference_source_selector": "ucsc",
"reference_source|requested_dbkey": dbkey,
},
options.api_key,
options.base_url,
wait=False,
)
else:
"dbkey (%s) was specified more than once, skipping additional specification." % (dbkey)
print('Genomes Queued for downloading.')
print("Genomes Queued for downloading.")
# Start indexers
indexing_tools = []
while dbkeys:
for dbkey, value in dbkeys.items():
if dataset_is_terminal(value['outputs'][0]['id'], options.api_key, options.base_url):
if dataset_is_terminal(value["outputs"][0]["id"], options.api_key, options.base_url):
del dbkeys[dbkey]
for tool_id in BUILD_INDEX_TOOLS_ID:
indexing_tools.append(run_tool(tool_id, None, {'all_fasta_source': dbkey}, options.api_key, options.base_url, wait=False))
indexing_tools.append(
run_tool(
tool_id, None, {"all_fasta_source": dbkey}, options.api_key, options.base_url, wait=False
)
)
if dbkeys:
time.sleep(options.sleep_time)
print('All genomes downloaded and indexers now queued.')
print("All genomes downloaded and indexers now queued.")
# Wait for indexers to finish
while indexing_tools:
for i, indexing_tool_value in enumerate(indexing_tools):
if dataset_is_terminal(indexing_tool_value['outputs'][0]['id'], options.api_key, options.base_url):
print('Finished:', indexing_tool_value)
if dataset_is_terminal(indexing_tool_value["outputs"][0]["id"], options.api_key, options.base_url):
print("Finished:", indexing_tool_value)
del indexing_tools[i]
break
if indexing_tools:
time.sleep(options.sleep_time)
print('All indexers have been run, please check results.')
print("All indexers have been run, please check results.")
scripts/api/delete.py
+1 -1
View File
@@ -9,7 +9,7 @@ import sys
from common import delete
data = {}
for k, v in [kwarg.split('=', 1) for kwarg in sys.argv[3:]]:
for k, v in [kwarg.split("=", 1) for kwarg in sys.argv[3:]]:
data[k] = v
delete(sys.argv[1], sys.argv[2], data)
scripts/api/display.py
+1 -1
View File
@@ -9,7 +9,7 @@ from common import display # noqa: I100,I202
try:
display(*sys.argv[1:3])
except TypeError as e:
print('usage: %s key url' % os.path.basename(sys.argv[0]))
print("usage: %s key url" % os.path.basename(sys.argv[0]))
print(e)
sys.exit(1)
except URLError as e:
scripts/api/example_watch_folder.py
+28 -25
View File
@@ -16,25 +16,28 @@ import shutil
import sys
import time
from common import display, submit
from common import (
display,
submit,
)
def main(api_key, api_url, in_folder, out_folder, data_library, workflow):
# Find/Create data library with the above name. Assume we're putting datasets in the root folder '/'
libs = display(api_key, api_url + 'libraries', return_formatted=False)
libs = display(api_key, api_url + "libraries", return_formatted=False)
library_id = None
for library in libs:
if library['name'] == data_library:
library_id = library['id']
if library["name"] == data_library:
library_id = library["id"]
if not library_id:
lib_create_data = {'name': data_library}
library = submit(api_key, api_url + 'libraries', lib_create_data, return_formatted=False)
library_id = library[0]['id']
lib_create_data = {"name": data_library}
library = submit(api_key, api_url + "libraries", lib_create_data, return_formatted=False)
library_id = library[0]["id"]
folders = display(api_key, api_url + "libraries/%s/contents" % library_id, return_formatted=False)
for f in folders:
if f['name'] == "/":
library_folder_id = f['id']
workflow = display(api_key, api_url + 'workflows/%s' % workflow, return_formatted=False)
if f["name"] == "/":
library_folder_id = f["id"]
workflow = display(api_key, api_url + "workflows/%s" % workflow, return_formatted=False)
if not workflow:
print("Workflow %s not found, terminating.")
sys.exit(1)
@@ -48,27 +51,27 @@ def main(api_key, api_url, in_folder, out_folder, data_library, workflow):
fullpath = os.path.join(in_folder, fname)
if os.path.isfile(fullpath):
data = {}
data['folder_id'] = library_folder_id
data['file_type'] = 'auto'
data['dbkey'] = ''
data['upload_option'] = 'upload_paths'
data['filesystem_paths'] = fullpath
data['create_type'] = 'file'
data["folder_id"] = library_folder_id
data["file_type"] = "auto"
data["dbkey"] = ""
data["upload_option"] = "upload_paths"
data["filesystem_paths"] = fullpath
data["create_type"] = "file"
libset = submit(api_key, api_url + "libraries/%s/contents" % library_id, data, return_formatted=False)
# TODO Handle this better, but the datatype isn't always
# set for the followup workflow execution without this
# pause.
time.sleep(5)
for ds in libset:
if 'id' in ds:
if "id" in ds:
# Successful upload of dataset, we have the ldda now. Run the workflow.
wf_data = {}
wf_data['workflow_id'] = workflow['id']
wf_data['history'] = "{} - {}".format(fname, workflow['name'])
wf_data['ds_map'] = {}
for step_id in workflow['inputs'].keys():
wf_data['ds_map'][step_id] = {'src': 'ld', 'id': ds['id']}
res = submit(api_key, api_url + 'workflows', wf_data, return_formatted=False)
wf_data["workflow_id"] = workflow["id"]
wf_data["history"] = "{} - {}".format(fname, workflow["name"])
wf_data["ds_map"] = {}
for step_id in workflow["inputs"].keys():
wf_data["ds_map"][step_id] = {"src": "ld", "id": ds["id"]}
res = submit(api_key, api_url + "workflows", wf_data, return_formatted=False)
if res:
print(res)
# Successful workflow execution, safe to move dataset.
@@ -76,7 +79,7 @@ def main(api_key, api_url, in_folder, out_folder, data_library, workflow):
time.sleep(10)
if __name__ == '__main__':
if __name__ == "__main__":
try:
api_key = sys.argv[1]
api_url = sys.argv[2]
@@ -85,6 +88,6 @@ if __name__ == '__main__':
data_library = sys.argv[5]
workflow = sys.argv[6]
except IndexError:
print('usage: %s key url in_folder out_folder data_library workflow' % os.path.basename(sys.argv[0]))
print("usage: %s key url in_folder out_folder data_library workflow" % os.path.basename(sys.argv[0]))
sys.exit(1)
main(api_key, api_url, in_folder, out_folder, data_library, workflow)
scripts/api/fetch_to_library.py
+6 -11
View File
@@ -6,28 +6,23 @@ import yaml
def main():
parser = argparse.ArgumentParser(description='Upload a directory into a data library')
parser = argparse.ArgumentParser(description="Upload a directory into a data library")
parser.add_argument("-u", "--url", dest="url", required=True, help="Galaxy URL")
parser.add_argument("-a", "--api", dest="api_key", required=True, help="API Key")
parser.add_argument('target', metavar='FILE', type=str,
help='file describing data library to fetch')
parser.add_argument("target", metavar="FILE", type=str, help="file describing data library to fetch")
args = parser.parse_args()
with open(args.target) as f:
target = yaml.safe_load(f)
histories_url = args.url + "/api/histories"
new_history_response = requests.post(histories_url, data={'key': args.api_key})
new_history_response = requests.post(histories_url, data={"key": args.api_key})
fetch_url = args.url + '/api/tools/fetch'
payload = {
'key': args.api_key,
'targets': json.dumps([target]),
'history_id': new_history_response.json()["id"]
}
fetch_url = args.url + "/api/tools/fetch"
payload = {"key": args.api_key, "targets": json.dumps([target]), "history_id": new_history_response.json()["id"]}
response = requests.post(fetch_url, data=payload)
print(response.content)
if __name__ == '__main__':
if __name__ == "__main__":
main()
scripts/api/filter_failed_datasets_from_collection.py
+22 -15
View File
@@ -9,13 +9,14 @@ python filter_failed_datasets_from_collection.py <GalaxyUrl> <ApiKey> MySpecialH
import sys
from bioblend.galaxy import (
dataset_collections as collections,
GalaxyInstance
)
from bioblend.galaxy import dataset_collections as collections
from bioblend.galaxy import GalaxyInstance
if len(sys.argv) < 5:
print("Usage: %s <GalaxyUrl> <ApiKey> <HistoryName (must be unique)> <CollectionHistoryId (i.e. the simple integer id)>" % sys.argv[0])
print(
"Usage: %s <GalaxyUrl> <ApiKey> <HistoryName (must be unique)> <CollectionHistoryId (i.e. the simple integer id)>"
% sys.argv[0]
)
exit(0)
galaxyUrl = sys.argv[1]
@@ -26,13 +27,13 @@ collectionHistoryId = int(sys.argv[4])
gi = GalaxyInstance(url=galaxyUrl, key=galaxyApiKey)
historyMatches = gi.histories.get_histories(name=historyName)
if (len(historyMatches) > 1):
if len(historyMatches) > 1:
print("Error: more than one history matches that name.")
exit(1)
historyId = historyMatches[0]['id']
historyId = historyMatches[0]["id"]
historyContents = gi.histories.show_history(historyId, contents=True, deleted=False, visible=True, details=False)
matchingCollections = [x for x in historyContents if x['hid'] == collectionHistoryId]
matchingCollections = [x for x in historyContents if x["hid"] == collectionHistoryId]
if len(matchingCollections) == 0:
print("Error: no collections matching that id found.")
@@ -42,21 +43,27 @@ if len(matchingCollections) > 1:
print("Error: more than one collection matching that id found (WTF?)")
exit(1)
collectionId = matchingCollections[0]['id']
collectionId = matchingCollections[0]["id"]
failedCollection = gi.histories.show_dataset_collection(historyId, collectionId)
okDatasets = [d for d in failedCollection['elements'] if d['object']['state'] == 'ok' and d['object']['file_size'] > 0]
notOkDatasets = [d for d in failedCollection['elements'] if d['object']['state'] != 'ok' or d['object']['file_size'] == 0]
okCollectionName = failedCollection['name'] + " (ok)"
notOkCollectionName = failedCollection['name'] + " (not ok)"
okDatasets = [d for d in failedCollection["elements"] if d["object"]["state"] == "ok" and d["object"]["file_size"] > 0]
notOkDatasets = [
d for d in failedCollection["elements"] if d["object"]["state"] != "ok" or d["object"]["file_size"] == 0
]
okCollectionName = failedCollection["name"] + " (ok)"
notOkCollectionName = failedCollection["name"] + " (not ok)"
gi.histories.create_dataset_collection(
history_id=historyId,
collection_description=collections.CollectionDescription(
name=okCollectionName,
elements=[collections.HistoryDatasetElement(d['object']['name'], d['object']['id']) for d in okDatasets]))
elements=[collections.HistoryDatasetElement(d["object"]["name"], d["object"]["id"]) for d in okDatasets],
),
)
gi.histories.create_dataset_collection(
history_id=historyId,
collection_description=collections.CollectionDescription(
name=notOkCollectionName,
elements=[collections.HistoryDatasetElement(d['object']['name'], d['object']['id']) for d in notOkDatasets]))
elements=[collections.HistoryDatasetElement(d["object"]["name"], d["object"]["id"]) for d in notOkDatasets],
),
)
scripts/api/history_create_history.py
+2 -2
View File
@@ -8,11 +8,11 @@ from common import submit
try:
assert sys.argv[2]
except IndexError:
print('usage: %s key url [name] ' % os.path.basename(sys.argv[0]))
print("usage: %s key url [name] " % os.path.basename(sys.argv[0]))
sys.exit(1)
try:
data = {}
data['name'] = sys.argv[3]
data["name"] = sys.argv[3]
except IndexError:
pass
scripts/api/history_delete_history.py
+2 -2
View File
@@ -8,11 +8,11 @@ from common import delete
try:
assert sys.argv[2]
except IndexError:
print('usage: %s key url [purge (true/false)] ' % os.path.basename(sys.argv[0]))
print("usage: %s key url [purge (true/false)] " % os.path.basename(sys.argv[0]))
sys.exit(1)
try:
data = {}
data['purge'] = sys.argv[3]
data["purge"] = sys.argv[3]
except IndexError:
pass
scripts/api/import_library_dataset_to_history.py
+3 -3
View File
@@ -8,10 +8,10 @@ from common import submit
try:
assert sys.argv[3]
data = {}
data['from_ld_id'] = sys.argv[3]
data["from_ld_id"] = sys.argv[3]
except IndexError:
print('usage: %s key url library_file_id' % os.path.basename(sys.argv[0]))
print(' library_file_id is from /api/libraries/<library_id>/contents/<library_file_id>')
print("usage: %s key url library_file_id" % os.path.basename(sys.argv[0]))
print(" library_file_id is from /api/libraries/<library_id>/contents/<library_file_id>")
sys.exit(1)
submit(sys.argv[1], sys.argv[2], data)
scripts/api/import_workflows_from_installed_tool_shed_repository.py
+32 -18
View File
@@ -8,36 +8,46 @@ python ./import_workflows_from_installed_tool_shed_repository.py -a 22be3b -l ht
import argparse
from common import display, submit
from common import (
display,
submit,
)
def clean_url(url):
if url.find('//') > 0:
if url.find("//") > 0:
# We have an url that includes a protocol, something like: http://localhost:9009
items = url.split('//')
return items[1].rstrip('/')
return url.rstrip('/')
items = url.split("//")
return items[1].rstrip("/")
return url.rstrip("/")
def main(options):
api_key = options.api
base_galaxy_url = options.local_url.rstrip('/')
base_tool_shed_url = options.tool_shed_url.rstrip('/')
base_galaxy_url = options.local_url.rstrip("/")
base_tool_shed_url = options.tool_shed_url.rstrip("/")
cleaned_tool_shed_url = clean_url(base_tool_shed_url)
installed_tool_shed_repositories_url = '%s/api/tool_shed_repositories' % base_galaxy_url
installed_tool_shed_repositories_url = "%s/api/tool_shed_repositories" % base_galaxy_url
tool_shed_repository_id = None
installed_tool_shed_repositories = display(api_key, installed_tool_shed_repositories_url, return_formatted=False)
for installed_tool_shed_repository in installed_tool_shed_repositories:
tool_shed = str(installed_tool_shed_repository['tool_shed'])
name = str(installed_tool_shed_repository['name'])
owner = str(installed_tool_shed_repository['owner'])
changeset_revision = str(installed_tool_shed_repository['changeset_revision'])
if tool_shed == cleaned_tool_shed_url and name == options.name and owner == options.owner and changeset_revision == options.changeset_revision:
tool_shed_repository_id = installed_tool_shed_repository['id']
tool_shed = str(installed_tool_shed_repository["tool_shed"])
name = str(installed_tool_shed_repository["name"])
owner = str(installed_tool_shed_repository["owner"])
changeset_revision = str(installed_tool_shed_repository["changeset_revision"])
if (
tool_shed == cleaned_tool_shed_url
and name == options.name
and owner == options.owner
and changeset_revision == options.changeset_revision
):
tool_shed_repository_id = installed_tool_shed_repository["id"]
break
if tool_shed_repository_id:
# Get the list of exported workflows contained in the installed repository.
url = '{}{}'.format(base_galaxy_url, '/api/tool_shed_repositories/%s/exported_workflows' % str(tool_shed_repository_id))
url = "{}{}".format(
base_galaxy_url, "/api/tool_shed_repositories/%s/exported_workflows" % str(tool_shed_repository_id)
)
exported_workflows = display(api_key, url, return_formatted=False)
if exported_workflows:
# Import all of the workflows in the list of exported workflows.
@@ -46,14 +56,18 @@ def main(options):
# data[ 'index' ] = 0
# and change the url to be ~/import_workflow (singular). For example,
# url = '%s%s' % ( base_galaxy_url, '/api/tool_shed_repositories/%s/import_workflow' % str( tool_shed_repository_id ) )
url = '{}{}'.format(base_galaxy_url, '/api/tool_shed_repositories/%s/import_workflows' % str(tool_shed_repository_id))
url = "{}{}".format(
base_galaxy_url, "/api/tool_shed_repositories/%s/import_workflows" % str(tool_shed_repository_id)
)
submit(options.api, url, data)
else:
print("Invalid tool_shed / name / owner / changeset_revision.")
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Import workflows contained in an installed tool shed repository via the Galaxy API.')
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Import workflows contained in an installed tool shed repository via the Galaxy API."
)
parser.add_argument("-a", "--api", dest="api", required=True, help="API Key")
parser.add_argument("-u", "--url", dest="tool_shed_url", required=True, help="Tool Shed URL")
parser.add_argument("-l", "--local", dest="local_url", required=True, help="URL of the galaxy instance.")
scripts/api/install_tool_shed_repositories.py
+47 -24
View File
@@ -24,45 +24,68 @@ from common import submit
def main(options):
"""Collect all user data and install the tools via the Galaxy API."""
data = {}
data['tool_shed_url'] = options.tool_shed_url
data['name'] = options.name
data['owner'] = options.owner
data["tool_shed_url"] = options.tool_shed_url
data["name"] = options.name
data["owner"] = options.owner
if options.changeset_revision:
data['changeset_revision'] = options.changeset_revision
data["changeset_revision"] = options.changeset_revision
else:
# If the changeset_revision is not specified, default to the latest installable revision.
revision_data = {}
revision_data['tool_shed_url'] = options.tool_shed_url.rstrip('/')
revision_data['name'] = options.name
revision_data['owner'] = options.owner
revision_url = '{}{}'.format(options.local_url.rstrip('/'), '/api/tool_shed_repositories/get_latest_installable_revision')
latest_installable_revision = submit(options.api,
revision_url,
revision_data,
return_formatted=False)
data['changeset_revision'] = latest_installable_revision
revision_data["tool_shed_url"] = options.tool_shed_url.rstrip("/")
revision_data["name"] = options.name
revision_data["owner"] = options.owner
revision_url = "{}{}".format(
options.local_url.rstrip("/"), "/api/tool_shed_repositories/get_latest_installable_revision"
)
latest_installable_revision = submit(options.api, revision_url, revision_data, return_formatted=False)
data["changeset_revision"] = latest_installable_revision
if options.tool_panel_section_id:
data['tool_panel_section_id'] = options.tool_panel_section_id
data["tool_panel_section_id"] = options.tool_panel_section_id
elif options.new_tool_panel_section_label:
data['new_tool_panel_section_label'] = options.new_tool_panel_section_label
data["new_tool_panel_section_label"] = options.new_tool_panel_section_label
if options.install_repository_dependencies:
data['install_repository_dependencies'] = options.install_repository_dependencies
data["install_repository_dependencies"] = options.install_repository_dependencies
if options.install_tool_dependencies:
data['install_tool_dependencies'] = options.install_tool_dependencies
submit(options.api, '{}{}'.format(options.local_url.rstrip('/'), '/api/tool_shed_repositories/new/install_repository_revision'), data)
data["install_tool_dependencies"] = options.install_tool_dependencies
submit(
options.api,
"{}{}".format(options.local_url.rstrip("/"), "/api/tool_shed_repositories/new/install_repository_revision"),
data,
)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Installation of tool shed repositories via the Galaxy API.')
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Installation of tool shed repositories via the Galaxy API.")
parser.add_argument("-u", "--url", dest="tool_shed_url", required=True, help="Tool Shed URL")
parser.add_argument("-a", "--api", dest="api", required=True, help="API Key")
parser.add_argument("-l", "--local", dest="local_url", required=True, help="URL of the galaxy instance.")
parser.add_argument("-n", "--name", required=True, help="Repository name.")
parser.add_argument("-o", "--owner", required=True, help="Repository owner.")
parser.add_argument("-r", "--revision", dest="changeset_revision", help="Repository revision.")
parser.add_argument("--panel-section-id", dest="tool_panel_section_id", help="Tool panel section id if you want to add your repository to an existing tool section.")
parser.add_argument("--panel-section-name", dest="new_tool_panel_section_label", help="New tool panel section label. If specified a new tool section will be created.")
parser.add_argument("--repository-deps", dest="install_repository_dependencies", action="store_true", default=False, help="Install repository dependencies. [False]")
parser.add_argument("--tool-deps", dest="install_tool_dependencies", action="store_true", default=False, help="Install tool dependencies. [False]")
parser.add_argument(
"--panel-section-id",
dest="tool_panel_section_id",
help="Tool panel section id if you want to add your repository to an existing tool section.",
)
parser.add_argument(
"--panel-section-name",
dest="new_tool_panel_section_label",
help="New tool panel section label. If specified a new tool section will be created.",
)
parser.add_argument(
"--repository-deps",
dest="install_repository_dependencies",
action="store_true",
default=False,
help="Install repository dependencies. [False]",
)
parser.add_argument(
"--tool-deps",
dest="install_tool_dependencies",
action="store_true",
default=False,
help="Install tool dependencies. [False]",
)
options = parser.parse_args()
main(options)
scripts/api/library_create_folder.py
+6 -6
View File
@@ -7,16 +7,16 @@ from common import submit
try:
data = {}
data['folder_id'] = sys.argv[3]
data['name'] = sys.argv[4]
data['create_type'] = 'folder'
data["folder_id"] = sys.argv[3]
data["name"] = sys.argv[4]
data["create_type"] = "folder"
except IndexError:
print('usage: %s key url folder_id name [description]' % os.path.basename(sys.argv[0]))
print("usage: %s key url folder_id name [description]" % os.path.basename(sys.argv[0]))
sys.exit(1)
try:
data['description'] = sys.argv[5]
data["description"] = sys.argv[5]
except IndexError:
print("Unable to set description; using empty description in its place")
data['description'] = ''
data["description"] = ""
submit(sys.argv[1], sys.argv[2], data)
scripts/api/library_create_library.py
+4 -4
View File
@@ -7,13 +7,13 @@ from common import submit
try:
data = {}
data['name'] = sys.argv[3]
data["name"] = sys.argv[3]
except IndexError:
print('usage: %s key url name [description] [synopsys]' % os.path.basename(sys.argv[0]))
print("usage: %s key url name [description] [synopsys]" % os.path.basename(sys.argv[0]))
sys.exit(1)
try:
data['description'] = sys.argv[4]
data['synopsis'] = sys.argv[5]
data["description"] = sys.argv[4]
data["synopsis"] = sys.argv[5]
except IndexError:
pass
scripts/api/library_upload_dir.py
+39 -30
View File
@@ -8,9 +8,7 @@ from bioblend import galaxy
class Uploader:
def __init__(self, url, api, library_id, folder_id, should_link,
non_local):
def __init__(self, url, api, library_id, folder_id, should_link, non_local):
self.gi = galaxy.GalaxyInstance(url=url, key=api)
self.library_id = library_id
self.folder_id = folder_id
@@ -35,22 +33,21 @@ class Uploader:
"""
existing = self.gi.libraries.show_library(self.library_id, contents=True)
uploading_to = [x for x in existing if x['id'] == self.folder_id]
uploading_to = [x for x in existing if x["id"] == self.folder_id]
if len(uploading_to) == 0:
raise Exception("Unknown folder [%s] in library [%s]" %
(self.folder_id, self.library_id))
raise Exception("Unknown folder [%s] in library [%s]" % (self.folder_id, self.library_id))
else:
uploading_to = uploading_to[0]
for x in existing:
# We only care if it's a subdirectory of where we're uploading to
if not x['name'].startswith(uploading_to['name']):
if not x["name"].startswith(uploading_to["name"]):
continue
name_part = x['name'].split(uploading_to['name'], 1)[-1]
if name_part.startswith('/'):
name_part = x["name"].split(uploading_to["name"], 1)[-1]
if name_part.startswith("/"):
name_part = name_part[1:]
self.memo_path[name_part] = x['id']
self.memo_path[name_part] = x["id"]
def memoized_path(self, path_parts, base_folder=None):
"""Get the folder ID for a given folder path specified by path_parts.
@@ -65,21 +62,21 @@ class Uploader:
base_folder = self.folder_id
dropped_prefix = []
fk = '/'.join(path_parts)
fk = "/".join(path_parts)
if fk in self.memo_path:
return self.memo_path[fk]
else:
for i in reversed(range(len(path_parts))):
fk = '/'.join(path_parts[0:i + 1])
fk = "/".join(path_parts[0 : i + 1])
if fk in self.memo_path:
dropped_prefix = path_parts[0:i + 1]
path_parts = path_parts[i + 1:]
dropped_prefix = path_parts[0 : i + 1]
path_parts = path_parts[i + 1 :]
base_folder = self.memo_path[fk]
break
nfk = []
for i in range(len(path_parts)):
nfk.append('/'.join(list(dropped_prefix) + list(path_parts[0:i + 1])))
nfk.append("/".join(list(dropped_prefix) + list(path_parts[0 : i + 1])))
# Recursively create the path from our base_folder starting points,
# getting the IDs of each folder per path component
@@ -99,19 +96,19 @@ class Uploader:
return ids
else:
pf = self.gi.libraries.create_folder(self.library_id, path_parts[0], base_folder_id=parent_folder_id)
ids.append(pf[0]['id'])
return self.recursively_build_path(path_parts[1:], pf[0]['id'], ids=ids)
ids.append(pf[0]["id"])
return self.recursively_build_path(path_parts[1:], pf[0]["id"], ids=ids)
# http://stackoverflow.com/questions/13505819/python-split-path-recursively/13505966#13505966
def rec_split(self, s):
if s == '/':
if s == "/":
return ()
rest, tail = os.path.split(s)
if tail == '.':
if tail == ".":
return ()
if rest == '':
return tail,
if rest == "":
return (tail,)
return self.rec_split(rest) + (tail,)
def upload(self):
@@ -131,7 +128,7 @@ class Uploader:
# So that we can check if it really needs to be uploaded.
already_uploaded = memo_key in self.memo_path.keys()
fid = self.memoized_path(basepath, base_folder=self.folder_id)
print(f'[{idx + 1}/{len(all_files)}] {fid}/{fname} uploaded={already_uploaded}')
print(f"[{idx + 1}/{len(all_files)}] {fid}/{fname} uploaded={already_uploaded}")
if not already_uploaded:
if self.non_local:
@@ -145,22 +142,34 @@ class Uploader:
self.library_id,
os.path.join(dirName, fname),
folder_id=fid,
link_data_only='link_to_files' if self.should_link else 'copy_files',
link_data_only="link_to_files" if self.should_link else "copy_files",
)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Upload a directory into a data library')
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Upload a directory into a data library")
parser.add_argument("-u", "--url", dest="url", required=True, help="Galaxy URL")
parser.add_argument("-a", "--api", dest="api", required=True, help="API Key")
parser.add_argument("-l", "--lib", dest="library_id", required=True, help="Library ID")
parser.add_argument("-f", "--folder", dest="folder_id", help="Folder ID. If not specified, will go to root of library.")
parser.add_argument(
"-f", "--folder", dest="folder_id", help="Folder ID. If not specified, will go to root of library."
)
parser.add_argument("--nonlocal", dest="non_local", action="store_true", default=False,
help="Set this flag if you are NOT running this script on your Galaxy head node with access to the full filesystem")
parser.add_argument("--link", dest="should_link", action="store_true", default=False,
help="Link datasets only, do not upload to Galaxy. ONLY Avaialble if you run 'locally' relative to your Galaxy head node/filesystem ")
parser.add_argument(
"--nonlocal",
dest="non_local",
action="store_true",
default=False,
help="Set this flag if you are NOT running this script on your Galaxy head node with access to the full filesystem",
)
parser.add_argument(
"--link",
dest="should_link",
action="store_true",
default=False,
help="Link datasets only, do not upload to Galaxy. ONLY Avaialble if you run 'locally' relative to your Galaxy head node/filesystem ",
)
args = parser.parse_args()
u = Uploader(**vars(args))
scripts/api/library_upload_from_import_dir.py
+7 -7
View File
@@ -11,14 +11,14 @@ from common import submit
try:
data = {}
data['folder_id'] = sys.argv[3]
data['file_type'] = sys.argv[4]
data['server_dir'] = sys.argv[5]
data['dbkey'] = sys.argv[6]
data['upload_option'] = 'upload_directory'
data['create_type'] = 'file'
data["folder_id"] = sys.argv[3]
data["file_type"] = sys.argv[4]
data["server_dir"] = sys.argv[5]
data["dbkey"] = sys.argv[6]
data["upload_option"] = "upload_directory"
data["create_type"] = "file"
except IndexError:
print('usage: %s key url folder_id file_type server_dir dbkey' % os.path.basename(sys.argv[0]))
print("usage: %s key url folder_id file_type server_dir dbkey" % os.path.basename(sys.argv[0]))
sys.exit(1)
submit(sys.argv[1], sys.argv[2], data)
scripts/api/load_data_with_metadata.py
+23 -20
View File
@@ -15,26 +15,29 @@ import json
import os
import sys
from common import display, submit
from common import (
display,
submit,
)
def load_file(fullpath, api_key, api_url, library_id, library_folder_id, uuid_field=None):
data = {}
data['folder_id'] = library_folder_id
data['file_type'] = 'auto'
data['dbkey'] = ''
data['upload_option'] = 'upload_paths'
data['filesystem_paths'] = fullpath
data['create_type'] = 'file'
data['link_data_only'] = 'link_to_files'
data["folder_id"] = library_folder_id
data["file_type"] = "auto"
data["dbkey"] = ""
data["upload_option"] = "upload_paths"
data["filesystem_paths"] = fullpath
data["create_type"] = "file"
data["link_data_only"] = "link_to_files"
handle = open(fullpath + ".json")
smeta = handle.read()
handle.close()
ext_meta = json.loads(smeta)
data['extended_metadata'] = ext_meta
data["extended_metadata"] = ext_meta
if uuid_field is not None and uuid_field in ext_meta:
data['uuid'] = ext_meta[uuid_field]
data["uuid"] = ext_meta[uuid_field]
libset = submit(api_key, api_url + "libraries/%s/contents" % library_id, data, return_formatted=True)
print(libset)
@@ -42,19 +45,19 @@ def load_file(fullpath, api_key, api_url, library_id, library_folder_id, uuid_fi
def main(api_key, api_url, in_folder, data_library, uuid_field=None):
# Find/Create data library with the above name. Assume we're putting datasets in the root folder '/'
libs = display(api_key, api_url + 'libraries', return_formatted=False)
libs = display(api_key, api_url + "libraries", return_formatted=False)
library_id = None
for library in libs:
if library['name'] == data_library:
library_id = library['id']
if library["name"] == data_library:
library_id = library["id"]
if not library_id:
lib_create_data = {'name': data_library}
library = submit(api_key, api_url + 'libraries', lib_create_data, return_formatted=False)
library_id = library['id']
lib_create_data = {"name": data_library}
library = submit(api_key, api_url + "libraries", lib_create_data, return_formatted=False)
library_id = library["id"]
folders = display(api_key, api_url + "libraries/%s/contents" % library_id, return_formatted=False)
for f in folders:
if f['name'] == "/":
library_folder_id = f['id']
if f["name"] == "/":
library_folder_id = f["id"]
if not library_id or not library_folder_id:
print("Failure to configure library destination.")
sys.exit(1)
@@ -72,10 +75,10 @@ def main(api_key, api_url, in_folder, data_library, uuid_field=None):
load_file(fullpath, api_key, api_url, library_id, library_folder_id, uuid_field)
if __name__ == '__main__':
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("api_key", help="API KEY")
parser.add_argument('api_url', help='API URL')
parser.add_argument("api_url", help="API URL")
parser.add_argument("in_folder", help="Input Folder")
parser.add_argument("data_library", help="Data Library")
parser.add_argument("--uuid_field", help="UUID Field", default=None)
scripts/api/repair_tool_shed_repository.py
+31 -21
View File
@@ -8,48 +8,58 @@ Here is a working example of how to use this script to repair a repository insta
import argparse
from common import display, submit
from common import (
display,
submit,
)
def clean_url(url):
if url.find('//') > 0:
if url.find("//") > 0:
# We have an url that includes a protocol, something like: http://localhost:9009
items = url.split('//')
return items[1].rstrip('/')
return url.rstrip('/')
items = url.split("//")
return items[1].rstrip("/")
return url.rstrip("/")
def main(options):
"""Collect all user data and install the tools via the Galaxy API."""
api_key = options.api
base_galaxy_url = options.local_url.rstrip('/')
base_tool_shed_url = options.tool_shed_url.rstrip('/')
base_galaxy_url = options.local_url.rstrip("/")
base_tool_shed_url = options.tool_shed_url.rstrip("/")
cleaned_tool_shed_url = clean_url(base_tool_shed_url)
installed_tool_shed_repositories_url = '{}/api/{}'.format(base_galaxy_url, 'tool_shed_repositories')
installed_tool_shed_repositories_url = "{}/api/{}".format(base_galaxy_url, "tool_shed_repositories")
data = {}
data['tool_shed_url'] = cleaned_tool_shed_url
data['name'] = options.name
data['owner'] = options.owner
data['changeset_revision'] = options.changeset_revision
data["tool_shed_url"] = cleaned_tool_shed_url
data["name"] = options.name
data["owner"] = options.owner
data["changeset_revision"] = options.changeset_revision
tool_shed_repository_id = None
installed_tool_shed_repositories = display(api_key, installed_tool_shed_repositories_url, return_formatted=False)
for installed_tool_shed_repository in installed_tool_shed_repositories:
tool_shed = str(installed_tool_shed_repository['tool_shed'])
name = str(installed_tool_shed_repository['name'])
owner = str(installed_tool_shed_repository['owner'])
changeset_revision = str(installed_tool_shed_repository['changeset_revision'])
if tool_shed == cleaned_tool_shed_url and name == options.name and owner == options.owner and changeset_revision == options.changeset_revision:
tool_shed_repository_id = installed_tool_shed_repository['id']
tool_shed = str(installed_tool_shed_repository["tool_shed"])
name = str(installed_tool_shed_repository["name"])
owner = str(installed_tool_shed_repository["owner"])
changeset_revision = str(installed_tool_shed_repository["changeset_revision"])
if (
tool_shed == cleaned_tool_shed_url
and name == options.name
and owner == options.owner
and changeset_revision == options.changeset_revision
):
tool_shed_repository_id = installed_tool_shed_repository["id"]
break
if tool_shed_repository_id:
url = '{}{}'.format(base_galaxy_url, '/api/tool_shed_repositories/%s/repair_repository_revision' % str(tool_shed_repository_id))
url = "{}{}".format(
base_galaxy_url, "/api/tool_shed_repositories/%s/repair_repository_revision" % str(tool_shed_repository_id)
)
submit(options.api, url, data)
else:
print("Invalid tool_shed / name / owner / changeset_revision.")
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Installation of tool shed repositories via the Galaxy API.')
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Installation of tool shed repositories via the Galaxy API.")
parser.add_argument("-u", "--url", dest="tool_shed_url", required=True, help="Tool Shed URL")
parser.add_argument("-a", "--api", dest="api", required=True, help="API Key")
parser.add_argument("-l", "--local", dest="local_url", required=True, help="URL of the galaxy instance.")
scripts/api/reset_metadata_on_installed_repositories.py
+6 -4
View File
@@ -14,13 +14,15 @@ from common import submit
def main(options):
base_galaxy_url = options.galaxy_url.rstrip('/')
url = '%s/api/tool_shed_repositories/reset_metadata_on_installed_repositories' % base_galaxy_url
base_galaxy_url = options.galaxy_url.rstrip("/")
url = "%s/api/tool_shed_repositories/reset_metadata_on_installed_repositories" % base_galaxy_url
submit(options.api, url, {})
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Reset metadata on all Tool Shed repositories installed into Galaxy via the Galaxy API.')
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Reset metadata on all Tool Shed repositories installed into Galaxy via the Galaxy API."
)
parser.add_argument("-a", "--api", dest="api", required=True, help="API Key")
parser.add_argument("-u", "--url", dest="galaxy_url", required=True, help="Galaxy URL")
options = parser.parse_args()
scripts/api/search.py
+5 -4
View File
@@ -9,7 +9,6 @@ import requests
class RemoteGalaxy:
def __init__(self, url, api_key):
self.url = url
self.api_key = api_key
@@ -17,15 +16,17 @@ class RemoteGalaxy:
def get(self, path):
c_url = self.url + path
params = {}
params['key'] = self.api_key
params["key"] = self.api_key
req = requests.get(c_url, params=params)
return req.json()
def post(self, path, payload):
c_url = self.url + path
params = {}
params['key'] = self.api_key
req = requests.post(c_url, data=json.dumps(payload), params=params, headers={'Content-Type': 'application/json'})
params["key"] = self.api_key
req = requests.post(
c_url, data=json.dumps(payload), params=params, headers={"Content-Type": "application/json"}
)
return req.json()
scripts/api/update.py
+1 -1
View File
@@ -9,7 +9,7 @@ import sys
from common import update
data = {}
for k, v in [kwarg.split('=', 1) for kwarg in sys.argv[3:]]:
for k, v in [kwarg.split("=", 1) for kwarg in sys.argv[3:]]:
data[k] = v
update(sys.argv[1], sys.argv[2], data)
scripts/api/upload_to_history.py
+22 -18
View File
@@ -10,44 +10,48 @@ import sys
try:
import requests
except ImportError:
print("Could not import the requests module. See http://docs.python-requests.org/en/latest/"
+ " or install with 'pip install requests'")
print(
"Could not import the requests module. See http://docs.python-requests.org/en/latest/"
+ " or install with 'pip install requests'"
)
raise
def upload_file(base_url, api_key, history_id, filepath, **kwargs):
full_url = base_url + '/api/tools'
full_url = base_url + "/api/tools"
payload = {
'key': api_key,
'tool_id': 'upload1',
'history_id': history_id,
"key": api_key,
"tool_id": "upload1",
"history_id": history_id,
}
inputs = {
'files_0|NAME': kwargs.get('filename', os.path.basename(filepath)),
'files_0|type': 'upload_dataset',
"files_0|NAME": kwargs.get("filename", os.path.basename(filepath)),
"files_0|type": "upload_dataset",
# TODO: the following doesn't work with tools.py
'dbkey': '?',
'file_type': kwargs.get('file_type', 'auto'),
'ajax_upload': 'true',
"dbkey": "?",
"file_type": kwargs.get("file_type", "auto"),
"ajax_upload": "true",
}
payload['inputs'] = json.dumps(inputs)
payload["inputs"] = json.dumps(inputs)
response = None
with open(filepath, 'rb') as file_to_upload:
files = {'files_0|file_data': file_to_upload}
with open(filepath, "rb") as file_to_upload:
files = {"files_0|file_data": file_to_upload}
response = requests.post(full_url, data=payload, files=files)
return response.json()
if __name__ == '__main__':
if __name__ == "__main__":
if len(sys.argv) < 5:
print("history_upload.py <api key> <galaxy base url> <history id> <filepath to upload>\n"
+ " (where galaxy base url is just the root url where your Galaxy is served; e.g. 'localhost:8080')")
print(
"history_upload.py <api key> <galaxy base url> <history id> <filepath to upload>\n"
+ " (where galaxy base url is just the root url where your Galaxy is served; e.g. 'localhost:8080')"
)
sys.exit(1)
api_key, base_url, history_id, filepath = sys.argv[1:5]
kwargs = dict([kwarg.split('=', 1) for kwarg in sys.argv[5:]])
kwargs = dict([kwarg.split("=", 1) for kwarg in sys.argv[5:]])
response = upload_file(base_url, api_key, history_id, filepath, **kwargs)
print(response, file=sys.stderr)
scripts/api/workflow_delete.py
+2 -2
View File
@@ -17,11 +17,11 @@ from common import delete
try:
assert sys.argv[2]
except IndexError:
print('usage: %s key url [purge (true/false)] ' % os.path.basename(sys.argv[0]))
print("usage: %s key url [purge (true/false)] " % os.path.basename(sys.argv[0]))
sys.exit(1)
try:
data = {}
data['purge'] = sys.argv[3]
data["purge"] = sys.argv[3]
except IndexError:
pass
scripts/api/workflow_execute.py
+7 -7
View File
@@ -15,19 +15,19 @@ from common import submit
def main():
try:
data = {}
data['workflow_id'] = sys.argv[3]
data['history'] = sys.argv[4]
data['ds_map'] = {}
data["workflow_id"] = sys.argv[3]
data["history"] = sys.argv[4]
data["ds_map"] = {}
# DBTODO If only one input is given, don't require a step
# mapping, just use it for everything?
for v in sys.argv[5:]:
step, src, ds_id = v.split('=')
data['ds_map'][step] = {'src': src, 'id': ds_id}
step, src, ds_id = v.split("=")
data["ds_map"][step] = {"src": src, "id": ds_id}
except IndexError:
print('usage: %s key url workflow_id history step=src=dataset_id' % os.path.basename(sys.argv[0]))
print("usage: %s key url workflow_id history step=src=dataset_id" % os.path.basename(sys.argv[0]))
sys.exit(1)
submit(sys.argv[1], sys.argv[2], data)
if __name__ == '__main__':
if __name__ == "__main__":
main()
scripts/api/workflow_execute_parameters.py
+10 -10
View File
@@ -20,12 +20,12 @@ def main():
try:
print("workflow_execute:py:")
data = {}
data['workflow_id'] = sys.argv[3]
data['history'] = sys.argv[4]
data['ds_map'] = {}
data["workflow_id"] = sys.argv[3]
data["history"] = sys.argv[4]
data["ds_map"] = {}
# Trying to pass in parameter for my own dictionary
data['parameters'] = {}
data["parameters"] = {}
# DBTODO If only one input is given, don't require a step
# mapping, just use it for everything?
@@ -34,22 +34,22 @@ def main():
print(v)
try:
step, src, ds_id = v.split('=')
data['ds_map'][step] = {'src': src, 'id': ds_id}
step, src, ds_id = v.split("=")
data["ds_map"][step] = {"src": src, "id": ds_id}
except ValueError:
print("VALUE ERROR:")
wtype, wtool, wparam, wvalue = v.split('=')
wtype, wtool, wparam, wvalue = v.split("=")
try:
data['parameters'][wtool] = {'param': wparam, 'value': wvalue}
data["parameters"][wtool] = {"param": wparam, "value": wvalue}
except ValueError:
print("TOOL ID ERROR:")
except IndexError:
print('usage: %s key url workflow_id history step=src=dataset_id' % os.path.basename(sys.argv[0]))
print("usage: %s key url workflow_id history step=src=dataset_id" % os.path.basename(sys.argv[0]))
sys.exit(1)
submit(sys.argv[1], sys.argv[2], data)
if __name__ == '__main__':
if __name__ == "__main__":
main()
scripts/api/workflow_import.py
+4 -4
View File
@@ -17,14 +17,14 @@ def main():
api_url = "%s/api/workflows" % api_base_url
try:
data = {}
data['installed_repository_file'] = sys.argv[3]
data["installed_repository_file"] = sys.argv[3]
if len(sys.argv) > 4 and sys.argv[4] == "--add_to_menu":
data['add_to_menu'] = True
data["add_to_menu"] = True
except IndexError:
print('usage: %s key galaxy_url workflow_file' % os.path.basename(sys.argv[0]))
print("usage: %s key galaxy_url workflow_file" % os.path.basename(sys.argv[0]))
sys.exit(1)
submit(api_key, api_url, data, return_formatted=False)
if __name__ == '__main__':
if __name__ == "__main__":
main()
scripts/api/workflow_import_from_file_rpark.py
+2 -2
View File
@@ -20,12 +20,12 @@ def openWorkflow(in_file):
try:
assert sys.argv[2]
except IndexError:
print('usage: %s key url [name] ' % os.path.basename(sys.argv[0]))
print("usage: %s key url [name] " % os.path.basename(sys.argv[0]))
sys.exit(1)
try:
data = {}
workflow_dict = openWorkflow(sys.argv[3])
data['workflow'] = workflow_dict
data["workflow"] = workflow_dict
except IndexError:
pass
scripts/apply_tags.py
+6 -4
View File
@@ -7,7 +7,6 @@ from bioblend.galaxy import GalaxyInstance
class ApplyTagsHistory:
@classmethod
def __init__(self, galaxy_url, galaxy_api_key, history_id=None):
self.galaxy_url = galaxy_url
@@ -54,7 +53,7 @@ class ApplyTagsHistory:
print("Total datasets: %d. Updating their tags may take a while..." % len(all_datasets))
for dataset in all_datasets:
try:
if dataset["deleted"] is False and dataset["state"] == 'ok':
if dataset["deleted"] is False and dataset["state"] == "ok":
parent_ids = list()
child_dataset_id = dataset["id"]
own_tags[child_dataset_id] = dataset["tags"]
@@ -89,7 +88,9 @@ class ApplyTagsHistory:
parent_dataset_ids = all_parents[dataset_id]
# update history tags for a dataset taking all from its parents if there is a parent
if len(parent_dataset_ids) > 0:
is_updated = self.propagate_tags(history, history_id, parent_dataset_ids, dataset_id, parent_tags, own_tags)
is_updated = self.propagate_tags(
history, history_id, parent_dataset_ids, dataset_id, parent_tags, own_tags
)
if is_updated is True:
count_datasets_updated += 1
print("Tags of %d datasets updated" % count_datasets_updated)
@@ -111,6 +112,7 @@ class ApplyTagsHistory:
recursive_parents.extend(dataset_parents)
for parent in dataset_parents:
find_parent_recursive(parent)
find_parent_recursive(item)
# take unique parents
recursive_parent_ids[item] = list(set(recursive_parents))
@@ -138,7 +140,7 @@ class ApplyTagsHistory:
# find unique tags from all parents
all_tags = set(all_tags)
self_tags_set = set(self_tags)
is_same = (all_tags == self_tags_set)
is_same = all_tags == self_tags_set
# update tags if there are new tags from parents
if is_same is False:
is_subset = all_tags.issubset(self_tags_set)
scripts/auth/pam_auth_helper.py
+5 -5
View File
@@ -11,7 +11,7 @@ TIMEOUT = 5
try:
import pam
except ImportError:
log.debug('PAM auth helper: Could not import pam module')
log.debug("PAM auth helper: Could not import pam module")
sys.exit(1)
@@ -32,10 +32,10 @@ signal.alarm(0)
p_auth = pam.pam()
authenticated = p_auth.authenticate(pam_username, pam_password, service=pam_service)
if authenticated:
log.debug(f'PAM auth helper: authentication successful for {pam_username}')
sys.stdout.write('True\n')
log.debug(f"PAM auth helper: authentication successful for {pam_username}")
sys.stdout.write("True\n")
sys.exit(0)
else:
log.debug(f'PAM auth helper: authentication failed for {pam_username}')
sys.stdout.write('False\n')
log.debug(f"PAM auth helper: authentication failed for {pam_username}")
sys.stdout.write("False\n")
sys.exit(1)
scripts/bootstrap_history.py
+84 -59
View File
@@ -36,12 +36,25 @@ RELEASE_DELTA_MONTHS = 4 # Number of months between releases.
# Uncredit pull requestors... kind of arbitrary at this point.
DEVTEAM = [
"afgane", "dannon", "blankenberg",
"davebx", "martenson", "jmchilton",
"tnabtaf", "natefoo", "jgoecks",
"guerler", "jennaj", "nekrut", "jxtx",
"VJalili", "WilliamHolden", "Nerdinacan",
"ic4f", "mvdbeek", "galaxyproject"
"afgane",
"dannon",
"blankenberg",
"davebx",
"martenson",
"jmchilton",
"tnabtaf",
"natefoo",
"jgoecks",
"guerler",
"jennaj",
"nekrut",
"jxtx",
"VJalili",
"WilliamHolden",
"Nerdinacan",
"ic4f",
"mvdbeek",
"galaxyproject",
]
TEMPLATE = """
@@ -79,7 +92,8 @@ Fixes
"""
ANNOUNCE_TEMPLATE = string.Template("""
ANNOUNCE_TEMPLATE = string.Template(
"""
===========================================================
${month_name} 20${year} Galaxy Release (v ${release})
===========================================================
@@ -124,9 +138,11 @@ Release Notes
:start-after: announce_start
.. include:: _thanks.rst
""")
"""
)
ANNOUNCE_USER_TEMPLATE = string.Template("""
ANNOUNCE_USER_TEMPLATE = string.Template(
"""
===========================================================
${month_name} 20${year} Galaxy Release (v ${release})
===========================================================
@@ -176,9 +192,11 @@ Please see the `full release notes <${release}_announce.html>`_ for more details
.. include:: ${release}_prs.rst
.. include:: _thanks.rst
""")
"""
)
NEXT_TEMPLATE = string.Template("""
NEXT_TEMPLATE = string.Template(
"""
:orphan:
===========================================================
@@ -190,13 +208,15 @@ Schedule
===========================================================
* Planned Freeze Date: ${freeze_date}
* Planned Release Date: ${release_date}
""")
"""
)
PRS_TEMPLATE = """
.. github_links
"""
RELEASE_ISSUE_TEMPLATE = string.Template("""
RELEASE_ISSUE_TEMPLATE = string.Template(
"""
- [X] **Prep**
@@ -301,17 +321,20 @@ RELEASE_ISSUE_TEMPLATE = string.Template("""
- [ ] Create release issue for next version ``make release-issue``.
- [ ] Schedule committer meeting to discuss re-alignment of priorities.
- [ ] Close this issue.
""")
"""
)
GROUPPED_TAGS = OrderedDict([
('area/visualizations', 'viz'),
('area/datatypes', 'datatypes'),
('area/tools', 'tools'),
('area/workflows', 'workflows'),
('area/client', 'ui'),
('area/jobs', 'jobs'),
('area/admin', 'admin'),
])
GROUPPED_TAGS = OrderedDict(
[
("area/visualizations", "viz"),
("area/datatypes", "datatypes"),
("area/tools", "tools"),
("area/workflows", "workflows"),
("area/client", "ui"),
("area/jobs", "jobs"),
("area/admin", "admin"),
]
)
# https://api.github.com/repos/galaxyproject/galaxy/pulls?base=dev&state=closed
# https://api.github.com/repos/galaxyproject/galaxy/pulls?base=release_15.07&state=closed
@@ -354,19 +377,11 @@ def do_release(argv):
month_name = calendar.month_name[month]
year = release_name.split(".")[0]
announce_info = ANNOUNCE_TEMPLATE.substitute(
month_name=month_name,
year=year,
release=release_name
)
announce_info = ANNOUNCE_TEMPLATE.substitute(month_name=month_name, year=year, release=release_name)
announce_file = _release_file(release_name + "_announce.rst")
_write_file(announce_file, announce_info, skip_if_exists=True)
announce_user_info = ANNOUNCE_USER_TEMPLATE.substitute(
month_name=month_name,
year=year,
release=release_name
)
announce_user_info = ANNOUNCE_USER_TEMPLATE.substitute(month_name=month_name, year=year, release=release_name)
announce_user_file = _release_file(release_name + "_announce_user.rst")
_write_file(announce_user_file, announce_user_info, skip_if_exists=True)
@@ -374,7 +389,7 @@ def do_release(argv):
seen_prs = set()
try:
with open(prs_file) as fh:
seen_prs = set(re.findall(r'\.\. _Pull Request (\d*): https', fh.read()))
seen_prs = set(re.findall(r"\.\. _Pull Request (\d*): https", fh.read()))
except FileNotFoundError:
pass
_write_file(prs_file, PRS_TEMPLATE, skip_if_exists=True)
@@ -387,7 +402,9 @@ def do_release(argv):
open(next_release_file, "w").write(next_announce)
releases_index = _release_file("index.rst")
releases_index_contents = _read_file(releases_index)
releases_index_contents = releases_index_contents.replace(".. announcements\n", ".. announcements\n " + next_version + "_announce\n")
releases_index_contents = releases_index_contents.replace(
".. announcements\n", ".. announcements\n " + next_version + "_announce\n"
)
_write_file(releases_index, releases_index_contents, skip_if_exists=True)
for pr in _get_prs(release_name):
@@ -398,7 +415,10 @@ def do_release(argv):
"head": pr.head,
"labels": _pr_to_labels(pr),
}
main([argv[0], "--release_file", "%s.rst" % release_name, "--request", as_dict, "pr" + str(pr.number)], seen_prs=seen_prs)
main(
[argv[0], "--release_file", "%s.rst" % release_name, "--request", as_dict, "pr" + str(pr.number)],
seen_prs=seen_prs,
)
def check_release(argv):
@@ -421,10 +441,14 @@ def check_blocking_issues(argv):
release_name = argv[2]
block = 0
github = _github_client()
repo = github.get_repo('galaxyproject/galaxy')
issues = repo.get_issues(state='open')
repo = github.get_repo("galaxyproject/galaxy")
issues = repo.get_issues(state="open")
for issue in issues:
if issue.milestone and issue.milestone.title == release_name and "Publication of Galaxy Release" not in issue.title:
if (
issue.milestone
and issue.milestone.title == release_name
and "Publication of Galaxy Release" not in issue.title
):
print("WARN: Blocking issue| %s" % _issue_to_str(issue))
block = 1
@@ -529,8 +553,8 @@ def main(argv, seen_prs=None):
if newest_release is None:
newest_release = sorted(os.listdir(RELEASES_PATH))[-1]
history_path = os.path.join(RELEASES_PATH, newest_release)
user_announce_path = history_path[0:-len(".rst")] + "_announce_user.rst"
prs_path = history_path[0:-len(".rst")] + "_prs.rst"
user_announce_path = history_path[0 : -len(".rst")] + "_announce_user.rst"
prs_path = history_path[0 : -len(".rst")] + "_prs.rst"
history = _read_file(history_path)
user_announce = _read_file(user_announce_path)
@@ -557,13 +581,13 @@ def main(argv, seen_prs=None):
message = commit["message"]
message = get_first_sentence(message)
elif ident.startswith("pr"):
pull_request = ident[len("pr"):]
pull_request = ident[len("pr") :]
api_url = urljoin(PROJECT_API, "pulls/%s" % pull_request)
if req is None:
req = requests.get(api_url).json()
message = req["title"]
elif ident.startswith("issue"):
issue = ident[len("issue"):]
issue = ident[len("issue") :]
api_url = urljoin(PROJECT_API, "issues/%s" % issue)
if req is None:
req = requests.get(api_url).json()
@@ -576,7 +600,7 @@ def main(argv, seen_prs=None):
owner = None
if ident.startswith("pr"):
pull_request = ident[len("pr"):]
pull_request = ident[len("pr") :]
if pull_request in seen_prs:
to_doc = None
else:
@@ -588,15 +612,16 @@ def main(argv, seen_prs=None):
prs_content = extend_target("github_links", text, prs_content)
if owner:
to_doc += "\n(thanks to `@{} <https://github.com/{}>`__).".format(
owner, owner,
owner,
owner,
)
to_doc += f"\n`Pull Request {pull_request}`_"
labels = None
if req and 'labels' in req:
labels = req['labels']
if req and "labels" in req:
labels = req["labels"]
text_target = _text_target(pull_request, labels=labels)
elif ident.startswith("issue"):
issue = ident[len("issue"):]
issue = ident[len("issue") :]
text = ".. _Issue {0}: {1}/issues/{0}".format(issue, PROJECT_URL)
prs_content = extend_target("github_links", text, prs_content)
to_doc += f"`Issue {issue}`_"
@@ -610,13 +635,13 @@ def main(argv, seen_prs=None):
to_doc = wrap(to_doc)
if text_target is not None:
history = extend_target(text_target, to_doc, history)
if req and req['labels']:
labels = req['labels']
if 'area/datatypes' in labels:
if req and req["labels"]:
labels = req["labels"]
if "area/datatypes" in labels:
user_announce = extend_target("datatypes", to_doc, user_announce)
if 'area/visualizations' in labels:
if "area/visualizations" in labels:
user_announce = extend_target("visualizations", to_doc, user_announce)
if 'area/tools' in labels:
if "area/tools" in labels:
user_announce = extend_target("tools", to_doc, user_announce)
_write_file(history_path, history)
_write_file(prs_path, prs_content)
@@ -652,7 +677,7 @@ def _text_target(pull_request, labels=None):
print(e)
is_bug = is_enhancement = is_feature = is_minor = is_major = is_merge = is_small_enhancement = False
if len(labels) == 0:
print('No labels found for %s' % pr_number)
print("No labels found for %s" % pr_number)
return None
for label_name in labels:
if label_name == "minor":
@@ -675,7 +700,7 @@ def _text_target(pull_request, labels=None):
is_some_kind_of_enhancement = is_enhancement or is_feature or is_small_enhancement
if not(is_bug or is_some_kind_of_enhancement or is_minor or is_merge):
if not (is_bug or is_some_kind_of_enhancement or is_minor or is_merge):
print("No 'kind/*' or 'minor' or 'merge' or 'procedures' label found for %s" % _pr_to_str(pull_request))
text_target = None
@@ -732,7 +757,7 @@ def _releases():
all_files = sorted(os.listdir(RELEASES_PATH))
release_note_file_pattern = re.compile(r"\d+\.\d+.rst")
release_note_files = [f for f in all_files if release_note_file_pattern.match(f)]
return sorted(f.rstrip('.rst') for f in release_note_files)
return sorted(f.rstrip(".rst") for f in release_note_files)
def _github_client():
@@ -765,14 +790,14 @@ def process_sentence(message):
message = re.sub(r"^\s*\[.*\]\s*", r"", message)
# Link issues and pull requests...
issue_url = f"https://github.com/{PROJECT_OWNER}/{PROJECT_NAME}/issues"
message = re.sub(r'#(\d+)', r'`#\1 <%s/\1>`__' % issue_url, message)
message = re.sub(r"#(\d+)", r"`#\1 <%s/\1>`__" % issue_url, message)
return message
def wrap(message):
message = process_sentence(message)
wrapper = textwrap.TextWrapper(initial_indent="* ")
wrapper.subsequent_indent = ' '
wrapper.subsequent_indent = " "
wrapper.width = 160
message_lines = message.splitlines()
first_lines = "\n".join(wrapper.wrap(message_lines[0]))
@@ -782,7 +807,7 @@ def wrap(message):
def next_weekday(d, weekday):
""" Return the next week day (0 for Monday, 6 for Sunday) starting from ``d``. """
"""Return the next week day (0 for Monday, 6 for Sunday) starting from ``d``."""
days_ahead = weekday - d.weekday()
if days_ahead <= 0: # Target day already happened this week
days_ahead += 7
scripts/build_toolbox.py
+29 -33
View File
@@ -5,9 +5,9 @@ from xml.etree import ElementTree as ET
def prettify(elem):
rough_string = ET.tostring(elem, 'utf-8')
rough_string = ET.tostring(elem, "utf-8")
repaired = minidom.parseString(rough_string)
return repaired.toprettyxml(indent=' ')
return repaired.toprettyxml(indent=" ")
# Build a list of all toolconf xml files in the tools directory
@@ -16,9 +16,9 @@ def getfilenamelist(startdir):
for root, _dirs, files in os.walk(startdir):
for fn in files:
fullfn = os.path.join(root, fn)
if fn.endswith('toolconf.xml'):
if fn.endswith("toolconf.xml"):
filenamelist.append(fullfn)
elif fn.endswith('.xml'):
elif fn.endswith(".xml"):
try:
doc = ET.parse(fullfn)
except Exception:
@@ -27,8 +27,8 @@ def getfilenamelist(startdir):
rootelement = doc.getroot()
# Only interpret those 'tool' XML files that have
# the 'section' element.
if rootelement.tag == 'tool':
if rootelement.findall('toolboxposition'):
if rootelement.tag == "tool":
if rootelement.findall("toolboxposition"):
filenamelist.append(fullfn)
else:
print("DBG> tool config does not have a <section>:", fullfn)
@@ -41,10 +41,10 @@ class ToolBox:
self.sectionorders = {}
def add(self, toolelement, toolboxpositionelement):
section = toolboxpositionelement.attrib.get('section', '')
label = toolboxpositionelement.attrib.get('label', '')
order = int(toolboxpositionelement.attrib.get('order', '0'))
sectionorder = int(toolboxpositionelement.attrib.get('sectionorder', '0'))
section = toolboxpositionelement.attrib.get("section", "")
label = toolboxpositionelement.attrib.get("label", "")
order = int(toolboxpositionelement.attrib.get("order", "0"))
sectionorder = int(toolboxpositionelement.attrib.get("sectionorder", "0"))
# If this is the first time we encounter the section, store its order
# number. If we have seen it before, ignore the given order and use
@@ -62,9 +62,9 @@ class ToolBox:
toolkeys.sort()
# Initialize the loop: IDs to zero, current section and label to ''
currentsection = ''
currentsection = ""
sectionnumber = 0
currentlabel = ''
currentlabel = ""
labelnumber = 0
for toolkey in toolkeys:
section = toolkey[3]
@@ -74,12 +74,11 @@ class ToolBox:
if currentsection != section:
currentsection = section
# Start the section with empty label
currentlabel = ''
currentlabel = ""
if section:
sectionnumber += 1
attrib = {'name': section,
'id': "section%d" % sectionnumber}
sectionelement = ET.Element('section', attrib)
attrib = {"name": section, "id": "section%d" % sectionnumber}
sectionelement = ET.Element("section", attrib)
rootelement.append(sectionelement)
currentelement = sectionelement
else:
@@ -91,9 +90,8 @@ class ToolBox:
currentlabel = label
if label:
labelnumber += 1
attrib = {'text': label,
'id': "label%d" % labelnumber}
labelelement = ET.Element('label', attrib)
attrib = {"text": label, "id": "label%d" % labelnumber}
labelelement = ET.Element("label", attrib)
currentelement.append(labelelement)
# Add the tools that are in this place
@@ -112,38 +110,36 @@ def scanfiles(filenamelist):
doc = ET.parse(fn)
root = doc.getroot()
if root.tag == 'tool':
if root.tag == "tool":
toolelements = [root]
else:
toolelements = doc.findall('tool')
toolelements = doc.findall("tool")
for toolelement in toolelements:
# Figure out where the tool XML file is, absolute path.
if 'file' in toolelement.attrib:
if "file" in toolelement.attrib:
# It is mentioned, we need to make it absolute
fileattrib = os.path.join(os.getcwd(),
os.path.dirname(fn),
toolelement.attrib['file'])
fileattrib = os.path.join(os.getcwd(), os.path.dirname(fn), toolelement.attrib["file"])
else:
# It is the current file
fileattrib = os.path.join(os.getcwd(), fn)
# Store the file in the attibutes of the new tool element
attrib = {'file': fileattrib}
attrib = {"file": fileattrib}
# Add the tags into the attributes
tags = toolelement.find('tags')
tags = toolelement.find("tags")
if tags:
tagarray = []
for tag in tags.findall('tag'):
for tag in tags.findall("tag"):
tagarray.append(tag.text)
attrib['tags'] = ",".join(tagarray)
attrib["tags"] = ",".join(tagarray)
else:
print("DBG> No tags in", fn)
# Build the tool element
newtoolelement = ET.Element('tool', attrib)
toolboxpositionelements = toolelement.findall('toolboxposition')
newtoolelement = ET.Element("tool", attrib)
toolboxpositionelements = toolelement.findall("toolboxposition")
if not toolboxpositionelements:
print("DBG> %s has no toolboxposition" % fn)
else:
@@ -154,13 +150,13 @@ def scanfiles(filenamelist):
def assemble():
filenamelist = []
for directorytree in ['tools']:
for directorytree in ["tools"]:
filenamelist.extend(getfilenamelist(directorytree))
filenamelist.sort()
toolbox = scanfiles(filenamelist)
toolboxelement = ET.Element('toolbox')
toolboxelement = ET.Element("toolbox")
toolbox.addElementsTo(toolboxelement)
scripts/celery_shell.py
+5 -4
View File
@@ -7,16 +7,17 @@ ipython -i scripts/celery_shell.py -- -c config/galaxy.yml
import logging
import os
WARNING_MODULES = ['parso', 'asyncio', 'galaxy.datatypes']
WARNING_MODULES = ["parso", "asyncio", "galaxy.datatypes"]
for mod in WARNING_MODULES:
logger = logging.getLogger(mod)
logger.setLevel('WARNING')
logger.setLevel("WARNING")
from scripts.db_shell import config
os.environ['GALAXY_CONFIG_FILE'] = os.environ.get('GALAXY_CONFIG_FILE', config['config_file'])
from galaxy.celery import get_galaxy_app
os.environ["GALAXY_CONFIG_FILE"] = os.environ.get("GALAXY_CONFIG_FILE", config["config_file"])
from galaxy.celery import tasks # noqa: F401
from galaxy.celery import get_galaxy_app
HELP = """
============
scripts/check_model.py
+8 -6
View File
@@ -8,14 +8,17 @@ import os
import sys
from collections import namedtuple
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib")))
from sqlalchemy import create_engine, MetaData
from sqlalchemy import (
create_engine,
MetaData,
)
from galaxy.model import mapping
from galaxy.model.orm.scripts import get_config
IndexTuple = namedtuple('IndexTuple', 'table column_names')
IndexTuple = namedtuple("IndexTuple", "table column_names")
def tuple_from_index(index):
@@ -26,7 +29,6 @@ def tuple_from_index(index):
def find_missing_indexes():
def load_indexes(metadata):
indexes = {}
for t in metadata.tables.values():
@@ -40,7 +42,7 @@ def find_missing_indexes():
mapping_indexes = load_indexes(metadata)
# create EMPTY metadata, then load from database
db_url = get_config(sys.argv)['db_url']
db_url = get_config(sys.argv)["db_url"]
metadata = MetaData(bind=create_engine(db_url))
metadata.reflect()
indexes_in_db = load_indexes(metadata)
@@ -50,7 +52,7 @@ def find_missing_indexes():
return [(mapping_indexes[index], index.table, index.column_names) for index in missing_indexes]
if __name__ == '__main__':
if __name__ == "__main__":
indexes = find_missing_indexes()
if indexes:
print(json.dumps(indexes, indent=4, sort_keys=True))
scripts/check_python.py
+7 -4
View File
@@ -11,19 +11,22 @@ def check_python():
# supported
return
else:
version_string = '.'.join(str(_) for _ in sys.version_info[:3])
msg = """\
version_string = ".".join(str(_) for _ in sys.version_info[:3])
msg = (
"""\
ERROR: Your Python version is: %s
Galaxy is currently supported on Python >=3.7 .
To run Galaxy, please install a supported Python version.
If a supported version is already installed but is not your default,
https://docs.galaxyproject.org/en/latest/admin/python.html contains instructions
on how to force Galaxy to use a different version.""" % version_string
on how to force Galaxy to use a different version."""
% version_string
)
print(msg, file=sys.stderr)
raise Exception(msg)
if __name__ == '__main__':
if __name__ == "__main__":
try:
check_python()
except Exception:
scripts/cleanup_datasets/admin_cleanup_datasets.py
+112 -94
View File
@@ -44,21 +44,30 @@ import shutil
import sys
import time
from collections import defaultdict
from datetime import datetime, timedelta
from datetime import (
datetime,
timedelta,
)
from time import strftime
import sqlalchemy as sa
from mako.template import Template
from sqlalchemy import and_, false
from sqlalchemy import (
and_,
false,
)
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "lib")))
from cleanup_datasets import CleanupDatasetsApplication # noqa: I100
import galaxy.config
import galaxy.model.mapping
import galaxy.util
from galaxy.util.script import app_properties_from_args, populate_config_args
from cleanup_datasets import CleanupDatasetsApplication # noqa: I100
from galaxy.util.script import (
app_properties_from_args,
populate_config_args,
)
log = logging.getLogger()
log.setLevel(logging.INFO)
@@ -74,34 +83,48 @@ def main():
the user will be notified by email using the specified template file.
"""
parser = argparse.ArgumentParser()
parser.add_argument('legacy_config', metavar='CONFIG', type=str,
default=None,
nargs='?',
help='config file (legacy, use --config instead)')
parser.add_argument("-d", "--days", dest="days", action="store",
type=int, help="number of days (60)", default=60)
parser.add_argument("--tool_id", default=None,
help="Text to match against tool_id"
"Default: match all")
parser.add_argument("--template", default=None,
help="Mako Template file to use as email "
"Variables are 'cutoff' for the cutoff in days, "
"'email' for users email and "
"'datasets' which is a list of tuples "
"containing 'dataset' and 'history' names. "
"Default: admin_cleanup_deletion_template.txt")
parser.add_argument("-i", "--info_only", action="store_true",
dest="info_only", help="info about the requested action",
default=False)
parser.add_argument("-e", "--email_only", action="store_true",
dest="email_only", help="Send emails only, don't delete",
default=False)
parser.add_argument("--smtp", default=None,
help="SMTP Server to use to send email. "
"Default: [read from galaxy ini file]")
parser.add_argument("--fromaddr", default=None,
help="From address to use to send email. "
"Default: [read from galaxy ini file]")
parser.add_argument(
"legacy_config",
metavar="CONFIG",
type=str,
default=None,
nargs="?",
help="config file (legacy, use --config instead)",
)
parser.add_argument("-d", "--days", dest="days", action="store", type=int, help="number of days (60)", default=60)
parser.add_argument("--tool_id", default=None, help="Text to match against tool_id" "Default: match all")
parser.add_argument(
"--template",
default=None,
help="Mako Template file to use as email "
"Variables are 'cutoff' for the cutoff in days, "
"'email' for users email and "
"'datasets' which is a list of tuples "
"containing 'dataset' and 'history' names. "
"Default: admin_cleanup_deletion_template.txt",
)
parser.add_argument(
"-i",
"--info_only",
action="store_true",
dest="info_only",
help="info about the requested action",
default=False,
)
parser.add_argument(
"-e",
"--email_only",
action="store_true",
dest="email_only",
help="Send emails only, don't delete",
default=False,
)
parser.add_argument(
"--smtp", default=None, help="SMTP Server to use to send email. " "Default: [read from galaxy ini file]"
)
parser.add_argument(
"--fromaddr", default=None, help="From address to use to send email. " "Default: [read from galaxy ini file]"
)
populate_config_args(parser)
args = parser.parse_args()
@@ -112,23 +135,21 @@ def main():
app_properties = app_properties_from_args(args, legacy_config_override=config_override)
if args.smtp is not None:
app_properties['smtp_server'] = args.smtp
if app_properties.get('smtp_server') is None:
parser.error("SMTP Server must be specified as an option (--smtp) "
"or in the config file (smtp_server)")
app_properties["smtp_server"] = args.smtp
if app_properties.get("smtp_server") is None:
parser.error("SMTP Server must be specified as an option (--smtp) " "or in the config file (smtp_server)")
if args.fromaddr is not None:
app_properties['email_from'] = args.fromaddr
if app_properties.get('email_from') is None:
parser.error("From address must be specified as an option "
"(--fromaddr) or in the config file "
"(email_from)")
app_properties["email_from"] = args.fromaddr
if app_properties.get("email_from") is None:
parser.error(
"From address must be specified as an option " "(--fromaddr) or in the config file " "(email_from)"
)
scriptdir = os.path.dirname(os.path.abspath(__file__))
template_file = args.template
if template_file is None:
default_template = os.path.join(scriptdir,
'admin_cleanup_deletion_template.txt')
default_template = os.path.join(scriptdir, "admin_cleanup_deletion_template.txt")
sample_template_file = "%s.sample" % default_template
if os.path.exists(default_template):
template_file = default_template
@@ -137,10 +158,12 @@ def main():
shutil.copyfile(sample_template_file, default_template)
template_file = default_template
else:
parser.error("Default template (%s) or sample template (%s) not "
"found, please specify template as an option "
"(--template)." % default_template,
sample_template_file)
parser.error(
"Default template (%s) or sample template (%s) not "
"found, please specify template as an option "
"(--template)." % default_template,
sample_template_file,
)
elif not os.path.exists(template_file):
parser.error("Specified template file (%s) not found." % template_file)
@@ -159,37 +182,40 @@ def main():
print("# Sending emails only, not deleting ( --email_only )\n")
administrative_delete_datasets(
app, cutoff_time, args.days, tool_id=args.tool_id,
template_file=template_file, config=config,
email_only=args.email_only, info_only=args.info_only)
app,
cutoff_time,
args.days,
tool_id=args.tool_id,
template_file=template_file,
config=config,
email_only=args.email_only,
info_only=args.info_only,
)
app.shutdown()
sys.exit(0)
def administrative_delete_datasets(app, cutoff_time, cutoff_days,
tool_id, template_file,
config, email_only=False,
info_only=False):
def administrative_delete_datasets(
app, cutoff_time, cutoff_days, tool_id, template_file, config, email_only=False, info_only=False
):
# Marks dataset history association deleted and email users
start = time.time()
# Get HDAs older than cutoff time (ignore tool_id at this point)
# We really only need the id column here, but sqlalchemy barfs when
# trying to select only 1 column
hda_ids_query = sa.select(
(app.model.HistoryDatasetAssociation.table.c.id,
app.model.HistoryDatasetAssociation.table.c.deleted),
(app.model.HistoryDatasetAssociation.table.c.id, app.model.HistoryDatasetAssociation.table.c.deleted),
whereclause=and_(
app.model.Dataset.table.c.deleted == false(),
app.model.HistoryDatasetAssociation.table.c.update_time < cutoff_time,
app.model.HistoryDatasetAssociation.table.c.deleted == false()),
from_obj=[sa.outerjoin(
app.model.Dataset.table,
app.model.HistoryDatasetAssociation.table)])
app.model.HistoryDatasetAssociation.table.c.deleted == false(),
),
from_obj=[sa.outerjoin(app.model.Dataset.table, app.model.HistoryDatasetAssociation.table)],
)
# Add all datasets associated with Histories to our list
hda_ids = []
hda_ids.extend(
[row.id for row in app.sa_session.execute(hda_ids_query)])
hda_ids.extend([row.id for row in app.sa_session.execute(hda_ids_query)])
# Now find the tool_id that generated the dataset (even if it was copied)
tool_matched_ids = []
@@ -206,39 +232,32 @@ def administrative_delete_datasets(app, cutoff_time, cutoff_days,
# Process each of the Dataset objects
for hda_id in hda_ids:
user_query = sa.select(
[app.model.HistoryDatasetAssociation.table,
app.model.History.table,
app.model.User.table],
whereclause=and_(
app.model.HistoryDatasetAssociation.table.c.id == hda_id),
from_obj=[sa.join(app.model.User.table,
app.model.History.table)
.join(app.model.HistoryDatasetAssociation.table)],
use_labels=True)
[app.model.HistoryDatasetAssociation.table, app.model.History.table, app.model.User.table],
whereclause=and_(app.model.HistoryDatasetAssociation.table.c.id == hda_id),
from_obj=[
sa.join(app.model.User.table, app.model.History.table).join(app.model.HistoryDatasetAssociation.table)
],
use_labels=True,
)
for result in app.sa_session.execute(user_query):
user_notifications[result[app.model.User.table.c.email]].append(
(result[app.model.HistoryDatasetAssociation.table.c.name],
result[app.model.History.table.c.name]))
(result[app.model.HistoryDatasetAssociation.table.c.name], result[app.model.History.table.c.name])
)
deleted_instance_count += 1
if not info_only and not email_only:
# Get the HistoryDatasetAssociation objects
hda = app.sa_session.query(
app.model.HistoryDatasetAssociation).get(hda_id)
hda = app.sa_session.query(app.model.HistoryDatasetAssociation).get(hda_id)
if not hda.deleted:
# Mark the HistoryDatasetAssociation as deleted
hda.deleted = True
app.sa_session.add(hda)
print("Marked HistoryDatasetAssociation id %d as "
"deleted" % hda.id)
print("Marked HistoryDatasetAssociation id %d as " "deleted" % hda.id)
app.sa_session.flush()
emailtemplate = Template(filename=template_file)
for (email, dataset_list) in user_notifications.items():
msgtext = emailtemplate.render(email=email,
datasets=dataset_list,
cutoff=cutoff_days)
subject = "Galaxy Server Cleanup " \
"- %d datasets DELETED" % len(dataset_list)
msgtext = emailtemplate.render(email=email, datasets=dataset_list, cutoff=cutoff_days)
subject = "Galaxy Server Cleanup " "- %d datasets DELETED" % len(dataset_list)
fromaddr = config.email_from
print()
print("From: %s" % fromaddr)
@@ -247,8 +266,7 @@ def administrative_delete_datasets(app, cutoff_time, cutoff_days,
print("----------")
print(msgtext)
if not info_only:
galaxy.util.send_mail(fromaddr, email, subject,
msgtext, config)
galaxy.util.send_mail(fromaddr, email, subject, msgtext, config)
stop = time.time()
print()
@@ -261,17 +279,17 @@ def _get_tool_id_for_hda(app, hda_id):
# TODO Some datasets don't seem to have an entry in jtod or a copied_from
if hda_id is None:
return None
job = app.sa_session.query(app.model.Job).\
join(app.model.JobToOutputDatasetAssociation).\
filter(app.model.JobToOutputDatasetAssociation.table.c.dataset_id
== hda_id).first()
job = (
app.sa_session.query(app.model.Job)
.join(app.model.JobToOutputDatasetAssociation)
.filter(app.model.JobToOutputDatasetAssociation.table.c.dataset_id == hda_id)
.first()
)
if job is not None:
return job.tool_id
else:
hda = app.sa_session.query(app.model.HistoryDatasetAssociation).\
get(hda_id)
return _get_tool_id_for_hda(app, hda.
copied_from_history_dataset_association_id)
hda = app.sa_session.query(app.model.HistoryDatasetAssociation).get(hda_id)
return _get_tool_id_for_hda(app, hda.copied_from_history_dataset_association_id)
if __name__ == "__main__":
scripts/cleanup_datasets/cleanup_datasets.py
+264 -104
View File
@@ -6,21 +6,32 @@ import os
import shutil
import sys
import time
from datetime import datetime, timedelta
from datetime import (
datetime,
timedelta,
)
from time import strftime
import sqlalchemy as sa
from sqlalchemy import and_, false, null, true
from sqlalchemy import (
and_,
false,
null,
true,
)
from sqlalchemy.orm import eagerload
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "lib")))
import galaxy.config
from galaxy.datatypes.registry import Registry
from galaxy.exceptions import ObjectNotFound
from galaxy.objectstore import build_object_store_from_config
from galaxy.util import unicodify
from galaxy.util.script import app_properties_from_args, populate_config_args
from galaxy.util.script import (
app_properties_from_args,
populate_config_args,
)
log = logging.getLogger()
log.setLevel(logging.INFO)
@@ -68,20 +79,87 @@ def main():
and Dataset objects may be mapped to History objects via HistoryDatasetAssociation objects.
"""
parser = argparse.ArgumentParser()
parser.add_argument('legacy_config', metavar='CONFIG', type=str,
default=None,
nargs='?',
help='config file (legacy, use --config instead)')
parser.add_argument(
"legacy_config",
metavar="CONFIG",
type=str,
default=None,
nargs="?",
help="config file (legacy, use --config instead)",
)
parser.add_argument("-d", "--days", dest="days", action="store", type=int, help="number of days (60)", default=60)
parser.add_argument("-r", "--remove_from_disk", action="store_true", dest="remove_from_disk", help="remove datasets from disk when purged", default=False)
parser.add_argument("-i", "--info_only", action="store_true", dest="info_only", help="info about the requested action", default=False)
parser.add_argument("-f", "--force_retry", action="store_true", dest="force_retry", help="performs the requested actions, but ignores whether it might have been done before. Useful when -r wasn't used, but should have been", default=False)
parser.add_argument("-1", "--delete_userless_histories", action="store_true", dest="delete_userless_histories", default=False, help="delete userless histories and datasets")
parser.add_argument("-2", "--purge_histories", action="store_true", dest="purge_histories", default=False, help="purge deleted histories")
parser.add_argument("-3", "--purge_datasets", action="store_true", dest="purge_datasets", default=False, help="purge deleted datasets")
parser.add_argument("-4", "--purge_libraries", action="store_true", dest="purge_libraries", default=False, help="purge deleted libraries")
parser.add_argument("-5", "--purge_folders", action="store_true", dest="purge_folders", default=False, help="purge deleted library folders")
parser.add_argument("-6", "--delete_datasets", action="store_true", dest="delete_datasets", default=False, help="mark deletable datasets as deleted and purge associated dataset instances")
parser.add_argument(
"-r",
"--remove_from_disk",
action="store_true",
dest="remove_from_disk",
help="remove datasets from disk when purged",
default=False,
)
parser.add_argument(
"-i",
"--info_only",
action="store_true",
dest="info_only",
help="info about the requested action",
default=False,
)
parser.add_argument(
"-f",
"--force_retry",
action="store_true",
dest="force_retry",
help="performs the requested actions, but ignores whether it might have been done before. Useful when -r wasn't used, but should have been",
default=False,
)
parser.add_argument(
"-1",
"--delete_userless_histories",
action="store_true",
dest="delete_userless_histories",
default=False,
help="delete userless histories and datasets",
)
parser.add_argument(
"-2",
"--purge_histories",
action="store_true",
dest="purge_histories",
default=False,
help="purge deleted histories",
)
parser.add_argument(
"-3",
"--purge_datasets",
action="store_true",
dest="purge_datasets",
default=False,
help="purge deleted datasets",
)
parser.add_argument(
"-4",
"--purge_libraries",
action="store_true",
dest="purge_libraries",
default=False,
help="purge deleted libraries",
)
parser.add_argument(
"-5",
"--purge_folders",
action="store_true",
dest="purge_folders",
default=False,
help="purge deleted library folders",
)
parser.add_argument(
"-6",
"--delete_datasets",
action="store_true",
dest="delete_datasets",
default=False,
help="mark deletable datasets as deleted and purge associated dataset instances",
)
populate_config_args(parser)
args = parser.parse_args()
@@ -89,9 +167,14 @@ def main():
if args.legacy_config:
config_override = args.legacy_config
if not (args.purge_folders ^ args.delete_userless_histories
^ args.purge_libraries ^ args.purge_histories
^ args.purge_datasets ^ args.delete_datasets):
if not (
args.purge_folders
^ args.delete_userless_histories
^ args.purge_libraries
^ args.purge_histories
^ args.purge_datasets
^ args.delete_datasets
):
parser.print_help()
sys.exit(0)
@@ -138,14 +221,17 @@ def delete_userless_histories(app, cutoff_time, info_only=False, force_retry=Fal
history_count = 0
start = time.time()
if force_retry:
histories = app.sa_session.query(app.model.History) \
.filter(and_(app.model.History.table.c.user_id == null(),
app.model.History.update_time < cutoff_time))
histories = app.sa_session.query(app.model.History).filter(
and_(app.model.History.table.c.user_id == null(), app.model.History.update_time < cutoff_time)
)
else:
histories = app.sa_session.query(app.model.History) \
.filter(and_(app.model.History.table.c.user_id == null(),
app.model.History.table.c.deleted == false(),
app.model.History.update_time < cutoff_time))
histories = app.sa_session.query(app.model.History).filter(
and_(
app.model.History.table.c.user_id == null(),
app.model.History.table.c.deleted == false(),
app.model.History.update_time < cutoff_time,
)
)
for history in histories:
if not info_only:
log.info("Deleting history id %d", history.id)
@@ -168,20 +254,29 @@ def purge_histories(app, cutoff_time, remove_from_disk, info_only=False, force_r
history_count = 0
start = time.time()
if force_retry:
histories = app.sa_session.query(app.model.History) \
.filter(and_(app.model.History.table.c.deleted == true(),
app.model.History.update_time < cutoff_time)) \
.options(eagerload('datasets'))
histories = (
app.sa_session.query(app.model.History)
.filter(and_(app.model.History.table.c.deleted == true(), app.model.History.update_time < cutoff_time))
.options(eagerload("datasets"))
)
else:
histories = app.sa_session.query(app.model.History) \
.filter(and_(app.model.History.table.c.deleted == true(),
app.model.History.table.c.purged == false(),
app.model.History.update_time < cutoff_time)) \
.options(eagerload('datasets'))
histories = (
app.sa_session.query(app.model.History)
.filter(
and_(
app.model.History.table.c.deleted == true(),
app.model.History.table.c.purged == false(),
app.model.History.update_time < cutoff_time,
)
)
.options(eagerload("datasets"))
)
for history in histories:
log.info("### Processing history id %d (%s)", history.id, unicodify(history.name))
for dataset_assoc in history.datasets:
_purge_dataset_instance(dataset_assoc, app, remove_from_disk, info_only=info_only) # mark a DatasetInstance as deleted, clear associated files, and mark the Dataset as deleted if it is deletable
_purge_dataset_instance(
dataset_assoc, app, remove_from_disk, info_only=info_only
) # mark a DatasetInstance as deleted, clear associated files, and mark the Dataset as deleted if it is deletable
if not info_only:
# TODO: should the Delete DefaultHistoryPermissions be deleted here? This was incorrectly
# done in the _list_delete() method of the history controller, so copied it here. Not sure
@@ -196,7 +291,7 @@ def purge_histories(app, cutoff_time, remove_from_disk, info_only=False, force_r
log.info("History id %d will be purged (without 'info_only' mode)", history.id)
history_count += 1
stop = time.time()
log.info('Purged %d histories.', history_count)
log.info("Purged %d histories.", history_count)
log.info("Elapsed time: %f", stop - start)
log.info("##########################################")
@@ -210,14 +305,17 @@ def purge_libraries(app, cutoff_time, remove_from_disk, info_only=False, force_r
library_count = 0
start = time.time()
if force_retry:
libraries = app.sa_session.query(app.model.Library) \
.filter(and_(app.model.Library.table.c.deleted == true(),
app.model.Library.table.c.update_time < cutoff_time))
libraries = app.sa_session.query(app.model.Library).filter(
and_(app.model.Library.table.c.deleted == true(), app.model.Library.table.c.update_time < cutoff_time)
)
else:
libraries = app.sa_session.query(app.model.Library) \
.filter(and_(app.model.Library.table.c.deleted == true(),
app.model.Library.table.c.purged == false(),
app.model.Library.table.c.update_time < cutoff_time))
libraries = app.sa_session.query(app.model.Library).filter(
and_(
app.model.Library.table.c.deleted == true(),
app.model.Library.table.c.purged == false(),
app.model.Library.table.c.update_time < cutoff_time,
)
)
for library in libraries:
_purge_folder(library.root_folder, app, remove_from_disk, info_only=info_only)
if not info_only:
@@ -227,7 +325,7 @@ def purge_libraries(app, cutoff_time, remove_from_disk, info_only=False, force_r
app.sa_session.flush()
library_count += 1
stop = time.time()
log.info('# Purged %d libraries .', library_count)
log.info("# Purged %d libraries .", library_count)
log.info("Elapsed time: %f", stop - start)
log.info("##########################################")
@@ -241,19 +339,25 @@ def purge_folders(app, cutoff_time, remove_from_disk, info_only=False, force_ret
folder_count = 0
start = time.time()
if force_retry:
folders = app.sa_session.query(app.model.LibraryFolder) \
.filter(and_(app.model.LibraryFolder.table.c.deleted == true(),
app.model.LibraryFolder.table.c.update_time < cutoff_time))
folders = app.sa_session.query(app.model.LibraryFolder).filter(
and_(
app.model.LibraryFolder.table.c.deleted == true(),
app.model.LibraryFolder.table.c.update_time < cutoff_time,
)
)
else:
folders = app.sa_session.query(app.model.LibraryFolder) \
.filter(and_(app.model.LibraryFolder.table.c.deleted == true(),
app.model.LibraryFolder.table.c.purged == false(),
app.model.LibraryFolder.table.c.update_time < cutoff_time))
folders = app.sa_session.query(app.model.LibraryFolder).filter(
and_(
app.model.LibraryFolder.table.c.deleted == true(),
app.model.LibraryFolder.table.c.purged == false(),
app.model.LibraryFolder.table.c.update_time < cutoff_time,
)
)
for folder in folders:
_purge_folder(folder, app, remove_from_disk, info_only=info_only)
folder_count += 1
stop = time.time()
log.info('# Purged %d folders.', folder_count)
log.info("# Purged %d folders.", folder_count)
log.info("Elapsed time: %f", stop - start)
log.info("##########################################")
@@ -262,30 +366,36 @@ def delete_datasets(app, cutoff_time, remove_from_disk, info_only=False, force_r
# Marks datasets as deleted if associated items are all deleted.
start = time.time()
if force_retry:
history_dataset_ids_query = sa.select((app.model.Dataset.table.c.id,
app.model.Dataset.table.c.state),
whereclause=app.model.HistoryDatasetAssociation.table.c.update_time < cutoff_time,
from_obj=[sa.outerjoin(app.model.Dataset.table,
app.model.HistoryDatasetAssociation.table)])
library_dataset_ids_query = sa.select((app.model.LibraryDataset.table.c.id,
app.model.LibraryDataset.table.c.deleted),
whereclause=app.model.LibraryDataset.table.c.update_time < cutoff_time,
from_obj=[app.model.LibraryDataset.table])
history_dataset_ids_query = sa.select(
(app.model.Dataset.table.c.id, app.model.Dataset.table.c.state),
whereclause=app.model.HistoryDatasetAssociation.table.c.update_time < cutoff_time,
from_obj=[sa.outerjoin(app.model.Dataset.table, app.model.HistoryDatasetAssociation.table)],
)
library_dataset_ids_query = sa.select(
(app.model.LibraryDataset.table.c.id, app.model.LibraryDataset.table.c.deleted),
whereclause=app.model.LibraryDataset.table.c.update_time < cutoff_time,
from_obj=[app.model.LibraryDataset.table],
)
else:
# We really only need the id column here, but sqlalchemy barfs when trying to select only 1 column
history_dataset_ids_query = sa.select((app.model.Dataset.table.c.id,
app.model.Dataset.table.c.state),
whereclause=and_(app.model.Dataset.table.c.deleted == false(),
app.model.HistoryDatasetAssociation.table.c.update_time < cutoff_time,
app.model.HistoryDatasetAssociation.table.c.deleted == true()),
from_obj=[sa.outerjoin(app.model.Dataset.table,
app.model.HistoryDatasetAssociation.table)])
library_dataset_ids_query = sa.select((app.model.LibraryDataset.table.c.id,
app.model.LibraryDataset.table.c.deleted),
whereclause=and_(app.model.LibraryDataset.table.c.deleted == true(),
app.model.LibraryDataset.table.c.purged == false(),
app.model.LibraryDataset.table.c.update_time < cutoff_time),
from_obj=[app.model.LibraryDataset.table])
history_dataset_ids_query = sa.select(
(app.model.Dataset.table.c.id, app.model.Dataset.table.c.state),
whereclause=and_(
app.model.Dataset.table.c.deleted == false(),
app.model.HistoryDatasetAssociation.table.c.update_time < cutoff_time,
app.model.HistoryDatasetAssociation.table.c.deleted == true(),
),
from_obj=[sa.outerjoin(app.model.Dataset.table, app.model.HistoryDatasetAssociation.table)],
)
library_dataset_ids_query = sa.select(
(app.model.LibraryDataset.table.c.id, app.model.LibraryDataset.table.c.deleted),
whereclause=and_(
app.model.LibraryDataset.table.c.deleted == true(),
app.model.LibraryDataset.table.c.purged == false(),
app.model.LibraryDataset.table.c.update_time < cutoff_time,
),
from_obj=[app.model.LibraryDataset.table],
)
deleted_dataset_count = 0
deleted_instance_count = 0
skip = []
@@ -331,7 +441,9 @@ def delete_datasets(app, cutoff_time, remove_from_disk, info_only=False, force_r
skip.append(dataset.id)
log.info("######### Processing dataset id: %d", dataset_id)
if not _dataset_is_deletable(dataset):
log.info("Dataset is not deletable (shared between multiple histories/libraries, at least one is not deleted)")
log.info(
"Dataset is not deletable (shared between multiple histories/libraries, at least one is not deleted)"
)
continue
deleted_dataset_count += 1
for dataset_instance in dataset.history_associations + dataset.library_associations:
@@ -339,7 +451,12 @@ def delete_datasets(app, cutoff_time, remove_from_disk, info_only=False, force_r
_purge_dataset_instance(dataset_instance, app, remove_from_disk, info_only=info_only, is_deletable=True)
deleted_instance_count += 1
stop = time.time()
log.info("Examined %d datasets, marked %d datasets and %d dataset instances (HDA) as deleted", len(skip), deleted_dataset_count, deleted_instance_count)
log.info(
"Examined %d datasets, marked %d datasets and %d dataset instances (HDA) as deleted",
len(skip),
deleted_dataset_count,
deleted_instance_count,
)
log.info("Total elapsed time: %f", stop - start)
log.info("##########################################")
@@ -351,16 +468,22 @@ def purge_datasets(app, cutoff_time, remove_from_disk, info_only=False, force_re
disk_space = 0
start = time.time()
if force_retry:
datasets = app.sa_session.query(app.model.Dataset) \
.filter(and_(app.model.Dataset.table.c.deleted == true(),
app.model.Dataset.table.c.purgable == true(),
app.model.Dataset.table.c.update_time < cutoff_time))
datasets = app.sa_session.query(app.model.Dataset).filter(
and_(
app.model.Dataset.table.c.deleted == true(),
app.model.Dataset.table.c.purgable == true(),
app.model.Dataset.table.c.update_time < cutoff_time,
)
)
else:
datasets = app.sa_session.query(app.model.Dataset) \
.filter(and_(app.model.Dataset.table.c.deleted == true(),
app.model.Dataset.table.c.purgable == true(),
app.model.Dataset.table.c.purged == false(),
app.model.Dataset.table.c.update_time < cutoff_time))
datasets = app.sa_session.query(app.model.Dataset).filter(
and_(
app.model.Dataset.table.c.deleted == true(),
app.model.Dataset.table.c.purgable == true(),
app.model.Dataset.table.c.purged == false(),
app.model.Dataset.table.c.update_time < cutoff_time,
)
)
for dataset in datasets:
file_size = dataset.file_size
_purge_dataset(app, dataset, remove_from_disk, info_only=info_only)
@@ -370,7 +493,7 @@ def purge_datasets(app, cutoff_time, remove_from_disk, info_only=False, force_re
except Exception:
pass
stop = time.time()
log.info('Purged %d datasets', dataset_count)
log.info("Purged %d datasets", dataset_count)
if remove_from_disk:
log.info("Freed disk space: %d", disk_space)
log.info("Elapsed time: %f", stop - start)
@@ -381,24 +504,38 @@ def _purge_dataset_instance(dataset_instance, app, remove_from_disk, info_only=F
# A dataset_instance is either a HDA or an LDDA. Purging a dataset instance marks the instance as deleted,
# and marks the associated dataset as deleted if it is not associated with another active DatsetInstance.
if not info_only:
log.info("Marking as deleted: %s id %d (for dataset id %d)",
dataset_instance.__class__.__name__, dataset_instance.id, dataset_instance.dataset.id)
log.info(
"Marking as deleted: %s id %d (for dataset id %d)",
dataset_instance.__class__.__name__,
dataset_instance.id,
dataset_instance.dataset.id,
)
dataset_instance.mark_deleted()
dataset_instance.clear_associated_files()
app.sa_session.add(dataset_instance)
app.sa_session.flush()
app.sa_session.refresh(dataset_instance.dataset)
else:
log.info("%s id %d (for dataset id %d) will be marked as deleted (without 'info_only' mode)",
dataset_instance.__class__.__name__, dataset_instance.id, dataset_instance.dataset.id)
log.info(
"%s id %d (for dataset id %d) will be marked as deleted (without 'info_only' mode)",
dataset_instance.__class__.__name__,
dataset_instance.id,
dataset_instance.dataset.id,
)
if is_deletable or _dataset_is_deletable(dataset_instance.dataset):
# Calling methods may have already checked _dataset_is_deletable, if so, is_deletable should be True
_delete_dataset(dataset_instance.dataset, app, remove_from_disk, info_only=info_only, is_deletable=is_deletable)
else:
if info_only:
log.info("Not deleting dataset %d, (will be possibly deleted without 'info_only' mode)", dataset_instance.dataset.id)
log.info(
"Not deleting dataset %d, (will be possibly deleted without 'info_only' mode)",
dataset_instance.dataset.id,
)
else:
log.info("Not deleting dataset %d (shared between multiple histories/libraries, at least one not deleted)", dataset_instance.dataset.id)
log.info(
"Not deleting dataset %d (shared between multiple histories/libraries, at least one not deleted)",
dataset_instance.dataset.id,
)
def _dataset_is_deletable(dataset):
@@ -416,27 +553,41 @@ def _delete_dataset(dataset, app, remove_from_disk, info_only=False, is_deletabl
metadata_files = []
# lets create a list of metadata files, then perform actions on them
for hda in dataset.history_associations:
for metadata_file in app.sa_session.query(app.model.MetadataFile) \
.filter(app.model.MetadataFile.table.c.hda_id == hda.id):
for metadata_file in app.sa_session.query(app.model.MetadataFile).filter(
app.model.MetadataFile.table.c.hda_id == hda.id
):
metadata_files.append(metadata_file)
for ldda in dataset.library_associations:
for metadata_file in app.sa_session.query(app.model.MetadataFile) \
.filter(app.model.MetadataFile.table.c.lda_id == ldda.id):
for metadata_file in app.sa_session.query(app.model.MetadataFile).filter(
app.model.MetadataFile.table.c.lda_id == ldda.id
):
metadata_files.append(metadata_file)
for metadata_file in metadata_files:
op_description = "marked as deleted"
if remove_from_disk:
op_description = op_description + " and purged from disk"
if info_only:
log.info("The following metadata files attached to associations of Dataset '%d' will be %s (without 'info_only' mode):", dataset.id, op_description)
log.info(
"The following metadata files attached to associations of Dataset '%d' will be %s (without 'info_only' mode):",
dataset.id,
op_description,
)
else:
log.info("The following metadata files attached to associations of Dataset '%d' have been %s:", dataset.id, op_description)
log.info(
"The following metadata files attached to associations of Dataset '%d' have been %s:",
dataset.id,
op_description,
)
if remove_from_disk:
try:
log.info("Removing disk file %s", metadata_file.file_name)
os.unlink(metadata_file.file_name)
except Exception as e:
log.info("Error, exception: %s caught attempting to purge metadata file %s\n", unicodify(e), metadata_file.file_name)
log.info(
"Error, exception: %s caught attempting to purge metadata file %s\n",
unicodify(e),
metadata_file.file_name,
)
metadata_file.purged = True
app.sa_session.add(metadata_file)
app.sa_session.flush()
@@ -465,7 +616,9 @@ def _purge_dataset(app, dataset, remove_from_disk, info_only=False):
os.unlink(dataset.file_name)
# Remove associated extra files from disk if they exist
if dataset.extra_files_path and os.path.exists(dataset.extra_files_path):
shutil.rmtree(dataset.extra_files_path) # we need to delete the directory and its contents; os.unlink would always fail on a directory
shutil.rmtree(
dataset.extra_files_path
) # we need to delete the directory and its contents; os.unlink would always fail on a directory
usage_users = []
for hda in dataset.history_associations:
if not hda.purged:
@@ -482,7 +635,11 @@ def _purge_dataset(app, dataset, remove_from_disk, info_only=False):
else:
log.info("Dataset %d will be purged (without 'info_only' mode)", dataset.id)
else:
log.info("This dataset (%d) is not purgable, the file (%s) will not be removed.\n", dataset.id, dataset.file_name)
log.info(
"This dataset (%d) is not purgable, the file (%s) will not be removed.\n",
dataset.id,
dataset.file_name,
)
except OSError as exc:
log.error("Error, dataset file has already been removed: %s", unicodify(exc))
log.error("Purging dataset id %d", dataset.id)
@@ -503,7 +660,9 @@ def _purge_folder(folder, app, remove_from_disk, info_only=False):
log.info("Deleting library dataset id %d", ld.id)
ld.deleted = True
for ldda in [ld.library_dataset_dataset_association] + ld.expired_datasets:
_purge_dataset_instance(ldda, app, remove_from_disk, info_only=info_only) # mark a DatasetInstance as deleted, clear associated files, and mark the Dataset as deleted if it is deletable
_purge_dataset_instance(
ldda, app, remove_from_disk, info_only=info_only
) # mark a DatasetInstance as deleted, clear associated files, and mark the Dataset as deleted if it is deletable
for sub_folder in folder.folders:
_purge_folder(sub_folder, app, remove_from_disk, info_only=info_only)
if not info_only:
@@ -516,6 +675,7 @@ def _purge_folder(folder, app, remove_from_disk, info_only=False):
class CleanupDatasetsApplication:
"""Encapsulates the state of a Universe application"""
def __init__(self, config):
self.object_store = build_object_store_from_config(config)
# Setup the database engine and ORM
scripts/cleanup_datasets/pgcleanup.py
+138 -133
View File
@@ -21,7 +21,7 @@ from psycopg2.extras import NamedTupleCursor
from sqlalchemy.engine.url import make_url
galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
sys.path.insert(1, os.path.join(galaxy_root, 'lib'))
sys.path.insert(1, os.path.join(galaxy_root, "lib"))
import galaxy.config
from galaxy.exceptions import ObjectNotFound
@@ -32,7 +32,7 @@ from galaxy.util.script import (
set_log_handler,
)
DEFAULT_LOG_DIR = os.path.join(galaxy_root, 'scripts', 'cleanup_datasets')
DEFAULT_LOG_DIR = os.path.join(galaxy_root, "scripts", "cleanup_datasets")
log = logging.getLogger(__name__)
@@ -51,7 +51,7 @@ class LevelFormatter(logging.Formatter):
fmt = self.warn_fmt
else:
fmt = self.def_fmt
if hasattr(self, '_style'): # py3
if hasattr(self, "_style"): # py3
self._style._fmt = fmt
else:
self._fmt = fmt
@@ -74,6 +74,7 @@ class Action:
Generally you should set at least ``_action_sql`` in subclasses (although it's possible to just override ``sql``
directly.)
"""
update_time_sql = ", update_time = NOW() AT TIME ZONE 'utc'"
force_retry_sql = " AND NOT purged"
primary_key = None
@@ -86,18 +87,18 @@ class Action:
@classmethod
def name_c(cls):
# special case - for more complex stuff you can always implement name_c() on subclasses
clsname = cls.__name__.replace('HDA', 'Hda')
clsname = cls.__name__.replace("HDA", "Hda")
actname = [clsname[0].lower()]
for c in clsname[1:]:
if c in string.ascii_uppercase:
c = '_' + c.lower()
c = "_" + c.lower()
actname.append(c)
return ''.join(actname)
return "".join(actname)
@classmethod
def doc_iter(cls):
for line in cls.__doc__.splitlines():
yield line.replace(' ', '', 4)
yield line.replace(" ", "", 4)
def __init__(self, app):
self._log_dir = app.args.log_dir
@@ -129,24 +130,24 @@ class Action:
if self._log_file:
logf = os.path.join(self._log_dir, self._log_file)
else:
logf = os.path.join(self._log_dir, self.name + '.log')
logf = os.path.join(self._log_dir, self.name + ".log")
if self._dry_run:
log.info('--dry-run specified, logging changes to stderr instead of log file: %s' % logf)
log.info("--dry-run specified, logging changes to stderr instead of log file: %s" % logf)
h = set_log_handler()
else:
log.info('Opening log file: %s' % logf)
log.info("Opening log file: %s" % logf)
h = set_log_handler(filename=logf)
h.setLevel(logging.DEBUG if self._debug else logging.INFO)
h.setFormatter(LevelFormatter())
self.__log = logging.getLogger(self.name)
self.__log.addHandler(h)
self.__log.propagate = False
m = ('==== Log opened: %s ' % datetime.datetime.now().isoformat()).ljust(72, '=')
m = ("==== Log opened: %s " % datetime.datetime.now().isoformat()).ljust(72, "=")
self.__log.info(m)
self.__log.info(f'Epoch time for this action: {self._epoch_time}')
self.__log.info(f"Epoch time for this action: {self._epoch_time}")
def __close_log(self):
m = ('==== Log closed: %s ' % datetime.datetime.now().isoformat()).ljust(72, '=')
m = ("==== Log closed: %s " % datetime.datetime.now().isoformat()).ljust(72, "=")
self.log.info(m)
self.__log = None
@@ -197,8 +198,8 @@ class Action:
@property
def sql_args(self):
args = self._action_sql_args.copy()
if 'days' not in args:
args['days'] = self._days
if "days" not in args:
args["days"] = self._days
return args
def _collect_row_results(self, row, results, primary_key):
@@ -214,11 +215,11 @@ class Action:
def _log_results(self, results, primary_key):
for primary in sorted(results.keys()):
self.log.info(f'{primary_key}: {primary}')
self.log.info(f"{primary_key}: {primary}")
for causal, s in zip(self.causals, results[primary]):
for r in sorted(s):
secondaries = ', '.join('%s: %s' % x for x in zip(causal[1:], r[1:]))
self.log.info(f'{causal[0]} {r[0]} caused {secondaries}')
secondaries = ", ".join("%s: %s" % x for x in zip(causal[1:], r[1:]))
self.log.info(f"{causal[0]} {r[0]} caused {secondaries}")
def handle_results(self, cur):
results = {}
@@ -244,11 +245,11 @@ class Action:
class RemovesObjects:
"""Base class for mixins that remove objects from object stores.
"""
"""Base class for mixins that remove objects from object stores."""
def _init(self):
self.objects_to_remove = set()
log.info('Initializing object store for action %s', self.name)
log.info("Initializing object store for action %s", self.name)
self.object_store = build_object_store_from_config(self._config)
self._register_row_method(self.collect_removed_object_info)
self._register_post_method(self.remove_objects)
@@ -273,13 +274,13 @@ class RemovesObjects:
# identifier" which in the case of Disk would be the path
if not check_exists or self.object_store.exists(object_to_remove, **object_store_kwargs):
filename = self.object_store.get_filename(object_to_remove, **object_store_kwargs)
self.log.info('removing %s at: %s', object_to_remove, filename)
self.log.info("removing %s at: %s", object_to_remove, filename)
if not self._dry_run:
self.object_store.delete(object_to_remove, entire_dir=entire_dir, **object_store_kwargs)
except ObjectNotFound as e:
[log_.warning('object store failure: %s: %s', object_to_remove, e) for log_ in loggers]
[log_.warning("object store failure: %s: %s", object_to_remove, e) for log_ in loggers]
except Exception as e:
[log_.error('delete failure: %s: %s', object_to_remove, e) for log_ in loggers]
[log_.error("delete failure: %s: %s", object_to_remove, e) for log_ in loggers]
def remove_object(self, object_to_remove):
raise NotImplementedError()
@@ -296,6 +297,7 @@ class PurgesHDAs:
To use, place ``{purge_hda_dependencies_sql}`` somewhere in your CTEs after a ``purged_hda_ids`` CTE returning HDA
ids. If you have additional CTEs after the template point, be sure to append a ``,``.
"""
_purge_hda_dependencies_sql = """deleted_metadata_file_ids
AS ( UPDATE metadata_file
SET deleted = true{update_time_sql}
@@ -353,6 +355,7 @@ class RequiresDiskUsageRecalculation:
To use, ensure your query returns a ``recalculate_disk_usage_user_id`` column.
"""
def _init(self):
self.__recalculate_disk_usage_user_ids = set()
self._register_row_method(self.collect_recalculate_disk_usage_user_id)
@@ -371,7 +374,7 @@ class RequiresDiskUsageRecalculation:
This could probably be done more efficiently.
"""
log.info('Recalculating disk usage for users whose data were purged')
log.info("Recalculating disk usage for users whose data were purged")
for user_id in sorted(self.__recalculate_disk_usage_user_ids):
# TODO: h.purged = false should be unnecessary once all hdas in purged histories are purged.
sql = """
@@ -392,11 +395,11 @@ class RequiresDiskUsageRecalculation:
WHERE id = %(user_id)s
RETURNING disk_usage;
"""
args = {'user_id': user_id}
args = {"user_id": user_id}
cur = self._update(sql, args, add_event=False)
for row in cur:
# disk_usage might be None (e.g. user has purged all data)
self.log.info('recalculate_disk_usage user_id %i to %s bytes' % (user_id, row.disk_usage))
self.log.info("recalculate_disk_usage user_id %i to %s bytes" % (user_id, row.disk_usage))
class RemovesMetadataFiles(RemovesObjects):
@@ -404,16 +407,15 @@ class RemovesMetadataFiles(RemovesObjects):
To use, ensure your query returns ``deleted_metadata_file_id`` and ``object_store_id`` columns.
"""
object_class = namedtuple('MetadataFile', ['id', 'object_store_id'])
id_column = 'deleted_metadata_file_id'
object_class = namedtuple("MetadataFile", ["id", "object_store_id"])
id_column = "deleted_metadata_file_id"
def remove_object(self, metadata_file):
self.remove_from_object_store(
metadata_file,
dict(
extra_dir='_metadata_files',
extra_dir_at_root=True,
alt_name="metadata_%d.dat" % metadata_file.id))
dict(extra_dir="_metadata_files", extra_dir_at_root=True, alt_name="metadata_%d.dat" % metadata_file.id),
)
class RemovesDatasets(RemovesObjects):
@@ -421,18 +423,15 @@ class RemovesDatasets(RemovesObjects):
To use, ensure your query returns ``purged_dataset_id`` and ``object_store_id`` columns.
"""
object_class = namedtuple('Dataset', ['id', 'object_store_id'])
id_column = 'purged_dataset_id'
object_class = namedtuple("Dataset", ["id", "object_store_id"])
id_column = "purged_dataset_id"
def remove_object(self, dataset):
self.remove_from_object_store(dataset, dict())
self.remove_from_object_store(
dataset,
dict(
dir_only=True,
extra_dir="dataset_%d_files" % dataset.id),
entire_dir=True,
check_exists=True)
dataset, dict(dir_only=True, extra_dir="dataset_%d_files" % dataset.id), entire_dir=True, check_exists=True
)
#
@@ -445,6 +444,7 @@ class UpdateHDAPurgedFlag(Action):
The old cleanup script does not mark HistoryDatasetAssociations as purged when deleted Histories
are purged. This action can be used to rectify that situation.
"""
# update_time is intentionally left unmodified.
_action_sql = """
WITH purged_hda_ids
@@ -471,6 +471,7 @@ class DeleteUserlessHistories(Action):
- Mark deleted all "anonymous" Histories (not owned by a registered user) that are older than
the specified number of days.
"""
_action_sql = """
WITH deleted_history_ids
AS ( UPDATE history
@@ -495,6 +496,7 @@ class DeleteInactiveUsers(Action):
- Mark deleted all users that are older than the specified number of days.
- Mark deleted (state = 'deleted') all Jobs whose user_ids are deleted in this step.
"""
force_retry_sql = " AND NOT deleted"
_action_sql = """
WITH deleted_user_ids
@@ -523,9 +525,7 @@ class DeleteInactiveUsers(Action):
ON deleted_job_ids.user_id = deleted_user_ids.id
ORDER BY deleted_user_ids.id
"""
causals = (
('purged_user_id', 'purged_job_id'),
)
causals = (("purged_user_id", "purged_job_id"),)
class PurgeDeletedUsers(PurgesHDAs, RemovesMetadataFiles, Action):
@@ -539,6 +539,7 @@ class PurgeDeletedUsers(PurgesHDAs, RemovesMetadataFiles, Action):
ROLE.
- Delete all UserAddresses whose user_ids are purged in this step.
"""
_action_sql = """
WITH purged_user_ids
AS ( UPDATE galaxy_user
@@ -629,13 +630,13 @@ class PurgeDeletedUsers(PurgesHDAs, RemovesMetadataFiles, Action):
ORDER BY purged_user_ids.id
"""
causals = (
('purged_user_id', 'purged_history_id'),
('purged_history_id', 'purged_hda_id'),
('purged_hda_id', 'deleted_metadata_file_id', 'object_store_id'),
('purged_hda_id', 'deleted_icda_id', 'deleted_icda_hda_id'),
('purged_user_id', 'deleted_uga_id'),
('purged_user_id', 'deleted_ura_id'),
('purged_user_id', 'deleted_ua_id'),
("purged_user_id", "purged_history_id"),
("purged_history_id", "purged_hda_id"),
("purged_hda_id", "deleted_metadata_file_id", "object_store_id"),
("purged_hda_id", "deleted_icda_id", "deleted_icda_hda_id"),
("purged_user_id", "deleted_uga_id"),
("purged_user_id", "deleted_ura_id"),
("purged_user_id", "deleted_ua_id"),
)
def _init(self):
@@ -650,16 +651,16 @@ class PurgeDeletedUsers(PurgesHDAs, RemovesMetadataFiles, Action):
def zero_disk_usage(self):
if not self.__zero_disk_usage_user_ids:
return
log.info('Zeroing disk usage for users who were purged')
log.info("Zeroing disk usage for users who were purged")
sql = """
UPDATE galaxy_user
SET disk_usage = 0
WHERE id IN %(user_ids)s
"""
user_ids = sorted(self.__zero_disk_usage_user_ids)
args = {'user_ids': tuple(user_ids)}
args = {"user_ids": tuple(user_ids)}
self._update(sql, args, add_event=False)
self.log.info('zero_disk_usage user_ids: %s', ' '.join(str(i) for i in user_ids))
self.log.info("zero_disk_usage user_ids: %s", " ".join(str(i) for i in user_ids))
class PurgeDeletedUsersGDPR(PurgesHDAs, RemovesMetadataFiles, Action):
@@ -670,6 +671,7 @@ class PurgeDeletedUsersGDPR(PurgesHDAs, RemovesMetadataFiles, Action):
NOTE: Your database must have the pgcrypto extension installed e.g. with:
CREATE EXTENSION IF NOT EXISTS pgcrypto;
"""
_action_sql = """
WITH purged_user_ids
AS ( UPDATE galaxy_user
@@ -763,18 +765,18 @@ class PurgeDeletedUsersGDPR(PurgesHDAs, RemovesMetadataFiles, Action):
ORDER BY purged_user_ids.id
"""
causals = (
('purged_user_id', 'purged_history_id'),
('purged_history_id', 'purged_hda_id'),
('purged_hda_id', 'deleted_metadata_file_id', 'object_store_id'),
('purged_hda_id', 'deleted_icda_id', 'deleted_icda_hda_id'),
('purged_user_id', 'deleted_uga_id'),
('purged_user_id', 'deleted_ura_id'),
('purged_user_id', 'deleted_ua_id'),
("purged_user_id", "purged_history_id"),
("purged_history_id", "purged_hda_id"),
("purged_hda_id", "deleted_metadata_file_id", "object_store_id"),
("purged_hda_id", "deleted_icda_id", "deleted_icda_hda_id"),
("purged_user_id", "deleted_uga_id"),
("purged_user_id", "deleted_ura_id"),
("purged_user_id", "deleted_ua_id"),
)
@classmethod
def name_c(cls):
return 'purge_deleted_users_gdpr'
return "purge_deleted_users_gdpr"
class PurgeDeletedHDAs(PurgesHDAs, RemovesMetadataFiles, RequiresDiskUsageRecalculation, Action):
@@ -787,6 +789,7 @@ class PurgeDeletedHDAs(PurgesHDAs, RemovesMetadataFiles, RequiresDiskUsageRecalc
- Mark purged all HistoryDatasetAssociations for which an ImplicitlyConvertedDatasetAssociation
with matching hda_id is deleted in this step.
"""
_action_sql = """
WITH purged_hda_ids
AS ( UPDATE history_dataset_association
@@ -817,8 +820,8 @@ class PurgeDeletedHDAs(PurgesHDAs, RemovesMetadataFiles, RequiresDiskUsageRecalc
ORDER BY purged_hda_ids.id
"""
causals = (
('purged_hda_id', 'deleted_metadata_file_id', 'object_store_id'),
('purged_hda_id', 'deleted_icda_id', 'deleted_icda_hda_id'),
("purged_hda_id", "deleted_metadata_file_id", "object_store_id"),
("purged_hda_id", "deleted_icda_id", "deleted_icda_hda_id"),
)
@@ -826,6 +829,7 @@ class PurgeHistorylessHDAs(PurgesHDAs, RemovesMetadataFiles, RequiresDiskUsageRe
"""
- Mark purged all HistoryDatasetAssociations whose history_id is null.
"""
_action_sql = """
WITH purged_hda_ids
AS ( UPDATE history_dataset_association
@@ -852,8 +856,8 @@ class PurgeHistorylessHDAs(PurgesHDAs, RemovesMetadataFiles, RequiresDiskUsageRe
ORDER BY purged_hda_ids.id
"""
causals = (
('purged_hda_id', 'deleted_metadata_file_id', 'object_store_id'),
('purged_hda_id', 'deleted_icda_id', 'deleted_icda_hda_id'),
("purged_hda_id", "deleted_metadata_file_id", "object_store_id"),
("purged_hda_id", "deleted_icda_id", "deleted_icda_hda_id"),
)
@@ -862,6 +866,7 @@ class PurgeErrorHDAs(PurgesHDAs, RemovesMetadataFiles, RequiresDiskUsageRecalcul
- Mark purged all HistoryDatasetAssociations whose dataset_id is state = 'error' that are older
than the specified number of days.
"""
force_retry_sql = " AND NOT history_dataset_association.purged"
_action_sql = """
WITH purged_hda_ids
@@ -895,8 +900,8 @@ class PurgeErrorHDAs(PurgesHDAs, RemovesMetadataFiles, RequiresDiskUsageRecalcul
ORDER BY purged_hda_ids.id
"""
causals = (
('purged_hda_id', 'deleted_metadata_file_id', 'object_store_id'),
('purged_hda_id', 'deleted_icda_id', 'deleted_icda_hda_id'),
("purged_hda_id", "deleted_metadata_file_id", "object_store_id"),
("purged_hda_id", "deleted_icda_id", "deleted_icda_hda_id"),
)
@@ -905,6 +910,7 @@ class PurgeHDAsOfPurgedHistories(PurgesHDAs, RequiresDiskUsageRecalculation, Act
- Mark purged all HistoryDatasetAssociations in histories that are purged and older than the
specified number of days.
"""
force_retry_sql = " AND NOT history_dataset_association.purged"
_action_sql = """
WITH purged_hda_ids
@@ -945,6 +951,7 @@ class PurgeDeletedHistories(PurgesHDAs, RequiresDiskUsageRecalculation, Action):
- Mark purged all HistoryDatasetAssociations in Histories marked purged in this step (if not
already purged).
"""
_action_sql = """
WITH purged_history_ids
AS ( UPDATE history
@@ -989,9 +996,9 @@ class PurgeDeletedHistories(PurgesHDAs, RequiresDiskUsageRecalculation, Action):
ORDER BY purged_history_ids.id
"""
causals = (
('purged_history_id', 'purged_hda_id'),
('purged_hda_id', 'deleted_metadata_file_id', 'object_store_id'),
('purged_hda_id', 'deleted_icda_id', 'deleted_icda_hda_id'),
("purged_history_id", "purged_hda_id"),
("purged_hda_id", "deleted_metadata_file_id", "object_store_id"),
("purged_hda_id", "deleted_icda_id", "deleted_icda_hda_id"),
)
@@ -1000,6 +1007,7 @@ class DeleteExportedHistories(Action):
- Mark deleted all Datasets that are derivative of JobExportHistoryArchives that are older than
the specified number of days.
"""
_action_sql = """
WITH deleted_dataset_ids
AS ( UPDATE dataset
@@ -1027,6 +1035,7 @@ class DeleteDatasets(Action):
- JobExportHistoryArchives have no deleted column, so the datasets for these will simply be
deleted after the specified number of days
"""
_action_sql = """
WITH deleted_dataset_ids
AS ( UPDATE dataset
@@ -1060,6 +1069,7 @@ class PurgeDatasets(RemovesDatasets, Action):
"""
- Mark purged all Datasets marked deleted that are older than the specified number of days.
"""
_action_sql = """
WITH purged_dataset_ids
AS ( UPDATE dataset
@@ -1105,7 +1115,12 @@ class Cleanup:
if self.__actions is None:
self.__actions = {}
for name, value in inspect.getmembers(sys.modules[__name__]):
if not name.startswith('_') and inspect.isclass(value) and value != Action and issubclass(value, Action):
if (
not name.startswith("_")
and inspect.isclass(value)
and value != Action
and issubclass(value, Action)
):
self.__actions[value.name_c()] = value
return self.__actions
@@ -1113,76 +1128,63 @@ class Cleanup:
def conn(self):
if self.__conn is None:
url = make_url(galaxy.config.get_database_url(self.config))
log.info(f'Connecting to database with URL: {url}')
args = url.translate_connect_args(username='user')
log.info(f"Connecting to database with URL: {url}")
args = url.translate_connect_args(username="user")
args.update(url.query)
assert url.get_dialect().name == 'postgresql', 'This script can only be used with PostgreSQL.'
assert url.get_dialect().name == "postgresql", "This script can only be used with PostgreSQL."
self.__conn = psycopg2.connect(cursor_factory=NamedTupleCursor, **args)
# TODO: is this per session or cursor?
if self.args.work_mem is not None:
log.info('Setting work_mem to %s' % self.args.work_mem)
self.__conn.cursor().execute('SET work_mem TO %s', (self.args.work_mem,))
log.info("Setting work_mem to %s" % self.args.work_mem)
self.__conn.cursor().execute("SET work_mem TO %s", (self.args.work_mem,))
return self.__conn
def __parse_args(self):
parser = argparse.ArgumentParser()
populate_config_args(parser)
parser.add_argument(
'-d', '--debug',
action='store_true',
default=False,
help='Enable debug logging (SQL queries)')
"-d", "--debug", action="store_true", default=False, help="Enable debug logging (SQL queries)"
)
parser.add_argument("--dry-run", action="store_true", default=False, help="Dry run (rollback all transactions)")
parser.add_argument(
'--dry-run',
action='store_true',
default=False,
help="Dry run (rollback all transactions)")
"--force-retry", action="store_true", default=False, help="Retry file removals (on applicable actions)"
)
parser.add_argument(
'--force-retry',
action='store_true',
default=False,
help="Retry file removals (on applicable actions)")
parser.add_argument(
'-o', '--older-than',
dest='days',
"-o",
"--older-than",
dest="days",
type=int,
default=14,
help='Only perform action(s) on objects that have not been updated since the specified number of days')
help="Only perform action(s) on objects that have not been updated since the specified number of days",
)
parser.add_argument(
'-U', '--no-update-time',
action='store_false',
dest='update_time',
"-U",
"--no-update-time",
action="store_false",
dest="update_time",
default=True,
help="Don't set update_time on updated objects")
help="Don't set update_time on updated objects",
)
parser.add_argument(
'-s', '--sequence',
dest='sequence',
default='',
help='DEPRECATED: Comma-separated sequence of actions')
"-s", "--sequence", dest="sequence", default="", help="DEPRECATED: Comma-separated sequence of actions"
)
parser.add_argument(
'-w', '--work-mem',
dest='work_mem',
default=None,
help='Set PostgreSQL work_mem for this connection')
"-w", "--work-mem", dest="work_mem", default=None, help="Set PostgreSQL work_mem for this connection"
)
parser.add_argument("-l", "--log-dir", default=DEFAULT_LOG_DIR, help="Log file directory")
parser.add_argument("-g", "--log-file", default=None, help="Log file name")
parser.add_argument(
'-l', '--log-dir',
default=DEFAULT_LOG_DIR,
help='Log file directory')
parser.add_argument(
'-g', '--log-file',
default=None,
help='Log file name')
parser.add_argument(
'actions',
nargs='*',
metavar='ACTION',
"actions",
nargs="*",
metavar="ACTION",
default=[],
help='Action(s) to perform, chosen from: %s' % ', '.join(sorted(self.actions.keys())))
help="Action(s) to perform, chosen from: %s" % ", ".join(sorted(self.actions.keys())),
)
self.args = parser.parse_args()
# add deprecated sequence arg to actions
self.args.sequence = [x.strip() for x in self.args.sequence.split(',')]
if self.args.sequence != ['']:
self.args.sequence = [x.strip() for x in self.args.sequence.split(",")]
if self.args.sequence != [""]:
self.args.actions.extend(self.args.sequence)
if not self.args.actions:
parser.error("Please specify one or more actions")
@@ -1190,16 +1192,17 @@ class Cleanup:
def __setup_logging(self):
logging.basicConfig(
level=logging.DEBUG if self.args.debug else logging.INFO,
format="%(asctime)s %(levelname)-5s %(funcName)s(): %(message)s")
format="%(asctime)s %(levelname)-5s %(funcName)s(): %(message)s",
)
def __validate_actions(self):
ok = True
for name in self.args.actions:
if name not in self.actions.keys():
log.error('Unknown action in sequence: %s' % name)
log.error("Unknown action in sequence: %s" % name)
ok = False
if not ok:
log.critical('Exiting due to previous error(s)')
log.critical("Exiting due to previous error(s)")
sys.exit(1)
def __load_config(self):
@@ -1219,17 +1222,19 @@ class Cleanup:
self.conn.commit()
log.info("An event must exist for the subsequent query to succeed, so a dummy event has been created")
else:
log.info("Not executing event creation (increments sequence even when rolling back), using an old "
"event ID (%i) for dry run" % max_id)
log.info(
"Not executing event creation (increments sequence even when rolling back), using an old "
"event ID (%i) for dry run" % max_id
)
return max_id
def _execute(self, sql, args):
cur = self.conn.cursor()
sql_str = cur.mogrify(sql, args).decode('utf-8')
sql_str = cur.mogrify(sql, args).decode("utf-8")
log.debug(f"SQL is: {sql_str}")
log.info("Executing SQL")
cur.execute(sql, args)
log.info('Database status: %s', cur.statusmessage)
log.info("Database status: %s", cur.statusmessage)
return cur
def _create_event(self, message=None):
@@ -1246,20 +1251,20 @@ class Cleanup:
RETURNING id;
"""
message = message or self.__current_action
args = {'message': message}
args = {"message": message}
event_id = self._execute(sql, args).fetchone()[0]
log.info("Created event %s for action: %s", event_id, self.__current_action)
return event_id
def _update(self, sql, args, add_event=True, event_message=None):
if add_event and 'event_id' not in args:
args['event_id'] = self._create_event(message=event_message)
if add_event and "event_id" not in args:
args["event_id"] = self._create_event(message=event_message)
cur = self._execute(sql, args)
if cur.rowcount <= 0:
log.info("Update resulted in no changes, rolling back transaction")
self.conn.rollback()
else:
log.info('Flushing transaction')
log.info("Flushing transaction")
self._flush()
return cur
@@ -1283,12 +1288,12 @@ class Cleanup:
self.__current_action = name
with cls(self) as action:
self._run_action(action)
log.info('Finished %s' % name)
log.info("Finished %s" % name)
if __name__ == '__main__':
if __name__ == "__main__":
with Cleanup() as app:
try:
app.run()
except Exception:
log.exception('Caught exception in run sequence:')
log.exception("Caught exception in run sequence:")
scripts/cleanup_datasets/populate_uuid.py
+5 -2
View File
@@ -12,10 +12,13 @@ import os
import sys
import uuid
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "lib")))
import galaxy.config
from galaxy.util.script import app_properties_from_args, populate_config_args
from galaxy.util.script import (
app_properties_from_args,
populate_config_args,
)
DESCRIPTION = """
Populates blank uuid fields in datasets with randomly generated values.
scripts/config_sample_to_kwalify.py
+2 -1
View File
@@ -12,7 +12,7 @@ def main():
sample = "config/galaxy.ini.sample"
for line in open(sample):
is_app_main = line.startswith('[app:main]')
is_app_main = line.startswith("[app:main]")
if not found_app_main and not is_app_main:
continue
if is_app_main:
@@ -37,6 +37,7 @@ def main():
def _dump_option(option, current_section_desc):
def print_line(line):
print((" " * 6) + line)
if "=" not in option:
print(option)
key, default = (s.strip() for s in option.split("=", 1))
scripts/create_db.py
+7 -7
View File
@@ -19,7 +19,7 @@ import logging
import os.path
import sys
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib")))
from galaxy.model.migrate.check import create_or_verify_database as create_db
from galaxy.model.orm.scripts import get_config
@@ -32,12 +32,12 @@ log = logging.getLogger(__name__)
def invoke_create():
config = get_config(sys.argv)
if config['database'] == 'galaxy':
create_db(config['db_url'], config['config_file'], map_install_models=not config['install_database_connection'])
elif config['database'] == 'tool_shed':
create_tool_shed_db(config['db_url'])
elif config['database'] == 'install':
create_install_db(config['db_url'])
if config["database"] == "galaxy":
create_db(config["db_url"], config["config_file"], map_install_models=not config["install_database_connection"])
elif config["database"] == "tool_shed":
create_tool_shed_db(config["db_url"])
elif config["database"] == "install":
create_install_db(config["db_url"])
if __name__ == "__main__":
scripts/cwl_conformance_to_test_cases.py
+26 -17
View File
@@ -8,7 +8,8 @@ THIS_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
GALAXY_ROOT_DIR = os.path.abspath(os.path.join(THIS_DIRECTORY, os.pardir))
CWL_API_TESTS_DIRECTORY = os.path.join(GALAXY_ROOT_DIR, "lib", "galaxy_test", "api", "cwl")
TEST_FILE_TEMPLATE = string.Template('''"""Test CWL conformance for version ${version}."""
TEST_FILE_TEMPLATE = string.Template(
'''"""Test CWL conformance for version ${version}."""
import pytest
@@ -17,9 +18,11 @@ from ..test_workflows_cwl import BaseCwlWorkflowTestCase
class CwlConformanceTestCase(BaseCwlWorkflowTestCase):
"""Test case mapping to CWL conformance tests for version ${version}."""
$tests''')
$tests'''
)
TEST_TEMPLATE = string.Template('''
TEST_TEMPLATE = string.Template(
'''
${marks} def test_conformance_${version_simple}_${label}(self):
"""${doc}
@@ -28,7 +31,8 @@ ${marks} def test_conformance_${version_simple}_${label}(self):
${cwl_test_def}
""" # noqa: W293
self.cwl_populator.run_conformance_test("""${version}""", """${doc}""")
''')
'''
)
RED_TESTS = {
"v1.0": [
@@ -356,7 +360,7 @@ def main():
for i, conformance_test in enumerate(conformance_tests_gen(os.path.join(conformance_tests_dir, version))):
test_with_doc = conformance_test.copy()
if 'doc' not in test_with_doc:
if "doc" not in test_with_doc:
raise Exception(f"No doc in test [{test_with_doc}]")
del test_with_doc["doc"]
cwl_test_def = yaml.dump(test_with_doc, default_flow_style=False)
@@ -375,15 +379,18 @@ def main():
marks += " @pytest.mark.green\n"
if not {"command_line_tool", "expression_tool", "workflow"}.intersection(tags):
print(f"PROBLEM - test [{label}] tagged with neither command_line_tool, expression_tool, nor workflow", file=sys.stderr)
print(
f"PROBLEM - test [{label}] tagged with neither command_line_tool, expression_tool, nor workflow",
file=sys.stderr,
)
template_kwargs = {
'version_simple': version_simple,
'version': version,
'doc': conformance_test['doc'],
'cwl_test_def': cwl_test_def,
'label': label.replace("-", "_"),
'marks': marks,
"version_simple": version_simple,
"version": version,
"doc": conformance_test["doc"],
"cwl_test_def": cwl_test_def,
"label": label.replace("-", "_"),
"marks": marks,
}
test_body = TEST_TEMPLATE.safe_substitute(template_kwargs)
tests += test_body
@@ -394,11 +401,13 @@ def main():
if is_red:
red_tests_found.add(label)
test_file_contents = TEST_FILE_TEMPLATE.safe_substitute({
'version': version,
'version_simple': version_simple,
'tests': tests,
})
test_file_contents = TEST_FILE_TEMPLATE.safe_substitute(
{
"version": version,
"version_simple": version_simple,
"tests": tests,
}
)
test_file = os.path.join(CWL_API_TESTS_DIRECTORY, f"test_cwl_conformance_{version_simple}.py")
with open(test_file, "w") as f:
scripts/db_shell.py
+8 -15
View File
@@ -21,11 +21,11 @@ import sys
# Setup DB scripting environment
from sqlalchemy import * # noqa
from sqlalchemy.orm import * # noqa
from sqlalchemy.exc import * # noqa
from sqlalchemy.orm import * # noqa
from sqlalchemy.sql import label # noqa
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib")))
from galaxy.datatypes.registry import Registry
from galaxy.model import * # noqa
@@ -37,8 +37,8 @@ registry = Registry()
registry.load_datatypes()
set_datatypes_registry(registry)
config = get_config(sys.argv)
db_url = config['db_url']
sa_session = init('/tmp/', db_url).context
db_url = config["db_url"]
sa_session = init("/tmp/", db_url).context
# Helper function for debugging sqlalchemy queries...
@@ -51,6 +51,7 @@ def printquery(statement, bind=None):
please also note that this function is quite slow
"""
import sqlalchemy.orm
if isinstance(statement, sqlalchemy.orm.Query):
if bind is None:
bind = statement.session.get_bind()
@@ -62,15 +63,9 @@ def printquery(statement, bind=None):
compiler = statement._compiler(dialect)
class LiteralCompiler(compiler.__class__):
def visit_bindparam(
self, bindparam, within_columns_clause=False,
literal_binds=False, **kwargs
):
def visit_bindparam(self, bindparam, within_columns_clause=False, literal_binds=False, **kwargs):
return super().render_literal_bindparam(
bindparam,
within_columns_clause=within_columns_clause,
literal_binds=literal_binds,
**kwargs
bindparam, within_columns_clause=within_columns_clause, literal_binds=literal_binds, **kwargs
)
def render_literal_value(self, value, type_):
@@ -96,9 +91,7 @@ def printquery(statement, bind=None):
return "TO_DATE('%s','YYYY-MM-DD HH24:MI:SS')" % value.strftime("%Y-%m-%d %H:%M:%S")
else:
raise NotImplementedError(
"Don't know how to literal-quote value %r" % value
)
raise NotImplementedError("Don't know how to literal-quote value %r" % value)
compiler = LiteralCompiler(dialect, statement)
print(compiler.process(statement))
scripts/docker_swarm_manager.py
+169 -122
View File
@@ -16,11 +16,13 @@ try:
import daemon.pidfile
import lockfile
except ImportError:
print('ERROR: The daemon module is required to use the swarm manager, '
'install it with `pip install python-daemon`', file=sys.stderr)
print(
"ERROR: The daemon module is required to use the swarm manager, " "install it with `pip install python-daemon`",
file=sys.stderr,
)
sys.exit(1)
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib")))
from galaxy.containers import (
build_container_interfaces,
@@ -33,32 +35,30 @@ from galaxy.containers.docker_model import (
IMAGE_CONSTRAINT,
)
DESCRIPTION = "Daemon to manage a Docker Swarm (running in Docker Swarm mode)."
SWARM_MANAGER_CONF_DEFAULTS = {
'pid_file': '{xdg_data_home}/galaxy_swarm_manager.pid',
'log_file': '{xdg_data_home}/galaxy_swarm_manager.log',
'service_wait_count_limit': 0,
'service_wait_time_limit': 5,
'slots_min_limit': 0,
'slots_max_limit': sys.maxsize,
'slots_min_spare': 0,
'node_idle_limit': 120,
'limits': [],
'spawn_wait_time': 30,
'spawn_command': '/bin/true',
'destroy_command': '/bin/true',
'command_failure_command': '/bin/true',
'command_retries': 0,
'command_retry_wait': 10,
'terminate_when_idle': True,
'log_environment_variables': [],
"pid_file": "{xdg_data_home}/galaxy_swarm_manager.pid",
"log_file": "{xdg_data_home}/galaxy_swarm_manager.log",
"service_wait_count_limit": 0,
"service_wait_time_limit": 5,
"slots_min_limit": 0,
"slots_max_limit": sys.maxsize,
"slots_min_spare": 0,
"node_idle_limit": 120,
"limits": [],
"spawn_wait_time": 30,
"spawn_command": "/bin/true",
"destroy_command": "/bin/true",
"command_failure_command": "/bin/true",
"command_retries": 0,
"command_retry_wait": 10,
"terminate_when_idle": True,
"log_environment_variables": [],
}
log = logging.getLogger(__name__)
class SwarmManager:
def __init__(self, conf, docker_interface):
self._conf = conf
self._cpus = docker_interface._conf.cpus
@@ -85,7 +85,7 @@ class SwarmManager:
if returncodes:
allowed_returncodes = returncodes
raw_cmd = command.format(**kwargs)
log.debug('running command: %s', raw_cmd)
log.debug("running command: %s", raw_cmd)
success = False
while not success and attempt < command_retries + 1:
attempt += 1
@@ -94,13 +94,15 @@ class SwarmManager:
if p.returncode not in allowed_returncodes:
msg = f"error running '{raw_cmd}': returned {p.returncode}"
if attempt < command_retries + 1:
msg += ', waiting %s seconds' % self._conf.command_retry_wait
msg += ", waiting %s seconds" % self._conf.command_retry_wait
time.sleep(self._conf.command_retry_wait)
log.warning(msg + "\nstdout: %s\nstderr: %s\n", stdout, stderr)
else:
msg += ' (final attempt)'
msg += " (final attempt)"
log.error(msg + "\nstdout: %s\nstderr: %s\n", stdout, stderr)
self._run_command(self._conf.command_failure_command.format(failed_command=raw_cmd), command_retries=0)
self._run_command(
self._conf.command_failure_command.format(failed_command=raw_cmd), command_retries=0
)
stdout = None
else:
stdout = stdout.strip()
@@ -116,9 +118,9 @@ class SwarmManager:
waiting = self._docker_interface.services_waiting_by_constraints()
active = self._docker_interface.nodes_active_by_constraints()
for constraints, needed_dict in self._state.slots_needed(waiting, active).items():
services = needed_dict['services']
nodes = needed_dict['nodes']
slots_needed = needed_dict['slots_needed']
services = needed_dict["services"]
nodes = needed_dict["nodes"]
slots_needed = needed_dict["slots_needed"]
if slots_needed > 0:
self._spawn_nodes(constraints, services, slots_needed)
elif slots_needed < 0:
@@ -127,17 +129,24 @@ class SwarmManager:
def _check_for_new_nodes(self):
nodes = None
for node_state in self._state.spawning_nodes():
name = node_state['name']
elapsed = node_state['elapsed']
constraints = node_state['constraints']
state = node_state['state']
name = node_state["name"]
elapsed = node_state["elapsed"]
constraints = node_state["constraints"]
state = node_state["state"]
if not nodes:
nodes = self._docker_interface.nodes()
node = ([x for x in nodes if x.name == name] + [None])[0]
if not node:
if elapsed > self._conf.spawn_wait_time:
log.warning("spawning node '%s' not found in `docker node ls` and spawn_wait_time exceeded! %d seconds have elapsed", name, elapsed)
self._run_command(self._conf.command_failure_command.format(failed_command='wait_for_spawning_node %s' % name), command_retries=0)
log.warning(
"spawning node '%s' not found in `docker node ls` and spawn_wait_time exceeded! %d seconds have elapsed",
name,
elapsed,
)
self._run_command(
self._conf.command_failure_command.format(failed_command="wait_for_spawning_node %s" % name),
command_retries=0,
)
self.mark_spawning_node_timeout(name)
elif node.is_ok():
node.set_labels_for_constraints(constraints)
@@ -154,13 +163,13 @@ class SwarmManager:
cleaned_services = self._docker_interface.services_clean()
if cleaned_services:
self._state.clean_services(cleaned_services)
log.info("cleaned services: %s", ', '.join(x.id for x in cleaned_services))
log.info("cleaned services: %s", ", ".join(x.id for x in cleaned_services))
@staticmethod
def _env_str(envs, service):
if envs.get(service.id):
return ' [' + ', '.join(envs.get(service.id, [])) + ']'
return ''
return " [" + ", ".join(envs.get(service.id, [])) + "]"
return ""
def _log_state(self, now=False):
if not now and not (self._last_log < (time.time() - self._log_interval)):
@@ -171,28 +180,58 @@ class SwarmManager:
node_task_ids = [t.id for nt in [n.tasks for n in nodes] for t in nt]
envs = {}
for service in services:
envs[service.id] = ['{}={}'.format(k, service.env.get(k, 'unset')) for k in self._conf.log_environment_variables]
log.info('%s nodes, %s services (%s terminal)', len(nodes), len(services), len(terminal))
envs[service.id] = [
"{}={}".format(k, service.env.get(k, "unset")) for k in self._conf.log_environment_variables
]
log.info("%s nodes, %s services (%s terminal)", len(nodes), len(services), len(terminal))
if terminal:
service_strs = [f'{s.name} (state: {s.state})' for s in terminal]
log.info('terminal services: %s', ', '.join(service_strs) or 'none')
service_strs = [f"{s.name} (state: {s.state})" for s in terminal]
log.info("terminal services: %s", ", ".join(service_strs) or "none")
for service in services:
unassigned_tasks = [t for t in service.tasks if t.id not in node_task_ids]
if service not in terminal and unassigned_tasks:
task = unassigned_tasks[0]
log.info('service %s (%s)%s is not assigned to a node; state: %s %s', service.name, service.id,
self._env_str(envs, service), service.state, task.current_state_time)
log.info(
"service %s (%s)%s is not assigned to a node; state: %s %s",
service.name,
service.id,
self._env_str(envs, service),
service.state,
task.current_state_time,
)
for node in nodes:
log.info('node %s (%s) state: %s, %s tasks (%s terminal)', node.name, node.id, node.state,
len(node.tasks), len([t for t in node.tasks if t.terminal]))
log.info(
"node %s (%s) state: %s, %s tasks (%s terminal)",
node.name,
node.id,
node.state,
len(node.tasks),
len([t for t in node.tasks if t.terminal]),
)
for task in node.tasks:
if not task.service:
log.warning('node %s (%s) task %s (%s) has no service! state: %s %s', node.name, node.id,
task.slot, task.id, task.state, task.current_state_time)
log.warning(
"node %s (%s) task %s (%s) has no service! state: %s %s",
node.name,
node.id,
task.slot,
task.id,
task.state,
task.current_state_time,
)
else:
log.info('node %s (%s) service %s (%s) task %s (%s)%s state: %s %s', node.name, node.id,
task.service.name, task.service.id, task.slot, task.id,
self._env_str(envs, task.service), task.state, task.current_state_time)
log.info(
"node %s (%s) service %s (%s) task %s (%s)%s state: %s %s",
node.name,
node.id,
task.service.name,
task.service.id,
task.slot,
task.id,
self._env_str(envs, task.service),
task.state,
task.current_state_time,
)
self._last_log = time.time()
def _terminate_if_idle(self):
@@ -211,13 +250,13 @@ class SwarmManager:
return # services are waiting
if needed < 0:
extra_slots = max(
self._state.get_limit(constraints, 'slots_min_limit'),
self._state.get_limit(constraints, 'slots_min_spare')
self._state.get_limit(constraints, "slots_min_limit"),
self._state.get_limit(constraints, "slots_min_spare"),
)
if total + needed != extra_slots:
return # otherwise, nodes remaining are for configured minimums
# FIXME: there's a race condition here
log.info('nothing to manage, shutting down')
log.info("nothing to manage, shutting down")
sys.exit(0)
# other methods
@@ -237,23 +276,30 @@ class SwarmManager:
def _spawn_nodes(self, constraints, services, slots_needed):
service_ids = [x.id for x in services]
if service_ids:
log.info("requesting node(s) for services needing %s slots with constraints [%s]: %s",
slots_needed, constraints, ', '.join(service_ids))
log.info(
"requesting node(s) for services needing %s slots with constraints [%s]: %s",
slots_needed,
constraints,
", ".join(service_ids),
)
else:
log.info("requesting node(s) for %s slots (due to minimum limits with constraints [%s]",
slots_needed, constraints)
log.info(
"requesting node(s) for %s slots (due to minimum limits with constraints [%s]",
slots_needed,
constraints,
)
command = self._conf.spawn_command.format(
service_ids=','.join(service_ids),
service_ids=",".join(service_ids),
service_count=len(services),
image=self._get_spawn_property(constraints, IMAGE_CONSTRAINT, services) or '',
cpus=self._get_spawn_property(constraints, CPUS_CONSTRAINT, services) or '',
image=self._get_spawn_property(constraints, IMAGE_CONSTRAINT, services) or "",
cpus=self._get_spawn_property(constraints, CPUS_CONSTRAINT, services) or "",
slots=slots_needed,
)
rc, output = self._run_command(command, returncodes=(0, 2))
if rc == 2:
log.info('spawn_command indicated that spawning should be retried: %s', output)
log.info("spawn_command indicated that spawning should be retried: %s", output)
elif not output:
log.warning('spawn_command returned no new nodes, cannot manage nodes')
log.warning("spawn_command returned no new nodes, cannot manage nodes")
self._state.mark_services_handled(services)
else:
log.info("node allocator will spawn: %s", output)
@@ -271,11 +317,10 @@ class SwarmManager:
destroy_nodes.append(node)
destroyed_slots += node_slots
if destroy_nodes:
command = self._conf.destroy_command.format(
nodes=' '.join(x.name for x in destroy_nodes))
command = self._conf.destroy_command.format(nodes=" ".join(x.name for x in destroy_nodes))
destroyed_nodes = self._run_command(command)
if not destroyed_nodes:
log.warning('destroy_command returned no destroyed nodes')
log.warning("destroy_command returned no destroyed nodes")
else:
log.info("destroyed nodes: %s", destroyed_nodes)
@@ -291,10 +336,9 @@ class SwarmManager:
class SwarmState:
def __init__(self, conf, interface_conf):
self._conf = conf
self._cpus = interface_conf.cpus # this is effectively the slot size
self._cpus = interface_conf.cpus # this is effectively the slot size
self._service_create_image_constraint = interface_conf.service_create_image_constraint
self._service_create_cpus_constraint = interface_conf.service_create_cpus_constraint
self._handled_services = set()
@@ -303,15 +347,15 @@ class SwarmState:
self._surplus_nodes = {}
self._limits = {}
for limit in conf.limits:
constraints = DockerServiceConstraints.from_constraint_string_list(limit.get('constraints', []))
constraints = DockerServiceConstraints.from_constraint_string_list(limit.get("constraints", []))
self._limits[constraints] = self._make_limit_dict(limit)
def _make_limit_dict(self, limit):
return {
'slots_min_limit': limit.get('slots_min_limit', self._conf.slots_min_limit),
'slots_max_limit': limit.get('slots_max_limit', self._conf.slots_max_limit),
'slots_min_spare': limit.get('slots_min_spare', self._conf.slots_min_spare),
'node_idle_limit': limit.get('node_idle_limit', self._conf.node_idle_limit),
"slots_min_limit": limit.get("slots_min_limit", self._conf.slots_min_limit),
"slots_max_limit": limit.get("slots_max_limit", self._conf.slots_max_limit),
"slots_min_spare": limit.get("slots_min_spare", self._conf.slots_min_spare),
"node_idle_limit": limit.get("node_idle_limit", self._conf.node_idle_limit),
}
def slots_needed(self, waiting_services, active_nodes):
@@ -326,11 +370,13 @@ class SwarmState:
all_constraints = services_constraints.union(nodes_constraints).union(limits_constraints)
if not all_constraints and (self._conf.slots_min_spare or self._conf.slots_min_limit):
if self._service_create_image_constraint or self._service_create_cpus_constraint:
raise Exception("Global 'slots_min_limit' and/or 'slots_min_spare' are set and "
raise Exception(
"Global 'slots_min_limit' and/or 'slots_min_spare' are set and "
"'service_create_image_constraint' and/or 'service_create_cpus_constraint' are set but "
"constraint-specific limits are unset, minimum nodes cannot be started since the constraints are not "
"known until service creation time. Either disable 'service_create_*_constraint' or create "
"constraint-specific limits in the 'limits' section of 'manager_conf' in containers_conf.yml")
"constraint-specific limits in the 'limits' section of 'manager_conf' in containers_conf.yml"
)
all_constraints.add(DockerServiceConstraints.from_constraint_string_list([]))
for constraints in all_constraints:
services = waiting_services.get(constraints, [])
@@ -345,9 +391,9 @@ class SwarmState:
elif not services and constraints in self._waiting_since:
del self._waiting_since[constraints]
rval[constraints] = {
'services': services,
'nodes': nodes,
'slots_needed': slots_needed,
"services": services,
"nodes": nodes,
"slots_needed": slots_needed,
}
return rval
@@ -361,17 +407,19 @@ class SwarmState:
used += sum(t.cpus for t in node.non_terminal_tasks) / self._cpus
total += node.cpus / self._cpus
# need at least this many slots
needed = used + self.get_limit(constraints, 'slots_min_spare')
if (len(services) > self._conf.service_wait_count_limit
and time.time() - self._waiting_since.get(constraints, time.time()) > self._conf.service_wait_time_limit):
needed = used + self.get_limit(constraints, "slots_min_spare")
if (
len(services) > self._conf.service_wait_count_limit
and time.time() - self._waiting_since.get(constraints, time.time()) > self._conf.service_wait_time_limit
):
# add slots for waiting services that have exceeded limits
needed += sum(s.cpus for s in services) / self._cpus
# subtract slots for spawning nodes
needed -= sum(n.get('slots', 0) for n in self._spawning_nodes.get(constraints, {}))
needed -= sum(n.get("slots", 0) for n in self._spawning_nodes.get(constraints, {}))
# ensure no less than slots_min_limit slots will exist (free or used)
needed = max(needed, self.get_limit(constraints, 'slots_min_limit'))
needed = max(needed, self.get_limit(constraints, "slots_min_limit"))
# ensure no more than slots_max_limit slots will exist
needed = min(needed, self.get_limit(constraints, 'slots_max_limit'))
needed = min(needed, self.get_limit(constraints, "slots_max_limit"))
# need to add/remove this many slots
return int(needed - total), total
@@ -384,9 +432,9 @@ class SwarmState:
for constraints in self._spawning_nodes.keys():
for name, node in self._spawning_nodes[constraints].items():
yval = {
'name': name,
'elapsed': now - node['time_requested'],
'constraints': constraints,
"name": name,
"elapsed": now - node["time_requested"],
"constraints": constraints,
}
yval.update(node)
yield yval
@@ -395,15 +443,15 @@ class SwarmState:
if constraints not in self._spawning_nodes:
self._spawning_nodes[constraints] = {}
for node in nodes:
name = node.split(':')[0]
name = node.split(":")[0]
try:
slots = int(node.split(':')[1])
slots = int(node.split(":")[1])
except IndexError:
slots = int(1 / self._cpus)
self._spawning_nodes[constraints][name] = {
'state': 'requested',
'time_requested': time.time(),
'slots': slots,
"state": "requested",
"time_requested": time.time(),
"slots": slots,
}
def mark_services_handled(self, services):
@@ -423,11 +471,11 @@ class SwarmState:
def mark_spawning_node_state(self, node_name, state):
for constraints in self._spawning_nodes.keys():
if node_name in self._spawning_nodes[constraints]:
self._spawning_nodes[constraints][node_name]['state'] = state
self._spawning_nodes[constraints][node_name]["state"] = state
def is_destruction_time(self, node):
now = time.time()
limit = self.get_limit(node.labels_as_constraints, 'node_idle_limit')
limit = self.get_limit(node.labels_as_constraints, "node_idle_limit")
return now - self._surplus_nodes.get(node.name, now) > limit
def mark_node_idle(self, node_name):
@@ -456,8 +504,7 @@ def _arg_parser():
parser.add_argument("-c", "--containers-config-file", default=None)
parser.add_argument("-f", "--foreground", action="store_true", default=False)
parser.add_argument("-d", "--debug", action="store_true", default=False)
parser.add_argument("-s", "--swarm", default="_default_",
help='Swarm name in containers config to manage')
parser.add_argument("-s", "--swarm", default="_default_", help="Swarm name in containers config to manage")
return parser
@@ -471,11 +518,11 @@ def _run_swarm_manager(args):
pidfile = _swarm_manager_pidfile(swarm_manager_conf)
if not args.foreground:
_swarm_manager_daemon(pidfile, swarm_manager_conf['log_file'], swarm_manager_conf, docker_interface)
_swarm_manager_daemon(pidfile, swarm_manager_conf["log_file"], swarm_manager_conf, docker_interface)
else:
if swarm_manager_conf['terminate_when_idle']:
log.info('running in the foreground, disabling automatic swarm manager termination')
swarm_manager_conf['terminate_when_idle'] = False
if swarm_manager_conf["terminate_when_idle"]:
log.info("running in the foreground, disabling automatic swarm manager termination")
swarm_manager_conf["terminate_when_idle"] = False
else:
log.info("running in the foreground")
try:
@@ -499,32 +546,32 @@ def _run_swarm_manager(args):
def _containers_config_file(args):
containers_config_file = args.containers_config_file
if not containers_config_file:
for path in ('./config', '.'):
testf = os.path.join(path, 'containers_conf.yml')
for path in ("./config", "."):
testf = os.path.join(path, "containers_conf.yml")
if os.path.exists(testf):
containers_config_file = testf
assert containers_config_file, \
"containers_conf.yml cannot be found, please set with '-c' or '--containers-config-file'"
assert (
containers_config_file
), "containers_conf.yml cannot be found, please set with '-c' or '--containers-config-file'"
return containers_config_file
def _container_conf(containers_conf, swarm):
assert swarm in containers_conf, \
"invalid container configuration name: %s" % swarm
assert containers_conf[swarm]['type'] == 'docker_swarm', \
assert swarm in containers_conf, "invalid container configuration name: %s" % swarm
assert containers_conf[swarm]["type"] == "docker_swarm", (
"'%s' container configuration is not 'docker_swarm' type" % swarm
assert containers_conf[swarm].get('managed', True), \
"'%s' swarm is not managed" % swarm
)
assert containers_conf[swarm].get("managed", True), "'%s' swarm is not managed" % swarm
return containers_conf[swarm]
def _swarm_manager_conf(new_conf):
conf = ContainerInterfaceConfig()
conf.update(SWARM_MANAGER_CONF_DEFAULTS)
conf.update(new_conf.get('manager_conf', {}))
conf.update(new_conf.get("manager_conf", {}))
xdg_env = _load_xdg_environment()
for opt in ('pid_file', 'log_file'):
conf[opt] = conf[opt].format(xdg_data_home=xdg_env['data_home'])
for opt in ("pid_file", "log_file"):
conf[opt] = conf[opt].format(xdg_data_home=xdg_env["data_home"])
return conf
@@ -533,13 +580,13 @@ def _configure_logging(args, conf):
if args and args.debug:
log_level = logging.DEBUG
else:
log_level = logging.getLevelName(conf.get('log_level', 'INFO').upper())
assert int(log_level), 'invalid log level: %s' % conf['log_level']
log_level = logging.getLevelName(conf.get("log_level", "INFO").upper())
assert int(log_level), "invalid log level: %s" % conf["log_level"]
log = logging.getLogger(__name__)
gxlog = logging.getLogger('galaxy')
gxlog = logging.getLogger("galaxy")
log.setLevel(log_level)
gxlog.setLevel(log_level)
log_format = conf.get('log_format', '%(name)s %(levelname)s %(asctime)s %(message)s')
log_format = conf.get("log_format", "%(name)s %(levelname)s %(asctime)s %(message)s")
formatter = logging.Formatter(log_format)
# file logging is handled by daemon
handler = logging.StreamHandler(sys.stdout)
@@ -550,22 +597,22 @@ def _configure_logging(args, conf):
def _load_xdg_environment():
return dict(
data_home=os.path.expanduser(os.environ.get('XDG_DATA_HOME', '~/.local/share')),
data_home=os.path.expanduser(os.environ.get("XDG_DATA_HOME", "~/.local/share")),
)
def _swarm_manager_pidfile(conf):
try:
os.makedirs(os.path.dirname(conf['pid_file']))
os.makedirs(os.path.dirname(conf["pid_file"]))
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
return daemon.pidfile.PIDLockFile(conf['pid_file'])
return daemon.pidfile.PIDLockFile(conf["pid_file"])
def _swarm_manager_daemon(pidfile, logfile, swarm_manager_conf, docker_interface):
log.info("daemonizing, logs will be written to '%s'", logfile)
with open(logfile, 'a') as logfh:
with open(logfile, "a") as logfh:
try:
with daemon.DaemonContext(
pidfile=pidfile,
@@ -588,6 +635,6 @@ def _swarm_manager(conf, docker_interface):
log.error("restarting due to fatal error")
if __name__ == '__main__':
__name__ = 'swarm_manager'
if __name__ == "__main__":
__name__ = "swarm_manager"
main()
scripts/drmaa_external_killer.py
+9 -3
View File
@@ -29,15 +29,21 @@ def set_user(uid):
os.setuid(uid)
except OSError as e:
if e.errno == errno.EPERM:
sys.stderr.write("error: setuid(%d) failed: permission denied. Did you setup 'sudo' correctly for this script?\n" % uid)
sys.stderr.write(
"error: setuid(%d) failed: permission denied. Did you setup 'sudo' correctly for this script?\n" % uid
)
exit(1)
else:
pass
if os.getuid() == 0:
sys.stderr.write("error: UID is 0 (root) after changing user. This script should not be run as root. aborting.\n")
sys.stderr.write(
"error: UID is 0 (root) after changing user. This script should not be run as root. aborting.\n"
)
exit(1)
if os.geteuid() == 0:
sys.stderr.write("error: EUID is 0 (root) after changing user. This script should not be run as root. aborting.\n")
sys.stderr.write(
"error: EUID is 0 (root) after changing user. This script should not be run as root. aborting.\n"
)
exit(1)
scripts/drmaa_external_runner.py
+22 -6
View File
@@ -14,8 +14,17 @@ import sys
import drmaa
DRMAA_jobTemplate_attributes = ['args', 'remoteCommand', 'outputPath', 'errorPath', 'nativeSpecification',
'workingDirectory', 'jobName', 'email', 'project']
DRMAA_jobTemplate_attributes = [
"args",
"remoteCommand",
"outputPath",
"errorPath",
"nativeSpecification",
"workingDirectory",
"jobName",
"email",
"project",
]
def load_job_template(jt, data):
@@ -86,6 +95,7 @@ def set_user(uid, assign_all_groups):
# Solves issue with permission denied for JSON files
gid = pwd.getpwuid(uid).pw_gid
import grp
os.setgid(gid)
if assign_all_groups:
# Added lines to assure read/write permission for groups
@@ -97,17 +107,23 @@ def set_user(uid, assign_all_groups):
except OSError as e:
if e.errno == errno.EPERM:
sys.stderr.write("error: setuid(%d) failed: permission denied. Did you setup 'sudo' correctly for this script?\n" % uid)
sys.stderr.write(
"error: setuid(%d) failed: permission denied. Did you setup 'sudo' correctly for this script?\n" % uid
)
exit(1)
else:
pass
if os.getuid() == 0:
sys.stderr.write("error: UID is 0 (root) after changing user. This script should not be run as root. aborting.\n")
sys.stderr.write(
"error: UID is 0 (root) after changing user. This script should not be run as root. aborting.\n"
)
exit(1)
if os.geteuid() == 0:
sys.stderr.write("error: EUID is 0 (root) after changing user. This script should not be run as root. aborting.\n")
sys.stderr.write(
"error: EUID is 0 (root) after changing user. This script should not be run as root. aborting.\n"
)
exit(1)
@@ -122,7 +138,7 @@ def main():
set_user(userid, assign_all_groups)
# Added to disable LSF generated messages that would interfer with this
# script. Fix thank to Chong Chen at IBM.
os.environ['BSUB_QUIET'] = 'Y'
os.environ["BSUB_QUIET"] = "Y"
s = drmaa.Session()
s.initialize()
jt = s.createJobTemplate()
scripts/dump_library_paths.py
+37 -37
View File
@@ -15,9 +15,12 @@ import logging
import os
import sys
from sqlalchemy import false, not_
from sqlalchemy import (
false,
not_,
)
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib")))
import galaxy.config
import galaxy.model.mapping
@@ -27,41 +30,37 @@ from galaxy.util.script import main_factory
DESCRIPTION = "Locate all datasets in libraries."
ARGUMENTS = (
(
('-v', '--verbose'),
("-v", "--verbose"),
dict(
action='store_true',
action="store_true",
default=False,
help='Verbose logging output',
help="Verbose logging output",
),
),
(
('-o', '--output'),
("-o", "--output"),
dict(
default='stdout',
help='Write output to file',
default="stdout",
help="Write output to file",
),
),
(
('-p', '--public'),
dict(
action='store_true',
default=False,
help='Only dump files in "public" libraries'
),
("-p", "--public"),
dict(action="store_true", default=False, help='Only dump files in "public" libraries'),
),
(
('--relative',),
("--relative",),
dict(
default=None,
help='Write paths relative to the given directory',
help="Write paths relative to the given directory",
),
),
(
('--exists',),
("--exists",),
dict(
action='store_true',
action="store_true",
default=False,
help='Check for dataset existence, warn if it does not exist',
help="Check for dataset existence, warn if it does not exist",
),
),
)
@@ -76,15 +75,19 @@ def _config_logging(args):
def _get_libraries(args, model):
log.debug('Setting up query')
log.debug("Setting up query")
library_access_action = model.security_agent.permitted_actions.LIBRARY_ACCESS.action
query = model.context.query(model.Library)
query = query.filter(model.Library.table.c.deleted == false())
if args.public:
restricted_library_ids = {lp.library_id for lp in (
model.context.query(model.LibraryPermissions).filter(
model.LibraryPermissions.table.c.action == library_access_action
).distinct())}
restricted_library_ids = {
lp.library_id
for lp in (
model.context.query(model.LibraryPermissions)
.filter(model.LibraryPermissions.table.c.action == library_access_action)
.distinct()
)
}
if restricted_library_ids:
query = query.filter(not_(model.Library.table.c.id.in_(restricted_library_ids)))
query = query.order_by(model.Library.table.c.name)
@@ -107,10 +110,10 @@ def _walk_libraries(args, model):
def _open_output(args):
if args.output == 'stdout':
if args.output == "stdout":
return sys.stdout
else:
return open(args.output, 'w')
return open(args.output, "w")
def _path(path, args):
@@ -124,21 +127,21 @@ def _get_library_dataset_paths(args, kwargs):
_config_logging(args)
config = galaxy.config.Configuration(**kwargs)
object_store = build_object_store_from_config(config)
model = galaxy.model.mapping.init('/tmp/', kwargs.get('database_connection'), object_store=object_store)
model = galaxy.model.mapping.init("/tmp/", kwargs.get("database_connection"), object_store=object_store)
output = _open_output(args)
last_library = None
log.debug('Beginning library walk')
log.debug("Beginning library walk")
for library, dataset in _walk_libraries(args, model):
if library != last_library:
log.info('Library: %s', library.name)
log.info("Library: %s", library.name)
filename = object_store.get_filename(dataset)
files_dir = dataset.get_extra_files_path()
if (args.exists and object_store.exists(dataset)) or not args.exists:
output.write('%s\n' % _path(filename, args))
output.write("%s\n" % _path(filename, args))
elif args.exists:
log.warning('Missing %s', filename)
log.warning("Missing %s", filename)
if files_dir and os.path.exists(files_dir):
output.write('%s\n' % _path(files_dir, args))
output.write("%s\n" % _path(files_dir, args))
last_library = library
output.close()
@@ -148,11 +151,8 @@ ACTIONS = {
}
if __name__ == '__main__':
if __name__ == "__main__":
main = main_factory(
description=DESCRIPTION,
actions=ACTIONS,
arguments=ARGUMENTS,
default_action="get_library_dataset_paths"
description=DESCRIPTION, actions=ACTIONS, arguments=ARGUMENTS, default_action="get_library_dataset_paths"
)
main()
scripts/edam_mapping.py
+6 -4
View File
@@ -18,7 +18,7 @@ from xml import etree
import requests
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib")))
import galaxy.datatypes.registry
import galaxy.model
@@ -30,7 +30,9 @@ CONFIG_FILE = os.path.join(PROJECT_DIR, "config", "datatypes_conf.xml.sample")
datatypes_registry = galaxy.datatypes.registry.Registry()
datatypes_registry.load_datatypes(root_dir=PROJECT_DIR, config=CONFIG_FILE)
EDAM_OWL_URL = "http://data.bioontology.org/ontologies/EDAM/submissions/25/download?apikey=8b5b7825-538d-40e0-9e9e-5ab9274a9aeb"
EDAM_OWL_URL = (
"http://data.bioontology.org/ontologies/EDAM/submissions/25/download?apikey=8b5b7825-538d-40e0-9e9e-5ab9274a9aeb"
)
if not os.path.exists("/tmp/edam.owl"):
@@ -39,13 +41,13 @@ if not os.path.exists("/tmp/edam.owl"):
owl_xml_tree = etree.ElementTree.parse("/tmp/edam.owl")
format_info = {}
for child in owl_xml_tree.getroot().findall('{http://www.w3.org/2002/07/owl#}Class'):
for child in owl_xml_tree.getroot().findall("{http://www.w3.org/2002/07/owl#}Class"):
about = child.attrib.get("{http://www.w3.org/1999/02/22-rdf-syntax-ns#}about")
if not about:
continue
if not about.startswith("http://edamontology.org/format_"):
continue
the_format = about[len("http://edamontology.org/"):]
the_format = about[len("http://edamontology.org/") :]
label = child.find("{http://www.w3.org/2000/01/rdf-schema#}label").text
definition = ""
def_el = child.find("{http://www.geneontology.org/formats/oboInOwl#}hasDefinition")
scripts/external_chown_script.py
+1 -1
View File
@@ -39,7 +39,7 @@ def validate_parameters():
def main():
path, galaxy_user_name, gid = validate_parameters()
for cmd in [['chown', '-Rh', galaxy_user_name, path], ['chgrp', '-Rh', gid, path]]:
for cmd in [["chown", "-Rh", galaxy_user_name, path], ["chgrp", "-Rh", gid, path]]:
p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(stdoutdata, stderrdata) = p.communicate()
exitcode = p.returncode
scripts/extract_dataset_part.py
+4 -4
View File
@@ -11,7 +11,7 @@ import logging
import os
import sys
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib")))
# This junk is here to prevent loading errors
import galaxy.model.mapping # need to load this before we unpickle, in order to setup properties assigned by the mappers
@@ -32,13 +32,13 @@ def __main__():
sys.exit(0)
data = json.load(open(file_path))
try:
class_name_parts = data['class_name'].split('.')
module_name = '.'.join(class_name_parts[:-1])
class_name_parts = data["class_name"].split(".")
module_name = ".".join(class_name_parts[:-1])
class_name = class_name_parts[-1]
mod = __import__(module_name, globals(), locals(), [class_name])
cls = getattr(mod, class_name)
if not cls.process_split_file(data):
sys.stderr.write('Writing split file failed\n')
sys.stderr.write("Writing split file failed\n")
sys.exit(1)
except Exception as e:
sys.stderr.write(str(e))
scripts/extract_toolbox_sections.py
+22 -23
View File
@@ -18,24 +18,24 @@ def main():
for rootchild in root:
currentsectionlabel = ""
if (rootchild.tag == "section"):
sectionname = rootchild.attrib['name']
if rootchild.tag == "section":
sectionname = rootchild.attrib["name"]
# per section tool index range 1-1000, current labels/tools
# divided between 20 and 750
toolindex = 250
toolfactor = int(500 / len(rootchild))
currentlabel = ""
for sectionchild in rootchild:
if (sectionchild.tag == "tool"):
if sectionchild.tag == "tool":
addToToolDict(sectionchild, sectionname, sectionindex, toolindex, currentlabel)
toolindex += toolfactor
elif (sectionchild.tag == "label"):
elif sectionchild.tag == "label":
currentlabel = sectionchild.attrib["text"]
sectionindex += sectionfactor
elif (rootchild.tag == "tool"):
elif rootchild.tag == "tool":
addToToolDict(rootchild, "", sectionindex, None, currentsectionlabel)
sectionindex += sectionfactor
elif (rootchild.tag == "label"):
elif rootchild.tag == "label":
currentsectionlabel = rootchild.attrib["text"]
sectionindex += sectionfactor
@@ -57,14 +57,14 @@ def main():
tooldocroot = tooldoc.getroot()
# check tags element, set flag
tagselement = tooldocroot.find("tags")
if (tagselement):
if tagselement:
hastags = True
# check if toolboxposition element already exists in this tooconfig file
toolboxposelement = tooldocroot.find("toolboxposition")
if (toolboxposelement):
if toolboxposelement:
hastoolboxpos = True
if (not (hastags and hastoolboxpos)):
if not (hastags and hastoolboxpos):
original = open(toolconffile)
contents = original.readlines()
original.close()
@@ -72,27 +72,26 @@ def main():
# the new elements will be added directly below the root tool element
addelementsatposition = 1
# but what's on the first line? Root or not?
if (contents[0].startswith("<?")):
if contents[0].startswith("<?"):
addelementsatposition = 2
newelements = []
if (not hastoolboxpos):
if (toolconffile in tooldict):
if not hastoolboxpos:
if toolconffile in tooldict:
for attributes in tooldict[toolconffile]:
# create toolboxposition element
sectionelement = ET.Element("toolboxposition")
sectionelement.attrib = attributes
sectionelement.tail = "\n "
newelements.append(ET.tostring(sectionelement, 'utf-8'))
newelements.append(ET.tostring(sectionelement, "utf-8"))
if (not hastags):
if not hastags:
# create empty tags element
newelements.append("<tags/>\n ")
contents = (contents[0:addelementsatposition] + newelements
+ contents[addelementsatposition:])
contents = contents[0:addelementsatposition] + newelements + contents[addelementsatposition:]
# add .new for testing/safety purposes :P
newtoolconffile = open(toolconffile, 'w')
newtoolconffile = open(toolconffile, "w")
newtoolconffile.writelines(contents)
newtoolconffile.close()
@@ -103,13 +102,13 @@ def addToToolDict(tool, sectionname, sectionindex, toolindex, currentlabel):
# define attributes for the toolboxposition xml-tag
attribdict = {}
if (sectionname):
if sectionname:
attribdict["section"] = sectionname
if (currentlabel):
if currentlabel:
attribdict["label"] = currentlabel
if (sectionindex):
if sectionindex:
attribdict["sectionorder"] = str(sectionindex)
if (toolindex):
if toolindex:
attribdict["order"] = str(toolindex)
tooldict[realtoolfile].append(attribdict)
@@ -120,14 +119,14 @@ def getfnl(startdir):
for root, _dirs, files in os.walk(startdir):
for fn in files:
fullfn = os.path.join(root, fn)
if fn.endswith('.xml'):
if fn.endswith(".xml"):
try:
doc = ET.parse(fullfn)
except Exception as e:
raise Exception(f"Oops, bad XML in '{fullfn}': {e}")
rootelement = doc.getroot()
# here we check if this xml file actually is a tool conf xml!
if rootelement.tag == 'tool':
if rootelement.tag == "tool":
filenamelist.append(fullfn)
return filenamelist
scripts/fetch_eggs.py
+3 -4
View File
@@ -4,7 +4,7 @@ from os import pardir
from os.path import (
abspath,
dirname,
join
join,
)
from sys import exit
@@ -30,7 +30,6 @@ cd {dir} && ./scripts/common_startup.sh --skip-venv
"""
galaxy = abspath(join(dirname(__file__), pardir))
venv = join(galaxy, '.venv')
print(msg.format(dir=abspath(join(dirname(__file__), pardir)),
venv=venv))
venv = join(galaxy, ".venv")
print(msg.format(dir=abspath(join(dirname(__file__), pardir)), venv=venv))
exit(1)
scripts/fix_dm_versions.py
+17 -14
View File
@@ -26,15 +26,18 @@ need to be corrected manually.
"""
parser = argparse.ArgumentParser(description=desc)
parser.add_argument('shed_data_manager_conf', metavar='CONFIG_FILE', type=str,
default="config/shed_data_manager_conf.xml",
help='an integer for the accumulator')
parser.add_argument('--all-entries', action='store_true',
help='modify all entries (default only those with duplicated guid)')
parser.add_argument('--add-version', action='store_true',
help='also add version attribute if absent')
parser.add_argument('--dry-run', action='store_true',
help='do not write resulting config file')
parser.add_argument(
"shed_data_manager_conf",
metavar="CONFIG_FILE",
type=str,
default="config/shed_data_manager_conf.xml",
help="an integer for the accumulator",
)
parser.add_argument(
"--all-entries", action="store_true", help="modify all entries (default only those with duplicated guid)"
)
parser.add_argument("--add-version", action="store_true", help="also add version attribute if absent")
parser.add_argument("--dry-run", action="store_true", help="do not write resulting config file")
args = parser.parse_args()
with open(args.shed_data_manager_conf) as fh:
@@ -42,7 +45,7 @@ with open(args.shed_data_manager_conf) as fh:
root = tree.getroot()
guid_mapping = dict()
for dm in root.iter('data_manager'):
for dm in root.iter("data_manager"):
guid = dm.attrib["guid"]
if guid not in guid_mapping:
guid_mapping[guid] = [dm]
@@ -51,7 +54,7 @@ for dm in root.iter('data_manager'):
for guid in guid_mapping:
if len(guid_mapping[guid]) > 1:
print(f'{guid} found {len(guid_mapping[guid])}x')
print(f"{guid} found {len(guid_mapping[guid])}x")
elif not args.all_entries:
continue
@@ -60,14 +63,14 @@ for guid in guid_mapping:
tool_version = tool_version.text
new_guid = f"{guid[:guid.rfind('/')]}/{tool_version}"
dm.attrib['guid'] = new_guid
dm.attrib["guid"] = new_guid
print(f"changing guid: {guid} -> {new_guid}")
if "version" in dm.attrib:
print(f"changing version: {dm.attrib['version']} -> {tool_version}")
dm.attrib['version'] = tool_version
dm.attrib["version"] = tool_version
elif args.add_version:
print(f"adding version: {tool_version}")
dm.attrib['version'] = tool_version
dm.attrib["version"] = tool_version
if not args.dry_run:
nfn = args.shed_data_manager_conf + datetime.now().isoformat()
scripts/functional_tests.py
+11 -6
View File
@@ -13,7 +13,10 @@ galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pa
sys.path[1:1] = [os.path.join(galaxy_root, "lib"), os.path.join(galaxy_root, "test")]
from galaxy.util import classproperty
from galaxy_test.base.api_util import get_admin_api_key, get_user_api_key
from galaxy_test.base.api_util import (
get_admin_api_key,
get_user_api_key,
)
from galaxy_test.driver import driver_util
log = driver_util.build_logger()
@@ -65,6 +68,7 @@ class SeleniumGalaxyTestDriver(driver_util.GalaxyTestDriver):
@classproperty
def default_web_host(cls):
from galaxy_test.selenium.framework import default_web_host_for_selenium_tests
return default_web_host_for_selenium_tests()
@@ -82,6 +86,7 @@ class DataManagersGalaxyTestDriver(driver_util.GalaxyTestDriver):
def build_tests(self):
"""Build data manager test methods."""
import functional.test_data_managers
functional.test_data_managers.data_managers = self.app.data_managers
functional.test_data_managers.build_tests(
tmp_dir=self.galaxy_test_tmp_dir,
@@ -94,11 +99,11 @@ class DataManagersGalaxyTestDriver(driver_util.GalaxyTestDriver):
TEST_DRIVERS = {
'-migrated': MigratedToolsGalaxyTestDriver,
'-installed': InstalledToolsGalaxyTestDriver,
'-framework': FrameworkToolsGalaxyTestDriver,
'-data_managers': DataManagersGalaxyTestDriver,
'-selenium': SeleniumGalaxyTestDriver,
"-migrated": MigratedToolsGalaxyTestDriver,
"-installed": InstalledToolsGalaxyTestDriver,
"-framework": FrameworkToolsGalaxyTestDriver,
"-data_managers": DataManagersGalaxyTestDriver,
"-selenium": SeleniumGalaxyTestDriver,
}
scripts/galaxy_main.py
+39 -38
View File
@@ -46,7 +46,7 @@ log = logging.getLogger(__name__)
real_file = os.path.realpath(__file__)
GALAXY_ROOT_DIR_ = os.path.abspath(os.path.join(os.path.dirname(real_file), os.pardir))
if not os.path.exists(os.path.join(GALAXY_ROOT_DIR_, 'run.sh')):
if not os.path.exists(os.path.join(GALAXY_ROOT_DIR_, "run.sh")):
# Galaxy is installed
GALAXY_ROOT_DIR = None
else:
@@ -73,20 +73,14 @@ DEFAULT_PID = "galaxy.pid"
DEFAULT_VERBOSE = True
DESCRIPTION = "Daemonized entry point for Galaxy."
SHUTDOWN_MSG = '__SHUTDOWN__'
UWSGI_FARMS_VAR = '_GALAXY_UWSGI_FARM_NAMES'
SHUTDOWN_MSG = "__SHUTDOWN__"
UWSGI_FARMS_VAR = "_GALAXY_UWSGI_FARM_NAMES"
exit = threading.Event()
def load_galaxy_app(
config_builder,
config_env=False,
log=None,
attach_to_pools=None,
**kwds
):
def load_galaxy_app(config_builder, config_env=False, log=None, attach_to_pools=None, **kwds):
# Allow specification of log so daemon can reuse properly configured one.
if log is None:
log = logging.getLogger(__name__)
@@ -102,25 +96,23 @@ def load_galaxy_app(
config_builder.setup_logging()
from galaxy.util.properties import load_app_properties
kwds = config_builder.app_kwds()
kwds = load_app_properties(**kwds)
from galaxy.app import UniverseApplication
app = UniverseApplication(
global_conf=config_builder.global_conf(),
attach_to_pools=attach_to_pools,
**kwds
)
app = UniverseApplication(global_conf=config_builder.global_conf(), attach_to_pools=attach_to_pools, **kwds)
app.database_heartbeat.start()
app.application_stack.log_startup()
return app
def handle_signal(signum, frame):
log.info('Received signal %d, exiting', signum)
if uwsgi and 'mule_id' in dir(uwsgi) and uwsgi.mule_id() > 0:
log.info("Received signal %d, exiting", signum)
if uwsgi and "mule_id" in dir(uwsgi) and uwsgi.mule_id() > 0:
farms = os.environ.get(UWSGI_FARMS_VAR, None)
if farms:
for farm in farms.split(','):
for farm in farms.split(","):
uwsgi.farm_msg(farm, SHUTDOWN_MSG)
else:
uwsgi.mule_msg(SHUTDOWN_MSG, uwsgi.mule_id())
@@ -128,8 +120,8 @@ def handle_signal(signum, frame):
def register_signals():
for name in ('TERM', 'INT', 'HUP'):
sig = getattr(signal, f'SIG{name}')
for name in ("TERM", "INT", "HUP"):
sig = getattr(signal, f"SIG{name}")
signal.signal(sig, handle_signal)
@@ -160,8 +152,7 @@ def app_loop(args, log):
class GalaxyConfigBuilder:
""" Generate paste-like configuration from supplied command-line arguments.
"""
"""Generate paste-like configuration from supplied command-line arguments."""
def __init__(self, args=None, **kwds):
self.config_file = None
@@ -170,8 +161,8 @@ class GalaxyConfigBuilder:
config_file = kwds.get("config_file", None) or (args and args.config_file)
# If given app_conf_path - use that - else we need to ensure we have a
# config file path.
if not config_file and 'config_file' in self.app_kwds():
config_file = self.app_kwds()['config_file']
if not config_file and "config_file" in self.app_kwds():
config_file = self.app_kwds()["config_file"]
if not config_file:
galaxy_root = kwds.get("galaxy_root", GALAXY_ROOT_DIR)
config_file = find_config(config_file, galaxy_root)
@@ -182,19 +173,32 @@ class GalaxyConfigBuilder:
self.config_section = f"app:{unicodify(kwds.get('app') or args and args.app or DEFAULT_INI_APP)}"
else:
self.config_section = self.app_name
self.log_file = (args and args.log_file)
self.log_file = args and args.log_file
@classmethod
def populate_options(cls, arg_parser):
arg_parser.add_argument("-c", "--config-file", default=None, help="Galaxy config file (defaults to config/galaxy.ini)")
arg_parser.add_argument(
"-c", "--config-file", default=None, help="Galaxy config file (defaults to config/galaxy.ini)"
)
arg_parser.add_argument("--ini-path", default=None, help="DEPRECATED: use -c/--config-file")
arg_parser.add_argument("--app", default=None, help="app section in config file (defaults to 'galaxy' for YAML/JSON, 'main' (w/ 'app:' prepended) for INI")
arg_parser.add_argument(
"--app",
default=None,
help="app section in config file (defaults to 'galaxy' for YAML/JSON, 'main' (w/ 'app:' prepended) for INI",
)
arg_parser.add_argument("-d", "--daemonize", default=False, help="Daemonize process", action="store_true")
arg_parser.add_argument("--daemon-log-file", default=None, help="log file for daemon script ")
arg_parser.add_argument("--log-file", default=None, help="Galaxy log file (overrides log configuration in config_file if set)")
arg_parser.add_argument(
"--log-file", default=None, help="Galaxy log file (overrides log configuration in config_file if set)"
)
arg_parser.add_argument("--pid-file", default=DEFAULT_PID, help=f"pid file (default is {DEFAULT_PID})")
arg_parser.add_argument("--server-name", default=None, help="set a galaxy server name")
arg_parser.add_argument("--attach-to-pool", action="append", default=None, help="attach to asynchronous worker pool (specify multiple times for multiple pools)")
arg_parser.add_argument(
"--attach-to-pool",
action="append",
default=None,
help="attach to asynchronous worker pool (specify multiple times for multiple pools)",
)
@property
def config_is_ini(self):
@@ -202,10 +206,10 @@ class GalaxyConfigBuilder:
def app_kwds(self):
kwds = get_app_kwds(self.app_name, app_name=self.app_name)
if 'config_file' not in kwds:
kwds['config_file'] = self.config_file
if 'config_section' not in kwds:
kwds['config_section'] = self.config_section
if "config_file" not in kwds:
kwds["config_file"] = self.config_file
if "config_section" not in kwds:
kwds["config_section"] = self.config_section
return kwds
def global_conf(self):
@@ -223,12 +227,9 @@ class GalaxyConfigBuilder:
if self.config_is_ini:
raw_config = ConfigParser()
raw_config.read([self.config_file])
if raw_config.has_section('loggers'):
if raw_config.has_section("loggers"):
config_file = os.path.abspath(self.config_file)
fileConfig(
config_file,
dict(__file__=config_file, here=os.path.dirname(config_file))
)
fileConfig(config_file, dict(__file__=config_file, here=os.path.dirname(config_file)))
def main(func=app_loop):
scripts/galaxy_schemadisp.py
+17 -11
View File
@@ -4,37 +4,43 @@ import sys
from db_shell import * # noqa
from sqlalchemy import MetaData
from sqlalchemy.orm import class_mapper
try:
from sqlalchemy_schemadisplay import create_schema_graph, create_uml_graph
from sqlalchemy_schemadisplay import (
create_schema_graph,
create_uml_graph,
)
except ImportError:
print("please install sqlalchemy_schemadisplay to use this script (pip install sqlalchemy_schemadisplay)")
raise
gxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
sys.path.insert(1, os.path.abspath(os.path.join(gxy_root, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(gxy_root, "lib")))
from galaxy import model
if __name__ == "__main__":
gxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
sqlitedb = os.path.join(gxy_root, 'database/universe.sqlite')
sqlitedb = os.path.join(gxy_root, "database/universe.sqlite")
# Try to build a representation of what's in the sqlite database
if os.path.exists(sqlitedb):
graph = create_schema_graph(metadata=MetaData('sqlite:///' + sqlitedb),
show_datatypes=False,
show_indexes=False,
rankdir='LR',
concentrate=False)
graph = create_schema_graph(
metadata=MetaData("sqlite:///" + sqlitedb),
show_datatypes=False,
show_indexes=False,
rankdir="LR",
concentrate=False,
)
print(f"Writing galaxy_universe.png, built from {sqlitedb}")
graph.write_png('galaxy_universe.png')
graph.write_png("galaxy_universe.png")
else:
print(f"No sqlitedb available at {sqlitedb}, skipping rendering")
# Build UML graph from loaded mapper
mappers = []
for attr in dir(model):
if attr[0] == '_':
if attr[0] == "_":
continue
try:
cls = getattr(model, attr)
@@ -47,4 +53,4 @@ if __name__ == "__main__":
show_operations=False,
)
print("Writing galaxy_uml.png")
graph.write_png('galaxy_uml.png') # write out the file
graph.write_png("galaxy_uml.png") # write out the file
scripts/get_platforms.py
+1
View File
@@ -9,4 +9,5 @@ lib = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib"))
sys.path.insert(1, lib)
import pkg_resources
print(pkg_resources.get_platform())
scripts/get_uwsgi_args.py
+61 -50
View File
@@ -2,32 +2,44 @@ import os
import shlex
import sys
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib")))
from galaxy.util.path import get_ext
from galaxy.util.properties import load_app_properties, nice_config_parser
from galaxy.util.properties import (
load_app_properties,
nice_config_parser,
)
from galaxy.util.script import main_factory
DESCRIPTION = "Script to determine uWSGI command line arguments"
# socket is not an alias for http, but it is assumed that if you configure a socket in your uwsgi config you do not
# want to run the default http server (or you can configure it yourself)
ALIASES = {
'virtualenv': ('home', 'venv', 'pyhome'),
'pythonpath': ('python-path', 'pp'),
'http': ('httprouter', 'socket', 'uwsgi-socket', 'suwsgi-socket', 'ssl-socket'),
'module': ('mount',), # mount is not actually an alias for module, but we don't want to set module if mount is set
"virtualenv": ("home", "venv", "pyhome"),
"pythonpath": ("python-path", "pp"),
"http": ("httprouter", "socket", "uwsgi-socket", "suwsgi-socket", "ssl-socket"),
"module": ("mount",), # mount is not actually an alias for module, but we don't want to set module if mount is set
}
DEFAULT_ARGS = {
'_all_': ('pythonpath', 'threads', 'buffer-size', 'http', 'static-map', 'die-on-term', 'hook-master-start', 'enable-threads', 'umask'),
'galaxy': ('py-call-osafterfork',),
'reports': (),
'tool_shed': ('cron',),
"_all_": (
"pythonpath",
"threads",
"buffer-size",
"http",
"static-map",
"die-on-term",
"hook-master-start",
"enable-threads",
"umask",
),
"galaxy": ("py-call-osafterfork",),
"reports": (),
"tool_shed": ("cron",),
}
DEFAULT_PORTS = {
'galaxy': 8080,
'reports': 9001,
'tool_shed': 9009,
"galaxy": 8080,
"reports": 9001,
"tool_shed": 9009,
}
@@ -41,13 +53,13 @@ def __arg_set(arg, kwargs):
def __add_arg(args, arg, value):
optarg = '--%s' % arg
optarg = "--%s" % arg
if isinstance(value, bool):
if value is True:
args.append(optarg)
elif isinstance(value, str):
# the = in --optarg=value is usually, but not always, optional
if value.startswith('='):
if value.startswith("="):
args.append(shlex.quote(optarg + value))
else:
args.append(optarg)
@@ -60,65 +72,64 @@ def __add_config_file_arg(args, config_file, app):
ext = None
if config_file:
ext = get_ext(config_file)
if ext in ('yaml', 'json'):
if ext in ("yaml", "json"):
__add_arg(args, ext, config_file)
elif ext == 'ini':
elif ext == "ini":
config = nice_config_parser(config_file)
has_logging = config.has_section('loggers')
if config.has_section('app:main'):
has_logging = config.has_section("loggers")
if config.has_section("app:main"):
# uWSGI does not have any way to set the app name when loading with paste.deploy:loadapp(), so hardcoding
# the name to `main` is fine
__add_arg(args, 'ini-paste' if not has_logging else 'ini-paste-logged', config_file)
__add_arg(args, "ini-paste" if not has_logging else "ini-paste-logged", config_file)
return # do not add --module
else:
__add_arg(args, ext, config_file)
if has_logging:
__add_arg(args, 'paste-logger', True)
__add_arg(args, "paste-logger", True)
def _get_uwsgi_args(cliargs, kwargs):
# it'd be nice if we didn't have to reparse here but we need things out of more than one section
config_file = cliargs.config_file or kwargs.get('__file__')
uwsgi_kwargs = load_app_properties(config_file=config_file, config_section='uwsgi')
config_file = cliargs.config_file or kwargs.get("__file__")
uwsgi_kwargs = load_app_properties(config_file=config_file, config_section="uwsgi")
args = []
ts_cron_config_option = '' if config_file is None else '-c %s' % config_file
ts_cron_config_option = "" if config_file is None else "-c %s" % config_file
defaults = {
'pythonpath': 'lib',
'threads': '4',
'buffer-size': '16384', # https://github.com/galaxyproject/galaxy/issues/1530
'http': f'localhost:{DEFAULT_PORTS[cliargs.app]}',
'static-map': (f'/static={os.getcwd()}/static',
f'/favicon.ico={os.getcwd()}/static/favicon.ico'),
'die-on-term': True,
'enable-threads': True,
'hook-master-start': ('unix_signal:2 gracefully_kill_them_all',
'unix_signal:15 gracefully_kill_them_all'),
'py-call-osafterfork': True,
'cron': '0 -1 -1 -1 -1 python scripts/tool_shed/build_ts_whoosh_index.py %s --config-section tool_shed -d' % ts_cron_config_option,
'umask': '027',
"pythonpath": "lib",
"threads": "4",
"buffer-size": "16384", # https://github.com/galaxyproject/galaxy/issues/1530
"http": f"localhost:{DEFAULT_PORTS[cliargs.app]}",
"static-map": (f"/static={os.getcwd()}/static", f"/favicon.ico={os.getcwd()}/static/favicon.ico"),
"die-on-term": True,
"enable-threads": True,
"hook-master-start": ("unix_signal:2 gracefully_kill_them_all", "unix_signal:15 gracefully_kill_them_all"),
"py-call-osafterfork": True,
"cron": "0 -1 -1 -1 -1 python scripts/tool_shed/build_ts_whoosh_index.py %s --config-section tool_shed -d"
% ts_cron_config_option,
"umask": "027",
}
__add_config_file_arg(args, config_file, cliargs.app)
if not __arg_set('module', uwsgi_kwargs):
if not __arg_set("module", uwsgi_kwargs):
if cliargs.app in ["tool_shed"]:
__add_arg(args, 'module', 'tool_shed.webapp.buildapp:uwsgi_app()')
__add_arg(args, "module", "tool_shed.webapp.buildapp:uwsgi_app()")
else:
__add_arg(args, 'module', f'galaxy.webapps.{cliargs.app}.buildapp:uwsgi_app()')
__add_arg(args, "module", f"galaxy.webapps.{cliargs.app}.buildapp:uwsgi_app()")
# only include virtualenv if it's set/exists, otherwise this breaks conda-env'd Galaxy
if not __arg_set('virtualenv', uwsgi_kwargs) and ('VIRTUAL_ENV' in os.environ or os.path.exists('.venv')):
__add_arg(args, 'virtualenv', os.environ.get('VIRTUAL_ENV', '.venv'))
if not __arg_set("virtualenv", uwsgi_kwargs) and ("VIRTUAL_ENV" in os.environ or os.path.exists(".venv")):
__add_arg(args, "virtualenv", os.environ.get("VIRTUAL_ENV", ".venv"))
# We always want to append client/src/assets as static-safe.
__add_arg(args, 'static-safe', f'{os.getcwd()}/client/src/assets')
__add_arg(args, "static-safe", f"{os.getcwd()}/client/src/assets")
# Do not let uwsgi remap stdin to /dev/null if galaxy is in debug mode
galaxy_kwargs = load_app_properties(config_file=config_file, config_section='galaxy')
if __arg_set('debug', galaxy_kwargs) and not __arg_set('honour-stdin', uwsgi_kwargs):
__add_arg(args, 'honour-stdin', True)
galaxy_kwargs = load_app_properties(config_file=config_file, config_section="galaxy")
if __arg_set("debug", galaxy_kwargs) and not __arg_set("honour-stdin", uwsgi_kwargs):
__add_arg(args, "honour-stdin", True)
for arg in DEFAULT_ARGS['_all_'] + DEFAULT_ARGS[cliargs.app]:
for arg in DEFAULT_ARGS["_all_"] + DEFAULT_ARGS[cliargs.app]:
if not __arg_set(arg, uwsgi_kwargs):
__add_arg(args, arg, defaults[arg])
print(' '.join(args))
print(" ".join(args))
ACTIONS = {
@@ -126,6 +137,6 @@ ACTIONS = {
}
if __name__ == '__main__':
if __name__ == "__main__":
main = main_factory(description=DESCRIPTION, actions=ACTIONS, default_action="get_uwsgi_args")
main()
scripts/grt/export.py
+150 -103
View File
@@ -14,19 +14,22 @@ from collections import defaultdict
import yaml
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "lib")))
import galaxy
import galaxy.app
import galaxy.config
from galaxy.objectstore import build_object_store_from_config
from galaxy.util import (
hash_util,
unicodify
unicodify,
)
from galaxy.util.script import (
app_properties_from_args,
populate_config_args,
)
from galaxy.util.script import app_properties_from_args, populate_config_args
sample_config = os.path.abspath(os.path.join(os.path.dirname(__file__), 'grt.yml.sample'))
default_config = os.path.abspath(os.path.join(os.path.dirname(__file__), 'grt.yml'))
sample_config = os.path.abspath(os.path.join(os.path.dirname(__file__), "grt.yml.sample"))
default_config = os.path.abspath(os.path.join(os.path.dirname(__file__), "grt.yml"))
def _init(args):
@@ -34,7 +37,9 @@ def _init(args):
config = galaxy.config.Configuration(**properties)
object_store = build_object_store_from_config(config)
if not config.database_connection:
logging.warning("The database connection is empty. If you are using the default value, please uncomment that in your galaxy.yml")
logging.warning(
"The database connection is empty. If you are using the default value, please uncomment that in your galaxy.yml"
)
model = galaxy.config.init_models_from_config(config, object_store=object_store)
return (
@@ -45,31 +50,40 @@ def _init(args):
def kw_metrics(job):
return {
f'{metric.plugin}_{metric.metric_name}': metric.metric_value
for metric in job.metrics
}
return {f"{metric.plugin}_{metric.metric_name}": metric.metric_value for metric in job.metrics}
def round_to_2sd(number):
if number:
return str(int(float('%.2g' % number)))
return str(int(float("%.2g" % number)))
else:
return '-1'
return "-1"
def main(argv):
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-r', '--report-directory', help='Directory to store reports in',
default=os.path.abspath(os.path.join('.', 'reports')))
parser.add_argument('-g', '--grt-config', help='Path to GRT config file',
default=default_config)
parser.add_argument("-l", "--loglevel", choices=['debug', 'info', 'warning', 'error', 'critical'],
help="Set the logging level", default='warning')
parser.add_argument("-b", "--batch-size", type=int, default=1000,
help="Batch size for sql queries")
parser.add_argument("-m", "--max-records", type=int, default=5000000,
help="Maximum number of records to include in a single report. This option should ONLY be used when reporting historical data. Setting this may require running GRT multiple times to capture all historical logs.")
parser.add_argument(
"-r",
"--report-directory",
help="Directory to store reports in",
default=os.path.abspath(os.path.join(".", "reports")),
)
parser.add_argument("-g", "--grt-config", help="Path to GRT config file", default=default_config)
parser.add_argument(
"-l",
"--loglevel",
choices=["debug", "info", "warning", "error", "critical"],
help="Set the logging level",
default="warning",
)
parser.add_argument("-b", "--batch-size", type=int, default=1000, help="Batch size for sql queries")
parser.add_argument(
"-m",
"--max-records",
type=int,
default=5000000,
help="Maximum number of records to include in a single report. This option should ONLY be used when reporting historical data. Setting this may require running GRT multiple times to capture all historical logs.",
)
populate_config_args(parser)
args = parser.parse_args()
@@ -83,18 +97,18 @@ def main(argv):
logging.info(human_label)
_times.append((label, time.time() - _start_time))
annotate('init_start', 'Loading GRT configuration...')
annotate("init_start", "Loading GRT configuration...")
try:
with open(args.grt_config) as handle:
config = yaml.safe_load(handle)
except Exception:
logging.info('Using default GRT configuration')
logging.info("Using default GRT configuration")
with open(sample_config) as handle:
config = yaml.safe_load(handle)
annotate('init_end')
annotate("init_end")
REPORT_DIR = args.report_directory
CHECK_POINT_FILE = os.path.join(REPORT_DIR, '.checkpoint')
CHECK_POINT_FILE = os.path.join(REPORT_DIR, ".checkpoint")
REPORT_IDENTIFIER = str(time.time())
REPORT_BASE = os.path.join(REPORT_DIR, REPORT_IDENTIFIER)
@@ -104,13 +118,13 @@ def main(argv):
else:
last_job_sent = -1
annotate('galaxy_init', 'Loading Galaxy...')
annotate("galaxy_init", "Loading Galaxy...")
model, object_store, gxconfig = _init(args)
# Galaxy overrides our logging level.
logging.getLogger().setLevel(getattr(logging, args.loglevel.upper()))
sa_session = model.context.current
annotate('galaxy_end')
annotate("galaxy_end")
# Fetch jobs COMPLETED with status OK that have not yet been sent.
@@ -122,17 +136,15 @@ def main(argv):
os.makedirs(REPORT_DIR)
# Pick an end point so our queries can return uniform data.
annotate('endpoint_start', 'Identifying a safe endpoint for SQL queries')
end_job_id = sa_session.query(model.Job.id) \
.order_by(model.Job.id.desc()) \
.first()[0]
annotate("endpoint_start", "Identifying a safe endpoint for SQL queries")
end_job_id = sa_session.query(model.Job.id).order_by(model.Job.id.desc()).first()[0]
# Allow users to only report N records at once.
if args.max_records > 0:
if end_job_id - last_job_sent > args.max_records:
end_job_id = last_job_sent + args.max_records
annotate('endpoint_end', f'Processing jobs ({last_job_sent}, {end_job_id}]')
annotate("endpoint_end", f"Processing jobs ({last_job_sent}, {end_job_id}]")
# Remember the last job sent.
if end_job_id == last_job_sent:
@@ -142,17 +154,26 @@ def main(argv):
# Unfortunately we have to keep this mapping for the sanitizer to work properly.
job_tool_map = {}
blacklisted_tools = config['sanitization']['tools']
blacklisted_tools = config["sanitization"]["tools"]
annotate('export_jobs_start', 'Exporting Jobs')
with open(REPORT_BASE + '.jobs.tsv', 'w', encoding='utf-8') as handle_job:
handle_job.write('\t'.join(('id', 'tool_id', 'tool_version', 'state', 'create_time')) + '\n')
annotate("export_jobs_start", "Exporting Jobs")
with open(REPORT_BASE + ".jobs.tsv", "w", encoding="utf-8") as handle_job:
handle_job.write("\t".join(("id", "tool_id", "tool_version", "state", "create_time")) + "\n")
for offset_start in range(last_job_sent, end_job_id, args.batch_size):
logging.debug("Processing %s:%s", offset_start, min(end_job_id, offset_start + args.batch_size))
for job in sa_session.query(model.Job.id, model.Job.user_id, model.Job.tool_id, model.Job.tool_version, model.Job.state, model.Job.create_time) \
.filter(model.Job.id > offset_start) \
.filter(model.Job.id <= min(end_job_id, offset_start + args.batch_size)) \
.all():
for job in (
sa_session.query(
model.Job.id,
model.Job.user_id,
model.Job.tool_id,
model.Job.tool_version,
model.Job.state,
model.Job.create_time,
)
.filter(model.Job.id > offset_start)
.filter(model.Job.id <= min(end_job_id, offset_start + args.batch_size))
.all()
):
# If the tool is blacklisted, exclude everywhere
if job[2] in blacklisted_tools:
continue
@@ -163,9 +184,9 @@ def main(argv):
job[2], # tool_id
job[3], # tool_version
job[4], # state
str(job[5]) # create_time
str(job[5]), # create_time
]
cline = unicodify('\t'.join(line) + '\n')
cline = unicodify("\t".join(line) + "\n")
handle_job.write(cline)
except Exception:
logging.warning("Unable to write out a 'handle_job' row. Ignoring the row.", exc_info=True)
@@ -174,46 +195,67 @@ def main(argv):
job_state_data[job[4]] += 1
active_users[job[1]] += 1
job_tool_map[job[0]] = job[2]
annotate('export_jobs_end')
annotate("export_jobs_end")
annotate('export_datasets_start', 'Exporting Datasets')
with open(REPORT_BASE + '.datasets.tsv', 'w', encoding='utf-8') as handle_datasets:
handle_datasets.write('\t'.join(('job_id', 'dataset_id', 'extension', 'file_size', 'param_name', 'type')) + '\n')
annotate("export_datasets_start", "Exporting Datasets")
with open(REPORT_BASE + ".datasets.tsv", "w", encoding="utf-8") as handle_datasets:
handle_datasets.write(
"\t".join(("job_id", "dataset_id", "extension", "file_size", "param_name", "type")) + "\n"
)
for offset_start in range(last_job_sent, end_job_id, args.batch_size):
logging.debug("Processing %s:%s", offset_start, min(end_job_id, offset_start + args.batch_size))
# four queries: JobToInputDatasetAssociation, JobToOutputDatasetAssociation, HistoryDatasetAssociation, Dataset
job_to_input_hda_ids = sa_session.query(model.JobToInputDatasetAssociation.job_id, model.JobToInputDatasetAssociation.dataset_id,
model.JobToInputDatasetAssociation.name) \
.filter(model.JobToInputDatasetAssociation.job_id > offset_start) \
.filter(model.JobToInputDatasetAssociation.job_id <= min(end_job_id, offset_start + args.batch_size)) \
job_to_input_hda_ids = (
sa_session.query(
model.JobToInputDatasetAssociation.job_id,
model.JobToInputDatasetAssociation.dataset_id,
model.JobToInputDatasetAssociation.name,
)
.filter(model.JobToInputDatasetAssociation.job_id > offset_start)
.filter(model.JobToInputDatasetAssociation.job_id <= min(end_job_id, offset_start + args.batch_size))
.all()
)
job_to_output_hda_ids = sa_session.query(model.JobToOutputDatasetAssociation.job_id, model.JobToOutputDatasetAssociation.dataset_id,
model.JobToOutputDatasetAssociation.name) \
.filter(model.JobToOutputDatasetAssociation.job_id > offset_start) \
.filter(model.JobToOutputDatasetAssociation.job_id <= min(end_job_id, offset_start + args.batch_size)) \
job_to_output_hda_ids = (
sa_session.query(
model.JobToOutputDatasetAssociation.job_id,
model.JobToOutputDatasetAssociation.dataset_id,
model.JobToOutputDatasetAssociation.name,
)
.filter(model.JobToOutputDatasetAssociation.job_id > offset_start)
.filter(model.JobToOutputDatasetAssociation.job_id <= min(end_job_id, offset_start + args.batch_size))
.all()
)
# add type and concat
job_to_hda_ids = [[list(i), "input"] for i in job_to_input_hda_ids] + [[list(i), "output"] for i in job_to_output_hda_ids]
job_to_hda_ids = [[list(i), "input"] for i in job_to_input_hda_ids] + [
[list(i), "output"] for i in job_to_output_hda_ids
]
# put all of the hda_ids into a list
hda_ids = [i[0][1] for i in job_to_hda_ids]
hdas = sa_session.query(model.HistoryDatasetAssociation.id, model.HistoryDatasetAssociation.dataset_id,
model.HistoryDatasetAssociation.extension) \
.filter(model.HistoryDatasetAssociation.id.in_(hda_ids)) \
hdas = (
sa_session.query(
model.HistoryDatasetAssociation.id,
model.HistoryDatasetAssociation.dataset_id,
model.HistoryDatasetAssociation.extension,
)
.filter(model.HistoryDatasetAssociation.id.in_(hda_ids))
.all()
)
# put all the dataset ids into a list
dataset_ids = [i[1] for i in hdas]
# get the sizes of the datasets
datasets = sa_session.query(model.Dataset.id, model.Dataset.total_size) \
.filter(model.Dataset.id.in_(dataset_ids)) \
datasets = (
sa_session.query(model.Dataset.id, model.Dataset.total_size)
.filter(model.Dataset.id.in_(dataset_ids))
.all()
)
# datasets to dictionay for easy search
hdas = {i[0]: i[1:] for i in hdas}
@@ -247,24 +289,31 @@ def main(argv):
str(hdas[hda_id][1]), # Extension
round_to_2sd(datasets[dataset_id][0]), # File size
job[2], # Parameter name
str(filetype) # input/output
str(filetype), # input/output
]
cline = unicodify('\t'.join(line) + '\n')
cline = unicodify("\t".join(line) + "\n")
handle_datasets.write(cline)
except Exception:
logging.warning("Unable to write out a 'handle_datasets' row. Ignoring the row.", exc_info=True)
continue
annotate('export_datasets_end')
annotate("export_datasets_end")
annotate('export_metric_num_start', 'Exporting Metrics (Numeric)')
with open(REPORT_BASE + '.metric_num.tsv', 'w', encoding='utf-8') as handle_metric_num:
handle_metric_num.write('\t'.join(('job_id', 'plugin', 'name', 'value')) + '\n')
annotate("export_metric_num_start", "Exporting Metrics (Numeric)")
with open(REPORT_BASE + ".metric_num.tsv", "w", encoding="utf-8") as handle_metric_num:
handle_metric_num.write("\t".join(("job_id", "plugin", "name", "value")) + "\n")
for offset_start in range(last_job_sent, end_job_id, args.batch_size):
logging.debug("Processing %s:%s", offset_start, min(end_job_id, offset_start + args.batch_size))
for metric in sa_session.query(model.JobMetricNumeric.job_id, model.JobMetricNumeric.plugin, model.JobMetricNumeric.metric_name, model.JobMetricNumeric.metric_value) \
.filter(model.JobMetricNumeric.job_id > offset_start) \
.filter(model.JobMetricNumeric.job_id <= min(end_job_id, offset_start + args.batch_size)) \
.all():
for metric in (
sa_session.query(
model.JobMetricNumeric.job_id,
model.JobMetricNumeric.plugin,
model.JobMetricNumeric.metric_name,
model.JobMetricNumeric.metric_value,
)
.filter(model.JobMetricNumeric.job_id > offset_start)
.filter(model.JobMetricNumeric.job_id <= min(end_job_id, offset_start + args.batch_size))
.all()
):
# No associated job
if metric[0] not in job_tool_map:
continue
@@ -273,57 +322,55 @@ def main(argv):
continue
try:
line = [
str(metric[0]), # job id
metric[1], # plugin
metric[2], # name
str(metric[3]) # value
]
line = [str(metric[0]), metric[1], metric[2], str(metric[3])] # job id # plugin # name # value
cline = unicodify('\t'.join(line) + '\n')
cline = unicodify("\t".join(line) + "\n")
handle_metric_num.write(cline)
except Exception:
logging.warning("Unable to write out a 'handle_metric_num' row. Ignoring the row.", exc_info=True)
continue
annotate('export_metric_num_end')
annotate("export_metric_num_end")
# Now on to outputs.
with tarfile.open(REPORT_BASE + '.tar.gz', 'w:gz') as handle:
for name in ('jobs', 'metric_num', 'datasets'):
path = REPORT_BASE + '.' + name + '.tsv'
with tarfile.open(REPORT_BASE + ".tar.gz", "w:gz") as handle:
for name in ("jobs", "metric_num", "datasets"):
path = REPORT_BASE + "." + name + ".tsv"
if os.path.exists(path):
handle.add(path)
for name in ('jobs', 'metric_num', 'datasets'):
path = REPORT_BASE + '.' + name + '.tsv'
for name in ("jobs", "metric_num", "datasets"):
path = REPORT_BASE + "." + name + ".tsv"
if os.path.exists(path):
os.unlink(REPORT_BASE + '.' + name + '.tsv')
os.unlink(REPORT_BASE + "." + name + ".tsv")
_times.append(('job_finish', time.time() - _start_time))
_times.append(("job_finish", time.time() - _start_time))
sha = hash_util.memory_bound_hexdigest(hash_func=hash_util.sha256, path=REPORT_BASE + ".tar.gz")
_times.append(('hash_finish', time.time() - _start_time))
_times.append(("hash_finish", time.time() - _start_time))
# Now serialize the individual report data.
with open(REPORT_BASE + '.json', 'w') as handle:
json.dump({
"version": 3,
"galaxy_version": gxconfig.version_major,
"generated": REPORT_IDENTIFIER,
"report_hash": "sha256:" + sha,
"metrics": {
"_times": _times,
with open(REPORT_BASE + ".json", "w") as handle:
json.dump(
{
"version": 3,
"galaxy_version": gxconfig.version_major,
"generated": REPORT_IDENTIFIER,
"report_hash": "sha256:" + sha,
"metrics": {
"_times": _times,
},
"users": {
"active": len(active_users.keys()),
"total": sa_session.query(model.User.id).count(),
},
"jobs": job_state_data,
},
"users": {
"active": len(active_users.keys()),
"total": sa_session.query(model.User.id).count(),
},
"jobs": job_state_data,
}, handle)
handle,
)
# Write our checkpoint file so we know where to start next time.
with open(CHECK_POINT_FILE, 'w') as handle:
with open(CHECK_POINT_FILE, "w") as handle:
handle.write(str(end_job_id))
if __name__ == '__main__':
if __name__ == "__main__":
main(sys.argv)
scripts/grt/upload.py
+30 -26
View File
@@ -11,64 +11,68 @@ import sys
import requests
import yaml
sample_config = os.path.abspath(os.path.join(os.path.dirname(__file__), 'grt.yml.sample'))
default_config = os.path.abspath(os.path.join(os.path.dirname(__file__), 'grt.yml'))
sample_config = os.path.abspath(os.path.join(os.path.dirname(__file__), "grt.yml.sample"))
default_config = os.path.abspath(os.path.join(os.path.dirname(__file__), "grt.yml"))
def main(argv):
parser = argparse.ArgumentParser()
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-r', '--report-directory', help='Directory in which reports are stored',
default=os.path.abspath(os.path.join('.', 'reports')))
parser.add_argument('-g', '--grt-config', help='Path to GRT config file',
default=default_config)
parser.add_argument("-l", "--loglevel", choices=['debug', 'info', 'warning', 'error', 'critical'],
help="Set the logging level", default='warning')
parser.add_argument(
"-r",
"--report-directory",
help="Directory in which reports are stored",
default=os.path.abspath(os.path.join(".", "reports")),
)
parser.add_argument("-g", "--grt-config", help="Path to GRT config file", default=default_config)
parser.add_argument(
"-l",
"--loglevel",
choices=["debug", "info", "warning", "error", "critical"],
help="Set the logging level",
default="warning",
)
args = parser.parse_args()
logging.getLogger().setLevel(getattr(logging, args.loglevel.upper()))
logging.info('Loading GRT configuration...')
logging.info("Loading GRT configuration...")
try:
with open(args.grt_config) as handle:
config = yaml.safe_load(handle)
except Exception:
logging.exception('Could not parse GRT configuration')
logging.exception("Could not parse GRT configuration")
sys.exit(1)
REPORT_DIR = args.report_directory
GRT_URL = config['grt']['url'].rstrip('/') + '/'
GRT_INSTANCE_ID = config['grt']['instance_id']
GRT_API_KEY = config['grt']['api_key']
GRT_URL = config["grt"]["url"].rstrip("/") + "/"
GRT_INSTANCE_ID = config["grt"]["instance_id"]
GRT_API_KEY = config["grt"]["api_key"]
# Contact the server and check auth details.
headers = {
'AUTHORIZATION': f'{GRT_INSTANCE_ID}:{GRT_API_KEY}'
}
r = requests.post(GRT_URL + 'api/whoami', headers=headers)
headers = {"AUTHORIZATION": f"{GRT_INSTANCE_ID}:{GRT_API_KEY}"}
r = requests.post(GRT_URL + "api/whoami", headers=headers)
data = r.json()
# Get back some information about which reports had previously been uploaded.
remote_reports = data['uploaded_reports']
remote_reports = data["uploaded_reports"]
logging.debug("Remote reports: %s", remote_reports)
local_reports = [x.strip('.json') for x in os.listdir(REPORT_DIR) if x.endswith('.json')]
local_reports = [x.strip(".json") for x in os.listdir(REPORT_DIR) if x.endswith(".json")]
logging.debug("Local reports: %s", local_reports)
# Now we know which to send.
for report_id in local_reports:
if report_id not in remote_reports:
logging.info("Uploading %s", report_id)
files = {
'meta': open(os.path.join(REPORT_DIR, report_id + '.json'), 'rb'),
'data': open(os.path.join(REPORT_DIR, report_id + '.tar.gz'), 'rb')
"meta": open(os.path.join(REPORT_DIR, report_id + ".json"), "rb"),
"data": open(os.path.join(REPORT_DIR, report_id + ".tar.gz"), "rb"),
}
data = {
'identifier': report_id
}
r = requests.post(GRT_URL + 'api/v2/upload', files=files, headers=headers, data=data)
data = {"identifier": report_id}
r = requests.post(GRT_URL + "api/v2/upload", files=files, headers=headers, data=data)
if r.ok:
logging.info("Uploaded successfully %s", report_id)
else:
logging.critical("Non-OK response: %s", r.status_code)
if __name__ == '__main__':
if __name__ == "__main__":
main(sys.argv)
scripts/helper.py
+10 -7
View File
@@ -9,23 +9,26 @@ import argparse
import os
import sys
sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, 'lib'))
sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, "lib"))
import galaxy.config
from galaxy.security import idencoding
from galaxy.util.script import app_properties_from_args, populate_config_args
from galaxy.util.script import (
app_properties_from_args,
populate_config_args,
)
parser = argparse.ArgumentParser()
populate_config_args(parser)
parser.add_argument('-e', '--encode-id', dest='encode_id', help='Encode an ID')
parser.add_argument('-d', '--decode-id', dest='decode_id', help='Decode an ID')
parser.add_argument('--hda', dest='hda_id', help='Display HistoryDatasetAssociation info')
parser.add_argument('--ldda', dest='ldda_id', help='Display LibraryDatasetDatasetAssociation info')
parser.add_argument("-e", "--encode-id", dest="encode_id", help="Encode an ID")
parser.add_argument("-d", "--decode-id", dest="decode_id", help="Decode an ID")
parser.add_argument("--hda", dest="hda_id", help="Display HistoryDatasetAssociation info")
parser.add_argument("--ldda", dest="ldda_id", help="Display LibraryDatasetDatasetAssociation info")
args = parser.parse_args()
app_properties = app_properties_from_args(args)
config = galaxy.config.Configuration(**app_properties)
helper = idencoding.IdEncodingHelper(id_secret=app_properties.get('id_secret'))
helper = idencoding.IdEncodingHelper(id_secret=app_properties.get("id_secret"))
model = galaxy.config.init_models_from_config(config)
if args.encode_id:
scripts/interactivetools/key_type_token_mapping.py
+15 -11
View File
@@ -5,14 +5,13 @@ from time import time
import uwsgi
realtime_db_file = uwsgi.opt["interactivetools_map"].decode('utf-8')
realtime_db_file = uwsgi.opt["interactivetools_map"].decode("utf-8")
db_conn = sqlite3.connect(realtime_db_file)
DATABASE_TABLE_NAME = 'gxitproxy'
DATABASE_TABLE_NAME = "gxitproxy"
class CacheEntry():
class CacheEntry:
def __init__(self, key, value, ttl=20):
self.key = key
self.value = value
@@ -21,12 +20,12 @@ class CacheEntry():
def expired(self):
if self._expired is False:
return (self.expires_at < time())
return self.expires_at < time()
else:
return self._expired
class CacheList():
class CacheList:
def __init__(self):
self.entries = []
self.lock = RLock()
@@ -53,8 +52,9 @@ key_type_token_mapped_cache = CacheList()
def args_as_unicode(func):
def wrap_args(*args):
args = (arg.decode('utf-8') if isinstance(arg, bytes) else arg for arg in args)
args = (arg.decode("utf-8") if isinstance(arg, bytes) else arg for arg in args)
return func(*args)
return wrap_args
@@ -81,9 +81,13 @@ def key_type_token_mapper(key, key_type, token, route_extra, url):
for _ in range(2):
# Order by rowid gives us the last row added
try:
row = db_conn.execute("SELECT host, port FROM %s WHERE key=? AND key_type=? AND token=? ORDER BY rowid DESC LIMIT 1" % (DATABASE_TABLE_NAME), (key, key_type, token)).fetchone()
row = db_conn.execute(
"SELECT host, port FROM %s WHERE key=? AND key_type=? AND token=? ORDER BY rowid DESC LIMIT 1"
% (DATABASE_TABLE_NAME),
(key, key_type, token),
).fetchone()
if row:
rval = '%s:%s' % (tuple(row))
rval = "%s:%s" % (tuple(row))
return rval.encode()
break
except sqlite3.ProgrammingError:
@@ -93,5 +97,5 @@ def key_type_token_mapper(key, key_type, token, route_extra, url):
return None
uwsgi.register_rpc('rtt_key_type_token_mapper', key_type_token_mapper)
uwsgi.register_rpc('rtt_key_type_token_mapper_cached', key_type_token_mapper_cached)
uwsgi.register_rpc("rtt_key_type_token_mapper", key_type_token_mapper)
uwsgi.register_rpc("rtt_key_type_token_mapper_cached", key_type_token_mapper_cached)
scripts/loc_files/create_all_fasta_loc.py
+262 -163
View File
@@ -29,152 +29,236 @@ import os
import sys
from xml.etree.ElementTree import parse
DEFAULT_TOOL_DATA_TABLE_CONF = 'tool_data_table_conf.xml'
DEFAULT_ALL_FASTA_LOC_BASE = 'all_fasta'
DEFAULT_BASE_GENOME_DIR = '/afs/bx.psu.edu/depot/data/genome'
EXEMPTIONS = 'bin,tmp,lengths,equCab2_chrM,microbes'
DEFAULT_TOOL_DATA_TABLE_CONF = "tool_data_table_conf.xml"
DEFAULT_ALL_FASTA_LOC_BASE = "all_fasta"
DEFAULT_BASE_GENOME_DIR = "/afs/bx.psu.edu/depot/data/genome"
EXEMPTIONS = "bin,tmp,lengths,equCab2_chrM,microbes"
INSPECT_DIR = None
FASTA_EXTS = '.fa,.fasta,.fna'
VARIANTS = 'chrM,chr21,full,canon,female,male,haps,nohaps'
FASTA_EXTS = ".fa,.fasta,.fna"
VARIANTS = "chrM,chr21,full,canon,female,male,haps,nohaps"
VARIANT_EXCLUSIONS = ':full'
VARIANT_EXCLUSIONS = ":full"
DBKEY_DESCRIPTION_MAP = {'AaegL1': 'Mosquito (Aedes aegypti): AaegL1',
'AgamP3': 'Mosquito (Anopheles gambiae): AgamP3',
'anoCar1': 'Lizard (Anolis carolinensis): anoCar1',
'anoGam1': 'Mosquito (Anopheles gambiae): anoGam1',
'apiMel1': 'Honeybee (Apis mellifera): apiMel1',
'apiMel2': 'Honeybee (Apis mellifera): apiMel2',
'apiMel3': 'Honeybee (Apis mellifera): apiMel3',
'Arabidopsis_thaliana_TAIR9': '',
'borEut13': 'Boreoeutherian: borEut13',
'bosTau2': 'Cow (Bos taurus): bosTau2',
'bosTau3': 'Cow (Bos taurus): bosTau3',
'bosTau4': 'Cow (Bos taurus): bosTau4',
'bosTauMd3': 'Cow (Bos taurus): bosTauMd3',
'calJac1': 'Marmoset (Callithrix jacchus): calJac1',
'canFam1': 'Dog (Canis lupus familiaris): canFam1',
'canFam2': 'Dog (Canis lupus familiaris): canFam2',
'cavPor3': 'Guinea Pig (Cavia porcellus): cavPor3',
'ce2': 'Caenorhabditis elegans: ce2',
'ce4': 'Caenorhabditis elegans: ce4',
'ce5': 'Caenorhabditis elegans: ce5',
'ce6': 'Caenorhabditis elegans: ce6',
'CpipJ1': 'Mosquito (Culex quinquefasciatus): CpipJ1',
'danRer2': 'Zebrafish (Danio rerio): danRer2',
'danRer3': 'Zebrafish (Danio rerio): danRer3',
'danRer4': 'Zebrafish (Danio rerio): danRer4',
'danRer5': 'Zebrafish (Danio rerio): danRer5',
'danRer6': 'Zebrafish (Danio rerio): danRer6',
'dm1': 'Fruit Fly (Drosophila melanogaster): dm1',
'dm2': 'Fruit Fly (Drosophila melanogaster): dm2',
'dm3': 'Fruit Fly (Drosophila melanogaster): dm3',
'dm4': 'Fruit Fly (Drosophila melanogaster): dm',
'dp3': 'Fruit Fly (Drosophila pseudoobscura): dp3',
'dp4': 'Fruit Fly (Drosophila pseudoobscura): dp4',
'droAna1': 'Fruit Fly (Drosophila ananassae): droAna1',
'droAna2': 'Fruit Fly (Drosophila ananassae): droAna2',
'droAna3': 'Fruit Fly (Drosophila ananassae): droAna3',
'droEre1': 'Fruit Fly (Drosophila erecta): droEre1',
'droEre2': 'Fruit Fly (Drosophila erecta): droEre2',
'droGri1': 'Fruit Fly (Drosophila grimshawi): droGri1',
'droGri2': 'Fruit Fly (Drosophila grimshawi): droGri2',
'droMoj1': 'Fruit Fly (Drosophila mojavensis): droMoj1',
'droMoj2': 'Fruit Fly (Drosophila mojavensis): droMoj2',
'droMoj3': 'Fruit Fly (Drosophila mojavensis): droMoj3',
'droPer1': 'Fruit Fly (Drosophila persimilis): droPer1',
'droSec1': 'Fruit Fly (Drosophila sechellia): droSec1',
'droSim1': 'Fruit Fly (Drosophila simulans): droSim1',
'droVir1': 'Fruit Fly (Drosophila virilis): droVir1',
'droVir2': 'Fruit Fly (Drosophila virilis): droVir2',
'droVir3': 'Fruit Fly (Drosophila virilis): droVir3',
'droYak1': 'Fruit Fly (Drosophila yakuba): droYak1',
'droYak2': 'Fruit Fly (Drosophila yakuba): droYak2',
'echTel1': 'Tenrec (Echinops telfairi): echTel1',
'equCab1': 'Horse (Equus caballus): equCab1',
'equCab2': 'Horse (Equus caballus): equCab2',
'eriEur1': 'Hedgehog (Erinaceus europaeus): eriEur1',
'felCat3': 'Cat (Felis catus): felCat3',
'fr1': 'Fugu (Takifugu rubripes): fr1',
'fr2': 'Fugu (Takifugu rubripes): fr2',
'galGal2': 'Chicken (Gallus gallus): galGal2',
'galGal3': 'Chicken (Gallus gallus): galGal3',
'gasAcu1': 'Stickleback (Gasterosteus aculeatus): gasAcu1',
'hg16': 'Human (Homo sapiens): hg16',
'hg17': 'Human (Homo sapiens): hg17',
'hg18': 'Human (Homo sapiens): hg18',
'hg19': 'Human (Homo sapiens): hg19',
'IscaW1': 'Deer Tick (Ixodes scapularis): IscaW1',
'lMaj5': 'Leishmania major: lMaj5',
'mm5': 'Mouse (Mus musculus): mm5',
'mm6': 'Mouse (Mus musculus): mm6',
'mm7': 'Mouse (Mus musculus): mm7',
'mm8': 'Mouse (Mus musculus): mm8',
'mm9': 'Mouse (Mus musculus): mm9',
'monDom4': 'Opossum (Monodelphis domestica): monDom4',
'monDom5': 'Opossum (Monodelphis domestica): monDom5',
'ornAna1': 'Platypus (Ornithorhynchus anatinus): ornAna1',
'oryCun1': 'Rabbit (Oryctolagus cuniculus): oryCun1',
'oryLat1': 'Medaka (Oryzias latipes): oryLat1',
'oryLat2': 'Medaka (Oryzias latipes): oryLat2',
'oryza_sativa_japonica_nipponbare_IRGSP4.0': 'Rice (Oryza sativa L. ssp. japonica var. Nipponbare): IRGSP4.0',
'otoGar1': 'Bushbaby (Otolemur garnetti): otoGar1',
'panTro1': 'Chimpanzee (Pan troglodytes): panTro1',
'panTro2': 'Chimpanzee (Pan troglodytes): panTro2',
'petMar1': 'Lamprey (Petromyzon marinus): petMar1',
'phiX': 'phiX174 (AF176034)',
'PhumU1': 'Head Louse (Pediculus humanus): PhumU1',
'ponAbe2': 'Orangutan (Pongo pygmaeus abelii): ponAbe2',
'pUC18': 'pUC18 (L09136)',
'rheMac2': 'Rhesus Macaque (Macaca mulatta): rheMac2',
'rn3': 'Rat (Rattus norvegicus): rn3',
'rn4': 'Rat (Rattus norvegicus): rn4',
'sacCer1': 'Yeast (Saccharomyces cerevisiae): sacCer1',
'sacCer2': 'Yeast (Saccharomyces cerevisiae): sacCer2',
'sorAra1': 'Common Shrew (Sorex araneus): sorAra1',
'Sscrofa9.58': 'Pig (Sus scrofa): Sscrofa9.58',
'strPur2': 'Purple Sea Urchin (Strongylocentrotus purpuratus): strPur2',
'susScr2': 'Pig (Sus scrofa): susScr2',
'taeGut1': 'Zebra Finch (Taeniopygia guttata): taeGut1',
'tetNig1': 'Tetraodon (Tetraodon nigroviridis): tetNig1',
'tetNig2': 'Tetraodon (Tetraodon nigroviridis): tetNig2',
'tupBel1': 'Tree Shrew (Tupaia belangeri): tupBel1',
'venter1': 'Human (J. Craig Venter): venter1',
'xenTro2': 'Frog (Xenopus tropicalis): xenTro2'}
DBKEY_DESCRIPTION_MAP = {
"AaegL1": "Mosquito (Aedes aegypti): AaegL1",
"AgamP3": "Mosquito (Anopheles gambiae): AgamP3",
"anoCar1": "Lizard (Anolis carolinensis): anoCar1",
"anoGam1": "Mosquito (Anopheles gambiae): anoGam1",
"apiMel1": "Honeybee (Apis mellifera): apiMel1",
"apiMel2": "Honeybee (Apis mellifera): apiMel2",
"apiMel3": "Honeybee (Apis mellifera): apiMel3",
"Arabidopsis_thaliana_TAIR9": "",
"borEut13": "Boreoeutherian: borEut13",
"bosTau2": "Cow (Bos taurus): bosTau2",
"bosTau3": "Cow (Bos taurus): bosTau3",
"bosTau4": "Cow (Bos taurus): bosTau4",
"bosTauMd3": "Cow (Bos taurus): bosTauMd3",
"calJac1": "Marmoset (Callithrix jacchus): calJac1",
"canFam1": "Dog (Canis lupus familiaris): canFam1",
"canFam2": "Dog (Canis lupus familiaris): canFam2",
"cavPor3": "Guinea Pig (Cavia porcellus): cavPor3",
"ce2": "Caenorhabditis elegans: ce2",
"ce4": "Caenorhabditis elegans: ce4",
"ce5": "Caenorhabditis elegans: ce5",
"ce6": "Caenorhabditis elegans: ce6",
"CpipJ1": "Mosquito (Culex quinquefasciatus): CpipJ1",
"danRer2": "Zebrafish (Danio rerio): danRer2",
"danRer3": "Zebrafish (Danio rerio): danRer3",
"danRer4": "Zebrafish (Danio rerio): danRer4",
"danRer5": "Zebrafish (Danio rerio): danRer5",
"danRer6": "Zebrafish (Danio rerio): danRer6",
"dm1": "Fruit Fly (Drosophila melanogaster): dm1",
"dm2": "Fruit Fly (Drosophila melanogaster): dm2",
"dm3": "Fruit Fly (Drosophila melanogaster): dm3",
"dm4": "Fruit Fly (Drosophila melanogaster): dm",
"dp3": "Fruit Fly (Drosophila pseudoobscura): dp3",
"dp4": "Fruit Fly (Drosophila pseudoobscura): dp4",
"droAna1": "Fruit Fly (Drosophila ananassae): droAna1",
"droAna2": "Fruit Fly (Drosophila ananassae): droAna2",
"droAna3": "Fruit Fly (Drosophila ananassae): droAna3",
"droEre1": "Fruit Fly (Drosophila erecta): droEre1",
"droEre2": "Fruit Fly (Drosophila erecta): droEre2",
"droGri1": "Fruit Fly (Drosophila grimshawi): droGri1",
"droGri2": "Fruit Fly (Drosophila grimshawi): droGri2",
"droMoj1": "Fruit Fly (Drosophila mojavensis): droMoj1",
"droMoj2": "Fruit Fly (Drosophila mojavensis): droMoj2",
"droMoj3": "Fruit Fly (Drosophila mojavensis): droMoj3",
"droPer1": "Fruit Fly (Drosophila persimilis): droPer1",
"droSec1": "Fruit Fly (Drosophila sechellia): droSec1",
"droSim1": "Fruit Fly (Drosophila simulans): droSim1",
"droVir1": "Fruit Fly (Drosophila virilis): droVir1",
"droVir2": "Fruit Fly (Drosophila virilis): droVir2",
"droVir3": "Fruit Fly (Drosophila virilis): droVir3",
"droYak1": "Fruit Fly (Drosophila yakuba): droYak1",
"droYak2": "Fruit Fly (Drosophila yakuba): droYak2",
"echTel1": "Tenrec (Echinops telfairi): echTel1",
"equCab1": "Horse (Equus caballus): equCab1",
"equCab2": "Horse (Equus caballus): equCab2",
"eriEur1": "Hedgehog (Erinaceus europaeus): eriEur1",
"felCat3": "Cat (Felis catus): felCat3",
"fr1": "Fugu (Takifugu rubripes): fr1",
"fr2": "Fugu (Takifugu rubripes): fr2",
"galGal2": "Chicken (Gallus gallus): galGal2",
"galGal3": "Chicken (Gallus gallus): galGal3",
"gasAcu1": "Stickleback (Gasterosteus aculeatus): gasAcu1",
"hg16": "Human (Homo sapiens): hg16",
"hg17": "Human (Homo sapiens): hg17",
"hg18": "Human (Homo sapiens): hg18",
"hg19": "Human (Homo sapiens): hg19",
"IscaW1": "Deer Tick (Ixodes scapularis): IscaW1",
"lMaj5": "Leishmania major: lMaj5",
"mm5": "Mouse (Mus musculus): mm5",
"mm6": "Mouse (Mus musculus): mm6",
"mm7": "Mouse (Mus musculus): mm7",
"mm8": "Mouse (Mus musculus): mm8",
"mm9": "Mouse (Mus musculus): mm9",
"monDom4": "Opossum (Monodelphis domestica): monDom4",
"monDom5": "Opossum (Monodelphis domestica): monDom5",
"ornAna1": "Platypus (Ornithorhynchus anatinus): ornAna1",
"oryCun1": "Rabbit (Oryctolagus cuniculus): oryCun1",
"oryLat1": "Medaka (Oryzias latipes): oryLat1",
"oryLat2": "Medaka (Oryzias latipes): oryLat2",
"oryza_sativa_japonica_nipponbare_IRGSP4.0": "Rice (Oryza sativa L. ssp. japonica var. Nipponbare): IRGSP4.0",
"otoGar1": "Bushbaby (Otolemur garnetti): otoGar1",
"panTro1": "Chimpanzee (Pan troglodytes): panTro1",
"panTro2": "Chimpanzee (Pan troglodytes): panTro2",
"petMar1": "Lamprey (Petromyzon marinus): petMar1",
"phiX": "phiX174 (AF176034)",
"PhumU1": "Head Louse (Pediculus humanus): PhumU1",
"ponAbe2": "Orangutan (Pongo pygmaeus abelii): ponAbe2",
"pUC18": "pUC18 (L09136)",
"rheMac2": "Rhesus Macaque (Macaca mulatta): rheMac2",
"rn3": "Rat (Rattus norvegicus): rn3",
"rn4": "Rat (Rattus norvegicus): rn4",
"sacCer1": "Yeast (Saccharomyces cerevisiae): sacCer1",
"sacCer2": "Yeast (Saccharomyces cerevisiae): sacCer2",
"sorAra1": "Common Shrew (Sorex araneus): sorAra1",
"Sscrofa9.58": "Pig (Sus scrofa): Sscrofa9.58",
"strPur2": "Purple Sea Urchin (Strongylocentrotus purpuratus): strPur2",
"susScr2": "Pig (Sus scrofa): susScr2",
"taeGut1": "Zebra Finch (Taeniopygia guttata): taeGut1",
"tetNig1": "Tetraodon (Tetraodon nigroviridis): tetNig1",
"tetNig2": "Tetraodon (Tetraodon nigroviridis): tetNig2",
"tupBel1": "Tree Shrew (Tupaia belangeri): tupBel1",
"venter1": "Human (J. Craig Venter): venter1",
"xenTro2": "Frog (Xenopus tropicalis): xenTro2",
}
VARIANT_MAP = {'canon': 'Canonical',
'full': 'Full',
'female': 'Female',
'male': 'Male'}
VARIANT_MAP = {"canon": "Canonical", "full": "Full", "female": "Female", "male": "Male"}
def __main__():
# command line variables
parser = optparse.OptionParser()
parser.add_option('-d', '--data-table-xml', dest='data_table_xml', type='string', default=DEFAULT_TOOL_DATA_TABLE_CONF, help='The name of the data table configuration file to get format of loc file')
parser.add_option('-t', '--data-table', dest='data_table_name', type='string', default=DEFAULT_ALL_FASTA_LOC_BASE, help='The name of the data table listed in the data table XML file')
parser.add_option('-g', '--genome_dir', dest='genome_dir', type='string', default=DEFAULT_BASE_GENOME_DIR, help='Genome directory to look in')
parser.add_option('-e', '--exemptions', dest='exemptions', type='string', default=EXEMPTIONS, help='Comma-separated list of subdirectories in genome dir to not look in')
parser.add_option('-i', '--inspect-dir', dest='inspect_dir', type='string', default=INSPECT_DIR, help='Comma-separated list of subdirectories inside genome dirs to look in (default is all)')
parser.add_option('-x', '--fasta_exts', dest='fasta_exts', type='string', default=FASTA_EXTS, help='Comma-separated list of all fasta extensions to list')
parser.add_option('-s', '--loc-sample', dest='loc_sample_name', type='string', help='The name of the sample loc file (to copy text into top of output loc file)')
parser.add_option('-f', '--unmatching-fasta', dest='unmatching_fasta', type='string', default=None, help='Name of file to output non-matching fasta files to')
parser.add_option('-v', '--variants', dest='variants', type='string', default=VARIANTS, help='Comma-separated list of recognized variants of fasta file names')
parser.add_option('-n', '--variant-exclusions', dest='variant_exclusions', type='string', default=VARIANT_EXCLUSIONS, help="List of files to exclude because they're duplicated by a variants; of the format: '<variant_to_keep_1>:<variant_to_remove_1>[,<variant_to_remove_2>[,...]][;<variant_to_keep_2>:<variant_to_remove_1>[,<variant_to_remove_2>[,...]]]'; default ':(full)' (if non-variant version present (like 'hg19'), full version (like 'hg19full') will be thrown out)")
parser.add_option('-a', '--append', dest='append', action='store_true', default=False, help='Append to existing all_fasta.loc file rather than create new')
parser.add_option('-p', '--sample-text', dest='sample_text', action='store_true', default='True', help='Copy over text from all_fasta.loc.sample file (false if set to append)')
parser.add_option(
"-d",
"--data-table-xml",
dest="data_table_xml",
type="string",
default=DEFAULT_TOOL_DATA_TABLE_CONF,
help="The name of the data table configuration file to get format of loc file",
)
parser.add_option(
"-t",
"--data-table",
dest="data_table_name",
type="string",
default=DEFAULT_ALL_FASTA_LOC_BASE,
help="The name of the data table listed in the data table XML file",
)
parser.add_option(
"-g",
"--genome_dir",
dest="genome_dir",
type="string",
default=DEFAULT_BASE_GENOME_DIR,
help="Genome directory to look in",
)
parser.add_option(
"-e",
"--exemptions",
dest="exemptions",
type="string",
default=EXEMPTIONS,
help="Comma-separated list of subdirectories in genome dir to not look in",
)
parser.add_option(
"-i",
"--inspect-dir",
dest="inspect_dir",
type="string",
default=INSPECT_DIR,
help="Comma-separated list of subdirectories inside genome dirs to look in (default is all)",
)
parser.add_option(
"-x",
"--fasta_exts",
dest="fasta_exts",
type="string",
default=FASTA_EXTS,
help="Comma-separated list of all fasta extensions to list",
)
parser.add_option(
"-s",
"--loc-sample",
dest="loc_sample_name",
type="string",
help="The name of the sample loc file (to copy text into top of output loc file)",
)
parser.add_option(
"-f",
"--unmatching-fasta",
dest="unmatching_fasta",
type="string",
default=None,
help="Name of file to output non-matching fasta files to",
)
parser.add_option(
"-v",
"--variants",
dest="variants",
type="string",
default=VARIANTS,
help="Comma-separated list of recognized variants of fasta file names",
)
parser.add_option(
"-n",
"--variant-exclusions",
dest="variant_exclusions",
type="string",
default=VARIANT_EXCLUSIONS,
help="List of files to exclude because they're duplicated by a variants; of the format: '<variant_to_keep_1>:<variant_to_remove_1>[,<variant_to_remove_2>[,...]][;<variant_to_keep_2>:<variant_to_remove_1>[,<variant_to_remove_2>[,...]]]'; default ':(full)' (if non-variant version present (like 'hg19'), full version (like 'hg19full') will be thrown out)",
)
parser.add_option(
"-a",
"--append",
dest="append",
action="store_true",
default=False,
help="Append to existing all_fasta.loc file rather than create new",
)
parser.add_option(
"-p",
"--sample-text",
dest="sample_text",
action="store_true",
default="True",
help="Copy over text from all_fasta.loc.sample file (false if set to append)",
)
(options, args) = parser.parse_args()
exemptions = [e.strip() for e in options.exemptions.split(',')]
fasta_exts = [x.strip() for x in options.fasta_exts.split(',')]
variants = [v.strip() for v in options.variants.split(',')]
exemptions = [e.strip() for e in options.exemptions.split(",")]
fasta_exts = [x.strip() for x in options.fasta_exts.split(",")]
variants = [v.strip() for v in options.variants.split(",")]
variant_exclusions = {}
try:
for ve in options.variant_exclusions.split(';'):
v, e = ve.split(':')
variant_exclusions[v] = e.split(',')
for ve in options.variant_exclusions.split(";"):
v, e = ve.split(":")
variant_exclusions[v] = e.split(",")
except Exception:
sys.stderr.write('Problem parsing the variant exclusion parameter (-n/--variant-exclusion). Make sure it follows the expected format\n')
sys.stderr.write(
"Problem parsing the variant exclusion parameter (-n/--variant-exclusion). Make sure it follows the expected format\n"
)
sys.exit(1)
if options.append:
sample_text = False
@@ -183,21 +267,21 @@ def __main__():
# all paths to look in
if options.inspect_dir:
paths_to_look_in = [os.path.join(options.genome_dir, '%s', id) for id in options.inspect_dir.split(',')]
paths_to_look_in = [os.path.join(options.genome_dir, "%s", id) for id in options.inspect_dir.split(",")]
else:
paths_to_look_in = [os.path.join(options.genome_dir, '%s')]
paths_to_look_in = [os.path.join(options.genome_dir, "%s")]
# say what we're looking in
print('\nLooking in:\n\t%s' % '\n\t'.join(p % '<build_name>' for p in paths_to_look_in))
poss_names = ['<build_name>%s' % _ for _ in variants]
print('for files that are named %s' % ', '.join(poss_names[:-1]), end=' ')
print("\nLooking in:\n\t%s" % "\n\t".join(p % "<build_name>" for p in paths_to_look_in))
poss_names = ["<build_name>%s" % _ for _ in variants]
print("for files that are named %s" % ", ".join(poss_names[:-1]), end=" ")
if len(poss_names) > 1:
print('or %s' % poss_names[-1], end=' ')
print("or %s" % poss_names[-1], end=" ")
if len(options.fasta_exts) == 1:
print('with the extension %s.' % ', '.join(fasta_exts[:-1]))
print("with the extension %s." % ", ".join(fasta_exts[:-1]))
else:
print('with the extension {} or {}.'.format(', '.join(fasta_exts[:-1]), fasta_exts[-1]))
print('\nSkipping the following:\n\t%s' % '\n\t'.join(exemptions))
print("with the extension {} or {}.".format(", ".join(fasta_exts[:-1]), fasta_exts[-1]))
print("\nSkipping the following:\n\t%s" % "\n\t".join(exemptions))
# get column names
col_values = []
@@ -205,18 +289,20 @@ def __main__():
tree = parse(options.data_table_xml)
tables = tree.getroot()
for table in tables.iter():
name = table.attrib.get('name')
name = table.attrib.get("name")
if name == options.data_table_name:
cols = None
for node in table.iter():
if node.tag == 'columns':
if node.tag == "columns":
cols = node.text
elif node.tag == 'file':
loc_path = node.attrib.get('path')
elif node.tag == "file":
loc_path = node.attrib.get("path")
if cols:
col_values = [col.strip() for col in cols.split(',')]
col_values = [col.strip() for col in cols.split(",")]
if not col_values or not loc_path:
raise Exception(f'No columns can be found for this data table ({options.data_table}) in {options.data_table_xml}')
raise Exception(
f"No columns can be found for this data table ({options.data_table}) in {options.data_table_xml}"
)
# get all fasta paths under genome directory
fasta_locs = {}
@@ -224,7 +310,7 @@ def __main__():
genome_subdirs = [dr for dr in os.listdir(options.genome_dir) if dr not in exemptions]
for genome_subdir in genome_subdirs:
possible_names = [genome_subdir]
possible_names.extend([f'{genome_subdir}{_}' for _ in variants])
possible_names.extend([f"{genome_subdir}{_}" for _ in variants])
# get paths to all fasta files
for path_to_look_in in paths_to_look_in:
for dirpath, _dirnames, filenames in os.walk(path_to_look_in % genome_subdir):
@@ -237,40 +323,53 @@ def __main__():
name = DBKEY_DESCRIPTION_MAP[genome_subdir]
else:
try:
name = '{} {}'.format(DBKEY_DESCRIPTION_MAP[genome_subdir], VARIANT_MAP[fasta_base.replace(genome_subdir, '')])
name = "{} {}".format(
DBKEY_DESCRIPTION_MAP[genome_subdir],
VARIANT_MAP[fasta_base.replace(genome_subdir, "")],
)
except KeyError:
name = '{} {}'.format(DBKEY_DESCRIPTION_MAP[genome_subdir], fasta_base.replace(genome_subdir, ''))
fasta_locs[fasta_base] = {'value': fasta_base, 'dbkey': genome_subdir, 'name': name, 'path': os.path.join(dirpath, fn)}
name = "{} {}".format(
DBKEY_DESCRIPTION_MAP[genome_subdir], fasta_base.replace(genome_subdir, "")
)
fasta_locs[fasta_base] = {
"value": fasta_base,
"dbkey": genome_subdir,
"name": name,
"path": os.path.join(dirpath, fn),
}
else:
unmatching_fasta_paths.append(os.path.join(dirpath, fn))
# remove redundant fasta files
for k, v in variant_exclusions.items():
leave_in = f'{genome_subdir}{k}'
leave_in = f"{genome_subdir}{k}"
if leave_in in fasta_locs:
to_remove = [f'{genome_subdir}{_}' for _ in v]
to_remove = [f"{genome_subdir}{_}" for _ in v]
for tr in to_remove:
if tr in fasta_locs:
del fasta_locs[tr]
# output results
print('\nThere were %s fasta files found that were not included because they did not have the expected file names.' % len(unmatching_fasta_paths))
print('%s fasta files were found and listed.\n' % len(fasta_locs.keys()))
print(
"\nThere were %s fasta files found that were not included because they did not have the expected file names."
% len(unmatching_fasta_paths)
)
print("%s fasta files were found and listed.\n" % len(fasta_locs.keys()))
# output unmatching fasta files
if options.unmatching_fasta and unmatching_fasta_paths:
open(options.unmatching_fasta, 'wb').write('%s\n' % '\n'.join(unmatching_fasta_paths))
open(options.unmatching_fasta, "wb").write("%s\n" % "\n".join(unmatching_fasta_paths))
# output loc file
if options.append:
all_fasta_loc = open(loc_path, 'ab')
all_fasta_loc = open(loc_path, "ab")
else:
all_fasta_loc = open(loc_path, 'wb')
all_fasta_loc = open(loc_path, "wb")
# put sample loc file text at top of file if appropriate
if sample_text:
if options.loc_sample_name:
all_fasta_loc.write('%s\n' % open(options.loc_sample_name, 'rb').read().strip())
all_fasta_loc.write("%s\n" % open(options.loc_sample_name, "rb").read().strip())
else:
all_fasta_loc.write('%s\n' % open('%s.sample' % loc_path, 'rb').read().strip())
all_fasta_loc.write("%s\n" % open("%s.sample" % loc_path, "rb").read().strip())
# output list of fasta files in alphabetical order
fasta_bases = list(fasta_locs.keys())
fasta_bases.sort(key=str.upper)
@@ -280,12 +379,12 @@ def __main__():
try:
out_line.append(fasta_locs[fb][col])
except KeyError:
raise Exception('Unexpected column (%s) encountered' % col)
raise Exception("Unexpected column (%s) encountered" % col)
if out_line:
all_fasta_loc.write('%s\n' % '\t'.join(out_line))
all_fasta_loc.write("%s\n" % "\t".join(out_line))
# close up output loc file
all_fasta_loc.close()
if __name__ == '__main__':
if __name__ == "__main__":
__main__()
scripts/manage_db.py
+3 -3
View File
@@ -7,7 +7,7 @@ import sys
from migrate.versioning.shell import main
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib")))
from galaxy.model.orm.scripts import get_config
@@ -18,8 +18,8 @@ log = logging.getLogger(__name__)
def invoke_migrate_main():
# Migrate has its own args, so cannot use argparse
config = get_config(sys.argv, use_argparse=False, cwd=os.getcwd())
db_url = config['db_url']
repo = config['repo']
db_url = config["db_url"]
repo = config["repo"]
main(repository=repo, url=db_url)
scripts/manage_tool_dependencies.py
+9 -5
View File
@@ -1,7 +1,7 @@
import os.path
import sys
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib")))
from galaxy.config import configure_logging
from galaxy.tool_util.deps import build_dependency_manager
@@ -23,10 +23,14 @@ def _build_dependency_manager_no_config(kwargs):
which we do not have available in this script (an optimization).
"""
configure_logging(kwargs)
base, ext = os.path.splitext(kwargs.get('dependency_resolvers_config_file', 'dependency_resolvers_conf.xml'))
dependency_resolvers_config_file = find_config_file(base, exts=[ext.lstrip('.')])
base, ext = os.path.splitext(kwargs.get("dependency_resolvers_config_file", "dependency_resolvers_conf.xml"))
dependency_resolvers_config_file = find_config_file(base, exts=[ext.lstrip(".")])
# FIXME: default is wrong for installed Galaxy
dependency_manager = build_dependency_manager(app_config_dict=kwargs, conf_file=dependency_resolvers_config_file, default_tool_dependency_dir="database/dependencies")
dependency_manager = build_dependency_manager(
app_config_dict=kwargs,
conf_file=dependency_resolvers_config_file,
default_tool_dependency_dir="database/dependencies",
)
return dependency_manager
@@ -35,6 +39,6 @@ ACTIONS = {
}
if __name__ == '__main__':
if __name__ == "__main__":
main = main_factory(description=DESCRIPTION, actions=ACTIONS)
main()
scripts/metagenomics/convert_title.py
+11 -8
View File
@@ -6,26 +6,26 @@ run formatdb in the command line: gunzip -c nt.gz |formatdb -i stdin -p F -n "nt
import sys
if __name__ == '__main__':
if __name__ == "__main__":
seq = []
len_seq = 0
invalid_lines = 0
gi = None
for line in sys.stdin:
line = line.rstrip('\r\n')
if line.startswith('>'):
line = line.rstrip("\r\n")
if line.startswith(">"):
if len_seq > 0:
if gi is None:
raise Exception('The first sequence does not have an header.')
raise Exception("The first sequence does not have an header.")
print(">%s_%d" % (gi, len_seq))
print("\n".join(seq))
title = line
fields = title.split('|')
if len(fields) >= 2 and fields[0] == '>gi':
fields = title.split("|")
if len(fields) >= 2 and fields[0] == ">gi":
gi = fields[1]
else:
gi = 'giunknown'
gi = "giunknown"
invalid_lines += 1
len_seq = 0
seq = []
@@ -36,4 +36,7 @@ if __name__ == '__main__':
print(">%s_%d" % (gi, len_seq))
print("\n".join(seq))
print("Unable to find gi number for %d sequences, the title is replaced as giunknown" % (invalid_lines), file=sys.stderr)
print(
"Unable to find gi number for %d sequences, the title is replaced as giunknown" % (invalid_lines),
file=sys.stderr,
)
scripts/microbes/create_bacteria_loc_file.py
+98 -25
View File
@@ -26,47 +26,120 @@ def __main__():
for line in info:
fields = line.replace("\n", "").split("=")
tmp_dict[fields[0]] = "=".join(fields[1:])
if 'genome project id' in tmp_dict.keys():
name = tmp_dict['genome project id']
if 'build' in tmp_dict.keys():
name = tmp_dict['build']
if "genome project id" in tmp_dict.keys():
name = tmp_dict["genome project id"]
if "build" in tmp_dict.keys():
name = tmp_dict["build"]
if name not in organisms.keys():
organisms[name] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[name] = {"chrs": {}, "base_dir": this_base_dir}
for key in tmp_dict.keys():
organisms[name][key] = tmp_dict[key]
else:
if tmp_dict['organism'] not in organisms.keys():
organisms[tmp_dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[tmp_dict['organism']]['chrs'][tmp_dict['chromosome']] = tmp_dict
if tmp_dict["organism"] not in organisms.keys():
organisms[tmp_dict["organism"]] = {"chrs": {}, "base_dir": this_base_dir}
organisms[tmp_dict["organism"]]["chrs"][tmp_dict["chromosome"]] = tmp_dict
for org in organisms:
org = organisms[org]
# if no gpi, then must be a ncbi chr which corresponds to a UCSC org, w/o matching UCSC designation
try:
build = org['genome project id']
build = org["genome project id"]
except KeyError:
continue
if 'build' in org:
build = org['build']
print("ORG\t{}\t{}\t{}\t{}\t{}\t{}\tUCSC".format(build, org['name'], org['kingdom'], org['group'], org['chromosomes'], org['info url']))
if "build" in org:
build = org["build"]
print(
"ORG\t{}\t{}\t{}\t{}\t{}\t{}\tUCSC".format(
build, org["name"], org["kingdom"], org["group"], org["chromosomes"], org["info url"]
)
)
else:
print("ORG\t{}\t{}\t{}\t{}\t{}\t{}\tNone".format(build, org['name'], org['kingdom'], org['group'], org['chromosomes'], org['info url']))
print(
"ORG\t{}\t{}\t{}\t{}\t{}\t{}\tNone".format(
build, org["name"], org["kingdom"], org["group"], org["chromosomes"], org["info url"]
)
)
for chr in org['chrs']:
chr = org['chrs'][chr]
print("CHR\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format(build, chr['chromosome'], chr['name'], chr['length'], chr['gi'], chr['gb'], "http://www.ncbi.nlm.nih.gov/entrez/viewer.fcgi?db=nucleotide&val=" + chr['refseq']))
for feature in ['CDS', 'tRNA', 'rRNA']:
print("DATA\t{}_{}_{}\t{}\t{}\t{}\t{}\t{}".format(build, chr['chromosome'], feature, build, chr['chromosome'], feature, "bed", os.path.join(org['base_dir'], "{}.{}.bed".format(chr['chromosome'], feature))))
for chr in org["chrs"]:
chr = org["chrs"][chr]
print(
"CHR\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format(
build,
chr["chromosome"],
chr["name"],
chr["length"],
chr["gi"],
chr["gb"],
"http://www.ncbi.nlm.nih.gov/entrez/viewer.fcgi?db=nucleotide&val=" + chr["refseq"],
)
)
for feature in ["CDS", "tRNA", "rRNA"]:
print(
"DATA\t{}_{}_{}\t{}\t{}\t{}\t{}\t{}".format(
build,
chr["chromosome"],
feature,
build,
chr["chromosome"],
feature,
"bed",
os.path.join(org["base_dir"], "{}.{}.bed".format(chr["chromosome"], feature)),
)
)
# FASTA
print("DATA\t{}_{}_{}\t{}\t{}\t{}\t{}\t{}".format(build, chr['chromosome'], "seq", build, chr['chromosome'], "sequence", "fasta", os.path.join(org['base_dir'], "%s.fna" % chr['chromosome'])))
print(
"DATA\t{}_{}_{}\t{}\t{}\t{}\t{}\t{}".format(
build,
chr["chromosome"],
"seq",
build,
chr["chromosome"],
"sequence",
"fasta",
os.path.join(org["base_dir"], "%s.fna" % chr["chromosome"]),
)
)
# GeneMark
if os.path.exists(os.path.join(org['base_dir'], "%s.GeneMark.bed" % chr['chromosome'])):
print("DATA\t{}_{}_{}\t{}\t{}\t{}\t{}\t{}".format(build, chr['chromosome'], "GeneMark", build, chr['chromosome'], "GeneMark", "bed", os.path.join(org['base_dir'], "%s.GeneMark.bed" % chr['chromosome'])))
if os.path.exists(os.path.join(org["base_dir"], "%s.GeneMark.bed" % chr["chromosome"])):
print(
"DATA\t{}_{}_{}\t{}\t{}\t{}\t{}\t{}".format(
build,
chr["chromosome"],
"GeneMark",
build,
chr["chromosome"],
"GeneMark",
"bed",
os.path.join(org["base_dir"], "%s.GeneMark.bed" % chr["chromosome"]),
)
)
# GenMarkHMM
if os.path.exists(os.path.join(org['base_dir'], "%s.GeneMarkHMM.bed" % chr['chromosome'])):
print("DATA\t{}_{}_{}\t{}\t{}\t{}\t{}\t{}".format(build, chr['chromosome'], "GeneMarkHMM", build, chr['chromosome'], "GeneMarkHMM", "bed", os.path.join(org['base_dir'], "%s.GeneMarkHMM.bed" % chr['chromosome'])))
if os.path.exists(os.path.join(org["base_dir"], "%s.GeneMarkHMM.bed" % chr["chromosome"])):
print(
"DATA\t{}_{}_{}\t{}\t{}\t{}\t{}\t{}".format(
build,
chr["chromosome"],
"GeneMarkHMM",
build,
chr["chromosome"],
"GeneMarkHMM",
"bed",
os.path.join(org["base_dir"], "%s.GeneMarkHMM.bed" % chr["chromosome"]),
)
)
# Glimmer3
if os.path.exists(os.path.join(org['base_dir'], "%s.Glimmer3.bed" % chr['chromosome'])):
print("DATA\t{}_{}_{}\t{}\t{}\t{}\t{}\t{}".format(build, chr['chromosome'], "Glimmer3", build, chr['chromosome'], "Glimmer3", "bed", os.path.join(org['base_dir'], "%s.Glimmer3.bed" % chr['chromosome'])))
if os.path.exists(os.path.join(org["base_dir"], "%s.Glimmer3.bed" % chr["chromosome"])):
print(
"DATA\t{}_{}_{}\t{}\t{}\t{}\t{}\t{}".format(
build,
chr["chromosome"],
"Glimmer3",
build,
chr["chromosome"],
"Glimmer3",
"bed",
os.path.join(org["base_dir"], "%s.Glimmer3.bed" % chr["chromosome"]),
)
)
if __name__ == "__main__":
scripts/microbes/create_bacteria_table.py
+13 -13
View File
@@ -26,21 +26,21 @@ def __main__():
for line in info:
fields = line.replace("\n", "").split("=")
tmp_dict[fields[0]] = "=".join(fields[1:])
if 'genome project id' in tmp_dict.keys():
name = tmp_dict['genome project id']
if 'build' in tmp_dict.keys():
name = tmp_dict['build']
if "genome project id" in tmp_dict.keys():
name = tmp_dict["genome project id"]
if "build" in tmp_dict.keys():
name = tmp_dict["build"]
if name not in organisms.keys():
organisms[name] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[name] = {"chrs": {}, "base_dir": this_base_dir}
for key in tmp_dict.keys():
organisms[name][key] = tmp_dict[key]
else:
if tmp_dict['organism'] not in organisms.keys():
organisms[tmp_dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[tmp_dict['organism']]['chrs'][tmp_dict['chromosome']] = tmp_dict
if tmp_dict["organism"] not in organisms.keys():
organisms[tmp_dict["organism"]] = {"chrs": {}, "base_dir": this_base_dir}
organisms[tmp_dict["organism"]]["chrs"][tmp_dict["chromosome"]] = tmp_dict
for org_name, org in list(organisms.items()):
if 'name' not in org:
if "name" not in org:
del organisms[org_name]
orgs = list(organisms.keys())
@@ -48,7 +48,7 @@ def __main__():
swap_test = False
for i in range(0, len(orgs) - 1):
for j in range(0, len(orgs) - i - 1):
if organisms[orgs[j]]['name'] > organisms[orgs[j + 1]]['name']:
if organisms[orgs[j]]["name"] > organisms[orgs[j + 1]]["name"]:
orgs[j], orgs[j + 1] = orgs[j + 1], orgs[j]
swap_test = True
if swap_test is False:
@@ -60,13 +60,13 @@ def __main__():
at_ucsc = False
# if no gpi, then must be a ncbi chr which corresponds to a UCSC org, w/o matching UCSC designation
try:
org['genome project id']
org["genome project id"]
except KeyError:
continue
if 'build' in org:
if "build" in org:
at_ucsc = True
out_str = "||" + org['name'] + "||" + org['kingdom'] + "||" + org['group'] + "||"
out_str = "||" + org["name"] + "||" + org["kingdom"] + "||" + org["group"] + "||"
if at_ucsc:
out_str = out_str + "Yes"
out_str = out_str + "||"
scripts/microbes/create_nib_seq_loc_file.py
+17 -17
View File
@@ -22,7 +22,7 @@ def __main__():
organisms = {}
loc_out = open(loc_out, 'wb')
loc_out = open(loc_out, "wb")
for result in os.walk(base_dir):
this_base_dir, sub_dirs, files = result
@@ -35,29 +35,29 @@ def __main__():
for line in info:
fields = line.replace("\n", "").split("=")
tmp_dict[fields[0]] = "=".join(fields[1:])
if 'genome project id' in tmp_dict.keys():
name = tmp_dict['genome project id']
if 'build' in tmp_dict.keys():
name = tmp_dict['build']
if "genome project id" in tmp_dict.keys():
name = tmp_dict["genome project id"]
if "build" in tmp_dict.keys():
name = tmp_dict["build"]
if name not in organisms.keys():
organisms[name] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[name] = {"chrs": {}, "base_dir": this_base_dir}
for key in tmp_dict.keys():
organisms[name][key] = tmp_dict[key]
else:
if tmp_dict['organism'] not in organisms.keys():
organisms[tmp_dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[tmp_dict['organism']]['chrs'][tmp_dict['chromosome']] = tmp_dict
if tmp_dict["organism"] not in organisms.keys():
organisms[tmp_dict["organism"]] = {"chrs": {}, "base_dir": this_base_dir}
organisms[tmp_dict["organism"]]["chrs"][tmp_dict["chromosome"]] = tmp_dict
for org in organisms:
org = organisms[org]
try:
build = org['genome project id']
build = org["genome project id"]
except KeyError:
continue
if 'build' in org:
build = org['build']
if "build" in org:
build = org["build"]
seq_path = os.path.join(org['base_dir'], "seq")
seq_path = os.path.join(org["base_dir"], "seq")
# create seq dir, if exists go to next org
# TODO: add better checking, i.e. for updating
@@ -71,11 +71,11 @@ def __main__():
# Print org info
for chr in org['chrs']:
chr = org['chrs'][chr]
for chr in org["chrs"]:
chr = org["chrs"][chr]
fasta_file = os.path.join(org['base_dir'], "%s.fna" % chr['chromosome'])
nib_out_file = os.path.join(seq_path, "%s.nib " % chr['chromosome'])
fasta_file = os.path.join(org["base_dir"], "%s.fna" % chr["chromosome"])
nib_out_file = os.path.join(seq_path, "%s.nib " % chr["chromosome"])
# create nibs using faToNib binary
# TODO: when bx supports writing nib, use it here instead
command = f"faToNib {fasta_file} {nib_out_file}"
scripts/microbes/get_builds_lengths.py
+15 -15
View File
@@ -26,34 +26,34 @@ def __main__():
for line in info:
fields = line.replace("\n", "").split("=")
tmp_dict[fields[0]] = "=".join(fields[1:])
if 'genome project id' in tmp_dict.keys():
name = tmp_dict['genome project id']
if 'build' in tmp_dict.keys():
name = tmp_dict['build']
if "genome project id" in tmp_dict.keys():
name = tmp_dict["genome project id"]
if "build" in tmp_dict.keys():
name = tmp_dict["build"]
if name not in organisms.keys():
organisms[name] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[name] = {"chrs": {}, "base_dir": this_base_dir}
for key in tmp_dict.keys():
organisms[name][key] = tmp_dict[key]
else:
if tmp_dict['organism'] not in organisms.keys():
organisms[tmp_dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[tmp_dict['organism']]['chrs'][tmp_dict['chromosome']] = tmp_dict
if tmp_dict["organism"] not in organisms.keys():
organisms[tmp_dict["organism"]] = {"chrs": {}, "base_dir": this_base_dir}
organisms[tmp_dict["organism"]]["chrs"][tmp_dict["chromosome"]] = tmp_dict
for org in organisms:
org = organisms[org]
# if no gpi, then must be a ncbi chr which corresponds to a UCSC org, w/o matching UCSC designation
try:
build = org['genome project id']
build = org["genome project id"]
except KeyError:
continue
if 'build' in org:
build = org['build']
if "build" in org:
build = org["build"]
chrs = []
for chrom in org['chrs']:
chrom = org['chrs'][chrom]
chrs.append("{}={}".format(chrom['chromosome'], chrom['length']))
print("{}\t{}\t{}".format(build, org['name'], ",".join(chrs)))
for chrom in org["chrs"]:
chrom = org["chrs"][chrom]
chrs.append("{}={}".format(chrom["chromosome"], chrom["length"]))
print("{}\t{}\t{}".format(build, org["name"], ",".join(chrs)))
if __name__ == "__main__":
scripts/microbes/harvest_bacteria.py
+40 -32
View File
@@ -12,6 +12,7 @@ from ftplib import FTP
from urllib.request import urlretrieve
import requests
try:
from bs4 import BeautifulSoup
except ImportError:
@@ -21,44 +22,49 @@ from util import ( # noqa: I202
get_bed_from_genbank,
get_bed_from_GeneMark,
get_bed_from_GeneMarkHMM,
get_bed_from_glimmer3
get_bed_from_glimmer3,
)
assert sys.version_info[:2] >= (2, 6)
# this defines the types of ftp files we are interested in, and how to process/convert them to a form for our use
desired_ftp_files = {'GeneMark': {'ext': 'GeneMark-2.5f', 'parser': 'process_GeneMark'},
'GeneMarkHMM': {'ext': 'GeneMarkHMM-2.6m', 'parser': 'process_GeneMarkHMM'},
'Glimmer3': {'ext': 'Glimmer3', 'parser': 'process_Glimmer3'},
'fna': {'ext': 'fna', 'parser': 'process_FASTA'},
'gbk': {'ext': 'gbk', 'parser': 'process_Genbank'}}
desired_ftp_files = {
"GeneMark": {"ext": "GeneMark-2.5f", "parser": "process_GeneMark"},
"GeneMarkHMM": {"ext": "GeneMarkHMM-2.6m", "parser": "process_GeneMarkHMM"},
"Glimmer3": {"ext": "Glimmer3", "parser": "process_Glimmer3"},
"fna": {"ext": "fna", "parser": "process_FASTA"},
"gbk": {"ext": "gbk", "parser": "process_Genbank"},
}
# number, name, chroms, kingdom, group, genbank, refseq, info_url, ftp_url
def iter_genome_projects(url="http://www.ncbi.nlm.nih.gov/genomes/lproks.cgi?view=1", info_url_base="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=genomeprj&cmd=Retrieve&dopt=Overview&list_uids="):
for row in BeautifulSoup(requests.get(url).text).findAll(name='tr', bgcolor=["#EEFFDD", "#E8E8DD"]):
def iter_genome_projects(
url="http://www.ncbi.nlm.nih.gov/genomes/lproks.cgi?view=1",
info_url_base="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=genomeprj&cmd=Retrieve&dopt=Overview&list_uids=",
):
for row in BeautifulSoup(requests.get(url).text).findAll(name="tr", bgcolor=["#EEFFDD", "#E8E8DD"]):
row = str(row).replace("\n", "").replace("\r", "")
fields = row.split("</td>")
org_num = fields[0].split("list_uids=")[-1].split("\"")[0]
org_num = fields[0].split("list_uids=")[-1].split('"')[0]
name = fields[1].split("\">")[-1].split("<")[0]
name = fields[1].split('">')[-1].split("<")[0]
kingdom = "archaea"
if "<td class=\"bacteria\" align=\"center\">B" in fields[2]:
if '<td class="bacteria" align="center">B' in fields[2]:
kingdom = "bacteria"
group = fields[3].split(">")[-1]
info_url = f"{info_url_base}{org_num}"
org_genbank = fields[7].split("\">")[-1].split("<")[0].split(".")[0]
org_refseq = fields[8].split("\">")[-1].split("<")[0].split(".")[0]
org_genbank = fields[7].split('">')[-1].split("<")[0].split(".")[0]
org_refseq = fields[8].split('">')[-1].split("<")[0].split(".")[0]
# seems some things donot have an ftp url, try and except it here:
try:
ftp_url = fields[22].split("href=\"")[1].split("\"")[0]
ftp_url = fields[22].split('href="')[1].split('"')[0]
except Exception:
print("FAILED TO AQUIRE FTP ADDRESS:", org_num, info_url)
ftp_url = None
@@ -68,7 +74,9 @@ def iter_genome_projects(url="http://www.ncbi.nlm.nih.gov/genomes/lproks.cgi?vie
yield org_num, name, chroms, kingdom, group, org_genbank, org_refseq, info_url, ftp_url
def get_chroms_by_project_id(org_num, base_url="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=genomeprj&cmd=Retrieve&dopt=Overview&list_uids="):
def get_chroms_by_project_id(
org_num, base_url="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=genomeprj&cmd=Retrieve&dopt=Overview&list_uids="
):
html_count = 0
html = None
while html_count < 500 and html is None:
@@ -114,7 +122,7 @@ def get_ftp_contents(ftp_url):
def scrape_ftp(ftp_contents, org_dir, org_num, refseq, ftp_url):
for file_type, items in desired_ftp_files.items():
ext = items['ext']
ext = items["ext"]
ftp_filename = f"{refseq}.{ext}"
target_filename = os.path.join(org_dir, f"{refseq}.{ext}")
if ftp_filename in ftp_contents:
@@ -133,8 +141,8 @@ def scrape_ftp(ftp_contents, org_dir, org_num, refseq, ftp_url):
return
# do special processing for each file type:
if items['parser'] is not None:
globals()[items['parser']](target_filename, org_num, refseq)
if items["parser"] is not None:
globals()[items["parser"]](target_filename, org_num, refseq)
else:
print("FTP filetype:", file_type, "not found for", org_num, refseq)
# FTP Files have been Loaded
@@ -142,7 +150,7 @@ def scrape_ftp(ftp_contents, org_dir, org_num, refseq, ftp_url):
def process_FASTA(filename, org_num, refseq):
fasta = []
fasta = [line.strip() for line in open(filename, 'rb').readlines()]
fasta = [line.strip() for line in open(filename, "rb").readlines()]
fasta_header = fasta.pop(0)[1:]
fasta_header_split = fasta_header.split("|")
chr_name = fasta_header_split.pop(-1).strip()
@@ -150,14 +158,14 @@ def process_FASTA(filename, org_num, refseq):
fasta = "".join(fasta)
# Create Chrom Info File:
chrom_info_file = open(os.path.join(os.path.split(filename)[0], "%s.info" % refseq), 'wb+')
chrom_info_file = open(os.path.join(os.path.split(filename)[0], "%s.info" % refseq), "wb+")
chrom_info_file.write(f"chromosome={refseq}\nname={chr_name}\nlength={len(fasta)}\norganism={org_num}\n")
try:
chrom_info_file.write("gi=%s\n" % accesions['gi'])
chrom_info_file.write("gi=%s\n" % accesions["gi"])
except Exception:
chrom_info_file.write("gi=None\n")
try:
chrom_info_file.write("gb=%s\n" % accesions['gb'])
chrom_info_file.write("gb=%s\n" % accesions["gb"])
except Exception:
chrom_info_file.write("gb=None\n")
try:
@@ -169,10 +177,10 @@ def process_FASTA(filename, org_num, refseq):
def process_Genbank(filename, org_num, refseq):
# extracts 'CDS', 'tRNA', 'rRNA' features from genbank file
features = get_bed_from_genbank(filename, refseq, ['CDS', 'tRNA', 'rRNA'])
features = get_bed_from_genbank(filename, refseq, ["CDS", "tRNA", "rRNA"])
for feature, values in features.items():
feature_file = open(os.path.join(os.path.split(filename)[0], f"{refseq}.{feature}.bed"), 'wb+')
feature_file.write('\n'.join(values))
feature_file = open(os.path.join(os.path.split(filename)[0], f"{refseq}.{feature}.bed"), "wb+")
feature_file.write("\n".join(values))
feature_file.close()
print("Genbank extraction finished for chrom:", refseq, "file:", filename)
@@ -183,8 +191,8 @@ def process_Glimmer3(filename, org_num, refseq):
except Exception as e:
print("Converting Glimmer3 to bed FAILED! For chrom:", refseq, "file:", filename, e)
glimmer3_bed = []
glimmer3_bed_file = open(os.path.join(os.path.split(filename)[0], "%s.Glimmer3.bed" % refseq), 'wb+')
glimmer3_bed_file.write('\n'.join(glimmer3_bed))
glimmer3_bed_file = open(os.path.join(os.path.split(filename)[0], "%s.Glimmer3.bed" % refseq), "wb+")
glimmer3_bed_file.write("\n".join(glimmer3_bed))
glimmer3_bed_file.close()
@@ -194,8 +202,8 @@ def process_GeneMarkHMM(filename, org_num, refseq):
except Exception as e:
print("Converting GeneMarkHMM to bed FAILED! For chrom:", refseq, "file:", filename, e)
geneMarkHMM_bed = []
geneMarkHMM_bed_bed_file = open(os.path.join(os.path.split(filename)[0], "%s.GeneMarkHMM.bed" % refseq), 'wb+')
geneMarkHMM_bed_bed_file.write('\n'.join(geneMarkHMM_bed))
geneMarkHMM_bed_bed_file = open(os.path.join(os.path.split(filename)[0], "%s.GeneMarkHMM.bed" % refseq), "wb+")
geneMarkHMM_bed_bed_file.write("\n".join(geneMarkHMM_bed))
geneMarkHMM_bed_bed_file.close()
@@ -205,8 +213,8 @@ def process_GeneMark(filename, org_num, refseq):
except Exception as e:
print("Converting GeneMark to bed FAILED! For chrom:", refseq, "file:", filename, e)
geneMark_bed = []
geneMark_bed_bed_file = open(os.path.join(os.path.split(filename)[0], "%s.GeneMark.bed" % refseq), 'wb+')
geneMark_bed_bed_file.write('\n'.join(geneMark_bed))
geneMark_bed_bed_file = open(os.path.join(os.path.split(filename)[0], "%s.GeneMark.bed" % refseq), "wb+")
geneMark_bed_bed_file.write("\n".join(geneMark_bed))
geneMark_bed_bed_file.close()
@@ -246,7 +254,7 @@ def __main__():
print("Org:", org_num, "chrom:", refseq, "[", time.time() - start_time, "seconds elapsed. ]")
# Create org info file
info_file = open(os.path.join(org_dir, "%s.info" % org_num), 'wb+')
info_file = open(os.path.join(org_dir, "%s.info" % org_num), "wb+")
info_file.write("genome project id=%s\n" % org_num)
info_file.write("name=%s\n" % name)
info_file.write("kingdom=%s\n" % kingdom)
scripts/microbes/ncbi_to_ucsc.py
+39 -27
View File
@@ -30,15 +30,15 @@ def __main__():
for line in info:
fields = line.replace("\n", "").split("=")
tmp_dict[fields[0]] = "=".join(fields[1:])
if 'genome project id' in tmp_dict.keys():
if tmp_dict['genome project id'] not in organisms.keys():
organisms[tmp_dict['genome project id']] = {'chrs': {}, 'base_dir': this_base_dir}
if "genome project id" in tmp_dict.keys():
if tmp_dict["genome project id"] not in organisms.keys():
organisms[tmp_dict["genome project id"]] = {"chrs": {}, "base_dir": this_base_dir}
for key in tmp_dict.keys():
organisms[tmp_dict['genome project id']][key] = tmp_dict[key]
organisms[tmp_dict["genome project id"]][key] = tmp_dict[key]
else:
if tmp_dict['organism'] not in organisms.keys():
organisms[tmp_dict['organism']] = {'chrs': {}, 'base_dir': this_base_dir}
organisms[tmp_dict['organism']]['chrs'][tmp_dict['chromosome']] = tmp_dict
if tmp_dict["organism"] not in organisms.keys():
organisms[tmp_dict["organism"]] = {"chrs": {}, "base_dir": this_base_dir}
organisms[tmp_dict["organism"]]["chrs"][tmp_dict["chromosome"]] = tmp_dict
# get UCSC data
@@ -62,9 +62,16 @@ def __main__():
builds = {}
for dsn in tree:
build = dsn.find("SOURCE").attrib['id']
build = dsn.find("SOURCE").attrib["id"]
try:
org_page = urlopen("http://archaea.ucsc.edu/cgi-bin/hgGateway?db=" + build).read().replace("\n", "").split("<table border=2 cellspacing=2 cellpadding=2>")[1].split("</table>")[0].split("</tr>")
org_page = (
urlopen("http://archaea.ucsc.edu/cgi-bin/hgGateway?db=" + build)
.read()
.replace("\n", "")
.split("<table border=2 cellspacing=2 cellpadding=2>")[1]
.split("</table>")[0]
.split("</tr>")
)
except Exception:
print("NO CHROMS FOR", build)
continue
@@ -76,36 +83,39 @@ def __main__():
chr = row.split("</a>")[0].split(">")[-1]
refseq = row.split("</a>")[-2].split(">")[-1]
for org in organisms:
for org_chr in organisms[org]['chrs']:
if organisms[org]['chrs'][org_chr]['chromosome'] == refseq:
for org_chr in organisms[org]["chrs"]:
if organisms[org]["chrs"][org_chr]["chromosome"] == refseq:
if org not in builds:
builds[org] = {'chrs': {}, 'build': build}
builds[org]['chrs'][refseq] = chr
builds[org] = {"chrs": {}, "build": build}
builds[org]["chrs"][refseq] = chr
print()
ext_to_edit = ['bed', 'info', ]
ext_to_edit = [
"bed",
"info",
]
for org in builds:
print(org, "changed to", builds[org]['build'])
print(org, "changed to", builds[org]["build"])
# org info file
info_file_old = os.path.join(base_dir + org, org + ".info")
info_file_new = os.path.join(base_dir + org, builds[org]['build'] + ".info")
info_file_new = os.path.join(base_dir + org, builds[org]["build"] + ".info")
old_dir = base_dir + org
new_dir = base_dir + builds[org]['build']
new_dir = base_dir + builds[org]["build"]
# open and edit org info file
info_file_contents = open(info_file_old).read()
info_file_contents = info_file_contents + "build=" + builds[org]['build'] + "\n"
for chrom in builds[org]['chrs']:
info_file_contents = info_file_contents.replace(chrom, builds[org]['chrs'][chrom])
info_file_contents = info_file_contents + "build=" + builds[org]["build"] + "\n"
for chrom in builds[org]["chrs"]:
info_file_contents = info_file_contents.replace(chrom, builds[org]["chrs"][chrom])
for result in os.walk(base_dir + org):
this_base_dir, sub_dirs, files = result
for file in files:
if file[0:len(chrom)] == chrom:
if file[0 : len(chrom)] == chrom:
# rename file
old_name = os.path.join(this_base_dir, file)
new_name = os.path.join(this_base_dir, builds[org]['chrs'][chrom] + file[len(chrom):])
new_name = os.path.join(this_base_dir, builds[org]["chrs"][chrom] + file[len(chrom) :])
move(old_name, new_name)
# edit contents of file, skiping those in list
@@ -113,20 +123,22 @@ def __main__():
continue
file_contents = open(new_name).read()
file_contents = file_contents.replace(chrom, builds[org]['chrs'][chrom])
file_contents = file_contents.replace(chrom, builds[org]["chrs"][chrom])
# special case fixes...
if file[-5:] == ".info":
file_contents = file_contents.replace("organism=" + org, "organism=" + builds[org]['build'])
file_contents = file_contents.replace("refseq=" + builds[org]['chrs'][chrom], "refseq=" + chrom)
file_contents = file_contents.replace("organism=" + org, "organism=" + builds[org]["build"])
file_contents = file_contents.replace(
"refseq=" + builds[org]["chrs"][chrom], "refseq=" + chrom
)
# write out new file
file_out = open(new_name, 'w')
file_out = open(new_name, "w")
file_out.write(file_contents)
file_out.close()
# write out org info file and remove old file
org_info_out = open(info_file_new, 'w')
org_info_out = open(info_file_new, "w")
org_info_out.write(info_file_contents)
org_info_out.close()
os.unlink(info_file_old)
scripts/microbes/util.py
+53 -20
View File
@@ -12,21 +12,21 @@ class Region:
self.qualifiers = {}
self.start = None
self.end = None
self.strand = '+'
self.strand = "+"
def set_coordinates_by_location(self, location):
location = location.strip().lower().replace('..', ',')
location = location.strip().lower().replace("..", ",")
if "complement(" in location: # if part of the sequence is on the negative strand, it all is?
self.strand = '-' # default of + strand
self.strand = "-" # default of + strand
for remove_text in ["join(", "order(", "complement(", ")"]:
location = location.replace(remove_text, "")
for number in location.split(','):
number = number.strip('\n\r\t <>,()')
for number in location.split(","):
number = number.strip("\n\r\t <>,()")
if number:
if "^" in number:
# a single point
# check that this is correct for points, ie: 413/NC_005027.gbk: misc_feature 6636286^6636287 ===> 6636285,6636286
end = int(number.split('^')[0])
end = int(number.split("^")[0])
start = end - 1
else:
end = int(number)
@@ -39,6 +39,7 @@ class Region:
class GenBankFeatureParser:
"""Parses Features from Single Locus GenBank file"""
def __init__(self, fh):
self.fh = fh
self.features = {}
@@ -48,7 +49,7 @@ class GenBankFeatureParser:
base_indent = 0
last_attr_name = None
for line in fh:
if not in_features and line.startswith('FEATURES'):
if not in_features and line.startswith("FEATURES"):
in_features = True
continue
if in_features:
@@ -71,8 +72,8 @@ class GenBankFeatureParser:
else:
# add info to last known feature
line = line.strip()
if line.startswith('/'):
fields = line[1:].split('=', 1)
if line.startswith("/"):
fields = line[1:].split("=", 1)
if len(fields) == 2:
last_attr_name, content = fields
else:
@@ -88,10 +89,14 @@ class GenBankFeatureParser:
self.features[last_feature_name][-1].set_coordinates_by_location(line)
else:
# continuation of multi-line qualifier content
if last_feature_name.lower() in ['translation']:
self.features[last_feature_name][-1].qualifiers[last_attr_name][-1] = "{}{}".format(self.features[last_feature_name][-1].qualifiers[last_attr_name][-1], line.rstrip('"'))
if last_feature_name.lower() in ["translation"]:
self.features[last_feature_name][-1].qualifiers[last_attr_name][-1] = "{}{}".format(
self.features[last_feature_name][-1].qualifiers[last_attr_name][-1], line.rstrip('"')
)
else:
self.features[last_feature_name][-1].qualifiers[last_attr_name][-1] = "{} {}".format(self.features[last_feature_name][-1].qualifiers[last_attr_name][-1], line.rstrip('"'))
self.features[last_feature_name][-1].qualifiers[last_attr_name][-1] = "{} {}".format(
self.features[last_feature_name][-1].qualifiers[last_attr_name][-1], line.rstrip('"')
)
def get_features_by_type(self, feature_type):
if feature_type not in self.features:
@@ -108,7 +113,7 @@ def get_bed_from_genbank(gb_file, chrom, feature_list):
features[feature_type] = []
for feature in genbank_parser.get_features_by_type(feature_type):
name = ""
for name_tag in ['gene', 'locus_tag', 'db_xref']:
for name_tag in ["gene", "locus_tag", "db_xref"]:
if name_tag in feature.qualifiers:
if name:
name = name + ";"
@@ -116,7 +121,9 @@ def get_bed_from_genbank(gb_file, chrom, feature_list):
if not name:
name = "unknown"
features[feature_type].append(f"{chrom}\t{feature.start}\t{feature.end}\t{name}\t{0}\t{feature.strand}") # append new bed field here
features[feature_type].append(
f"{chrom}\t{feature.start}\t{feature.end}\t{name}\t{0}\t{feature.strand}"
) # append new bed field here
return features
@@ -136,7 +143,14 @@ def get_bed_from_GeneMark(geneMark_filename, chr):
for block in orfs.split("\n\n"):
if block.startswith("List of Regions of interest"):
break
best_block = {'start': 0, 'end': 0, 'strand': '+', 'avg_prob': -sys.maxsize, 'start_prob': -sys.maxsize, 'name': 'DNE'}
best_block = {
"start": 0,
"end": 0,
"strand": "+",
"avg_prob": -sys.maxsize,
"start_prob": -sys.maxsize,
"name": "DNE",
}
ctr += 1
ctr2 = 0
for line in block.split("\n"):
@@ -145,7 +159,7 @@ def get_bed_from_GeneMark(geneMark_filename, chr):
start = int(fields.pop(0)) - 1
end = int(fields.pop(0))
strand = fields.pop(0)
if strand == 'complement':
if strand == "complement":
strand = "-"
else:
strand = "+"
@@ -157,9 +171,28 @@ def get_bed_from_GeneMark(geneMark_filename, chr):
except Exception:
start_prob = 0
name = "orf_" + str(ctr) + "_" + str(ctr2)
if avg_prob >= best_block['avg_prob'] and start_prob > best_block['start_prob']:
best_block = {'start': start, 'end': end, 'strand': strand, 'avg_prob': avg_prob, 'start_prob': start_prob, 'name': name}
regions.append(chr + "\t" + str(best_block['start']) + "\t" + str(best_block['end']) + "\t" + best_block['name'] + "\t" + str(int(best_block['avg_prob'] * 1000)) + "\t" + best_block['strand'])
if avg_prob >= best_block["avg_prob"] and start_prob > best_block["start_prob"]:
best_block = {
"start": start,
"end": end,
"strand": strand,
"avg_prob": avg_prob,
"start_prob": start_prob,
"name": name,
}
regions.append(
chr
+ "\t"
+ str(best_block["start"])
+ "\t"
+ str(best_block["end"])
+ "\t"
+ best_block["name"]
+ "\t"
+ str(int(best_block["avg_prob"] * 1000))
+ "\t"
+ best_block["strand"]
)
return regions
@@ -214,7 +247,7 @@ def get_bed_from_glimmer3(glimmer3_filename, chr):
else:
strand = "+"
start = start - 1
score = (float(fields.pop(0)))
score = float(fields.pop(0))
if score > max_score:
max_score = score
if score < min_score:
scripts/nosetests.py
+1 -1
View File
@@ -7,5 +7,5 @@ from pkg_resources import load_entry_point
assert sys.version_info[:2] >= (2, 7)
nose_core_TestProgram = load_entry_point('nose', 'console_scripts', 'nosetests')
nose_core_TestProgram = load_entry_point("nose", "console_scripts", "nosetests")
nose_core_TestProgram()
scripts/objectstore/copy_files_to_irods.py
+77 -28
View File
@@ -7,8 +7,11 @@ import uuid
from datetime import datetime
import irods.keywords as kw
from irods.exception import (CollectionDoesNotExist, DataObjectDoesNotExist,
NetworkException)
from irods.exception import (
CollectionDoesNotExist,
DataObjectDoesNotExist,
NetworkException,
)
from irods.session import iRODSSession
from psycopg2 import connect
@@ -48,7 +51,9 @@ last_accessed_sql_statement = """SELECT iq.dataset_id, MAX(iq.create_time) AS ma
"""
def copy_files_to_irods(start_dataset_id, end_dataset_id, object_store_info_file, irods_info_file, db_connection_info_file, copy_or_checksum):
def copy_files_to_irods(
start_dataset_id, end_dataset_id, object_store_info_file, irods_info_file, db_connection_info_file, copy_or_checksum
):
conn = None
session = None
osi_keys = None
@@ -97,7 +102,7 @@ def copy_files_to_irods(start_dataset_id, end_dataset_id, object_store_info_file
dbname=db_connection_info["dbname"],
user=db_connection_info["user"],
host=db_connection_info["host"],
password=db_connection_info["password"]
password=db_connection_info["password"],
)
conn.cursor()
@@ -105,7 +110,13 @@ def copy_files_to_irods(start_dataset_id, end_dataset_id, object_store_info_file
print(e)
return
session = iRODSSession(host=irods_info["host"], port=irods_info["port"], user=irods_info["user"], password=irods_info["password"], zone=irods_info["zone"])
session = iRODSSession(
host=irods_info["host"],
port=irods_info["port"],
user=irods_info["user"],
password=irods_info["password"],
zone=irods_info["zone"],
)
session.connection_timeout = int(irods_info["timeout"])
osi_keys = tuple(object_store_info.keys())
@@ -124,7 +135,7 @@ def copy_files_to_irods(start_dataset_id, end_dataset_id, object_store_info_file
try:
read_cursor = conn.cursor()
args = ('ok', start_dataset_id, end_dataset_id, osi_keys)
args = ("ok", start_dataset_id, end_dataset_id, osi_keys)
read_cursor.execute(read_sql_statement, args)
rows = read_cursor.fetchall()
for row in rows:
@@ -144,16 +155,20 @@ def copy_files_to_irods(start_dataset_id, end_dataset_id, object_store_info_file
irods_sub_folder = os.path.join(*directory_hash_id(uuid_with_dash))
disk_file_path = os.path.join(object_store_path, disk_sub_folder, "dataset_" + str(objectid) + ".dat")
disk_folder_path = os.path.join(object_store_path, disk_sub_folder, "dataset_" + str(objectid) + "_files")
irods_file_path = os.path.join(irods_info["home"], irods_sub_folder, "dataset_" + str(uuid_with_dash) + ".dat")
irods_file_path = os.path.join(
irods_info["home"], irods_sub_folder, "dataset_" + str(uuid_with_dash) + ".dat"
)
irods_file_collection_path = os.path.join(irods_info["home"], irods_sub_folder)
irods_folder_collection_path = os.path.join(irods_file_collection_path, "dataset_" + str(uuid_with_dash) + "_files")
irods_folder_collection_path = os.path.join(
irods_file_collection_path, "dataset_" + str(uuid_with_dash) + "_files"
)
if copy_or_checksum == "copy":
# Create the collection
session.collections.create(irods_file_collection_path)
# Add disk file to collection
options = {kw.REG_CHKSUM_KW: '', kw.RESC_NAME_KW: irods_resc}
options = {kw.REG_CHKSUM_KW: "", kw.RESC_NAME_KW: irods_resc}
session.data_objects.put(disk_file_path, irods_file_path, **options)
print(f"Copied disk file {disk_file_path} to irods {irods_file_path}")
@@ -163,7 +178,14 @@ def copy_files_to_irods(start_dataset_id, end_dataset_id, object_store_info_file
# Create the collection
session.collections.create(irods_folder_collection_path)
iput_command = "iput -R " + irods_resc + " -rk " + disk_folder_path_all_files + " " + irods_folder_collection_path
iput_command = (
"iput -R "
+ irods_resc
+ " -rk "
+ disk_folder_path_all_files
+ " "
+ irods_folder_collection_path
)
subprocess.call(iput_command, shell=True)
print(f"Copied disk folder {disk_folder_path} to irods {irods_folder_collection_path}")
@@ -177,7 +199,9 @@ def copy_files_to_irods(start_dataset_id, end_dataset_id, object_store_info_file
# obj.checksum is prepended with 'sha2:'. Remove that so we can compare it to disk file checksum
irods_file_checksum = obj.checksum[5:]
if irods_file_checksum != disk_file_checksum:
print(f"Error: irods file checksum {irods_file_checksum} does not match disk file checksum {disk_file_checksum} for irods file {irods_file_path} and disk file {disk_file_path}")
print(
f"Error: irods file checksum {irods_file_checksum} does not match disk file checksum {disk_file_checksum} for irods file {irods_file_path} and disk file {disk_file_path}"
)
continue
except (DataObjectDoesNotExist, CollectionDoesNotExist) as e:
print(e)
@@ -207,7 +231,9 @@ def copy_files_to_irods(start_dataset_id, end_dataset_id, object_store_info_file
# obj.checksum is prepended with 'sha2:'. Remove that so we can compare it to disk file checksum
an_irods_file_checksum = obj.checksum[5:]
if an_irods_file_checksum != a_disk_file_checksum:
print(f"Error: irods file checksum {an_irods_file_checksum} does not match disk file checksum {a_disk_file_checksum} for irods file {an_irods_file_path} and disk file {a_disk_file_path}")
print(
f"Error: irods file checksum {an_irods_file_checksum} does not match disk file checksum {a_disk_file_checksum} for irods file {an_irods_file_path} and disk file {a_disk_file_path}"
)
continue
except (DataObjectDoesNotExist, CollectionDoesNotExist) as e:
print(e)
@@ -225,9 +251,17 @@ def copy_files_to_irods(start_dataset_id, end_dataset_id, object_store_info_file
update_cursor.execute(update_sql_statement, (irods_info["object_store_id"], objectid))
updated_rows = update_cursor.rowcount
if updated_rows == 1:
print("Updated object store ID to {} in dataset table for object ID {}".format(irods_info["object_store_id"], objectid))
print(
"Updated object store ID to {} in dataset table for object ID {}".format(
irods_info["object_store_id"], objectid
)
)
else:
print("Error: Failed to update object store ID to {} in dataset table for object ID {}".format(irods_info["object_store_id"], objectid))
print(
"Error: Failed to update object store ID to {} in dataset table for object ID {}".format(
irods_info["object_store_id"], objectid
)
)
update_cursor.close()
# Delete file on disk
@@ -257,16 +291,20 @@ def get_irods_resource(conn, objectid, object_store_id, irods_info):
irods_tape_resc_cuttoff_dt = datetime.strptime(irods_tape_resc_cuttoff, "%m/%d/%Y")
read_cursor = conn.cursor()
args = ('ok', objectid, objectid, (object_store_id, ), 'ok', objectid, objectid, (object_store_id, ))
args = ("ok", objectid, objectid, (object_store_id,), "ok", objectid, objectid, (object_store_id,))
read_cursor.execute(last_accessed_sql_statement, args)
row = read_cursor.fetchone()
if row is None:
print(f"Could not find the last access time for dataset with id {objectid}. Returning the default resc {irods_resc}.")
print(
f"Could not find the last access time for dataset with id {objectid}. Returning the default resc {irods_resc}."
)
return irods_resc
dataset_id = row[0]
if int(dataset_id) != objectid:
print(f"The returned dataset id {dataset_id} does not match the passed in datsetid {objectid}. Returning the default resc {irods_resc}.")
print(
f"The returned dataset id {dataset_id} does not match the passed in datsetid {objectid}. Returning the default resc {irods_resc}."
)
return irods_resc
max_create_time = row[1]
@@ -274,10 +312,14 @@ def get_irods_resource(conn, objectid, object_store_id, irods_info):
max_create_time_dt = max_create_time.replace(tzinfo=None)
# If the last time a dataset was accessed was prior to a cuttoff date, use the tape resource. Otherwise, use the regular (non-tape) resource
if max_create_time_dt < irods_tape_resc_cuttoff_dt:
print(f"The last time dataset with id {objectid} was accessed {max_create_time_dt} is prior to tape resource cuttoff {irods_tape_resc_cuttoff_dt}. Using tape resource in irods.")
print(
f"The last time dataset with id {objectid} was accessed {max_create_time_dt} is prior to tape resource cuttoff {irods_tape_resc_cuttoff_dt}. Using tape resource in irods."
)
return irods_tape_resc
print(f"The last time dataset with id {objectid} was accessed {max_create_time_dt} is after the tape resource cuttoff {irods_tape_resc_cuttoff_dt}. Using regular (non-tape) resource in irods.")
print(
f"The last time dataset with id {objectid} was accessed {max_create_time_dt} is after the tape resource cuttoff {irods_tape_resc_cuttoff_dt}. Using regular (non-tape) resource in irods."
)
return irods_resc
except Exception as e:
@@ -291,7 +333,7 @@ def get_file_checksum(disk_file_path):
disk_file_checksum = subprocess.check_output(checksum_cmd, shell=True)
# remove '\n' from the end of disk_file_checksum
disk_file_checksum_len = len(disk_file_checksum)
disk_file_checksum_trimmed = disk_file_checksum[0:(disk_file_checksum_len - 1)]
disk_file_checksum_trimmed = disk_file_checksum[0 : (disk_file_checksum_len - 1)]
# Return Unicode string
return disk_file_checksum_trimmed.decode("utf-8")
@@ -336,15 +378,15 @@ def print_help_msg():
print(help_msg)
if __name__ == '__main__':
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('-s', '--start_dataset_id', type=int, required=True)
parser.add_argument('-e', '--end_dataset_id', type=int, required=True)
parser.add_argument('-o', '--object_store_info_file', type=str, required=True)
parser.add_argument('-i', '--irods_info_file', type=str, required=True)
parser.add_argument('-d', '--db_connection_info_file', type=str, required=True)
parser.add_argument('-c', '--copy_or_checksum', type=str, required=True, choices=['copy', 'checksum'])
parser.add_argument("-s", "--start_dataset_id", type=int, required=True)
parser.add_argument("-e", "--end_dataset_id", type=int, required=True)
parser.add_argument("-o", "--object_store_info_file", type=str, required=True)
parser.add_argument("-i", "--irods_info_file", type=str, required=True)
parser.add_argument("-d", "--db_connection_info_file", type=str, required=True)
parser.add_argument("-c", "--copy_or_checksum", type=str, required=True, choices=["copy", "checksum"])
args = parser.parse_args()
print(args)
@@ -356,4 +398,11 @@ if __name__ == '__main__':
db_connection_info_file = args.db_connection_info_file
copy_or_checksum = args.copy_or_checksum
copy_files_to_irods(start_dataset_id=start_dataset_id, end_dataset_id=end_dataset_id, object_store_info_file=object_store_info_file, irods_info_file=irods_info_file, db_connection_info_file=db_connection_info_file, copy_or_checksum=copy_or_checksum)
copy_files_to_irods(
start_dataset_id=start_dataset_id,
end_dataset_id=end_dataset_id,
object_store_info_file=object_store_info_file,
irods_info_file=irods_info_file,
db_connection_info_file=db_connection_info_file,
copy_or_checksum=copy_or_checksum,
)
scripts/pages_identifier_conversion.py
+20 -10
View File
@@ -4,23 +4,29 @@ import logging
import os
import sys
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib")))
import galaxy
import galaxy.app
import galaxy.config
from galaxy.managers.pages import PageContentProcessor, placeholderRenderForSave
from galaxy.managers.pages import (
PageContentProcessor,
placeholderRenderForSave,
)
from galaxy.objectstore import build_object_store_from_config
from galaxy.security.idencoding import IdEncodingHelper
from galaxy.util import unicodify
from galaxy.util.bunch import Bunch
from galaxy.util.script import app_properties_from_args, populate_config_args
from galaxy.util.script import (
app_properties_from_args,
populate_config_args,
)
def main(argv):
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-k', '--secret-key', help='Key to convert pages with', default='')
parser.add_argument('-d', '--dry-run', help='No changes, just test it.', action='store_true')
parser.add_argument("-k", "--secret-key", help="Key to convert pages with", default="")
parser.add_argument("-d", "--dry-run", help="No changes, just test it.", action="store_true")
populate_config_args(parser)
args = parser.parse_args()
properties = app_properties_from_args(args)
@@ -29,7 +35,9 @@ def main(argv):
security_helper = IdEncodingHelper(id_secret=secret)
object_store = build_object_store_from_config(config)
if not config.database_connection:
print("The database connection is empty. If you are using the default value, please uncomment that in your galaxy.yml")
print(
"The database connection is empty. If you are using the default value, please uncomment that in your galaxy.yml"
)
model = galaxy.config.init_models_from_config(config, object_store=object_store)
session = model.context.current
@@ -39,19 +47,21 @@ def main(argv):
try:
processor = PageContentProcessor(mock_trans, placeholderRenderForSave)
processor.feed(p.content)
newcontent = unicodify(processor.output(), 'utf-8')
newcontent = unicodify(processor.output(), "utf-8")
if p.content != newcontent:
if not args.dry_run:
p.content = unicodify(processor.output(), 'utf-8')
p.content = unicodify(processor.output(), "utf-8")
session.add(p)
session.flush()
else:
print("Modifying revision %s." % p.id)
print(difflib.unified_diff(p.content, newcontent))
except Exception:
logging.exception("Error parsing page, rolling changes back and skipping revision %s. Please report this error." % p.id)
logging.exception(
"Error parsing page, rolling changes back and skipping revision %s. Please report this error." % p.id
)
session.rollback()
if __name__ == '__main__':
if __name__ == "__main__":
main(sys.argv)
scripts/paster.py
+3 -2
View File
@@ -8,10 +8,11 @@ top level directly.
import os
import sys
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib")))
from check_python import check_python # noqa: I100, I201
from galaxy.util.pastescript import serve
from check_python import check_python # noqa: I100, I201
# ensure supported version
try:
scripts/plugin_staging.py
+6 -4
View File
@@ -6,8 +6,8 @@ GXY_ROOT = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file_
def stage_static(f):
src = os.path.join(GXY_ROOT, 'config/plugins', f)
dest = os.path.join(GXY_ROOT, 'static/plugins', f)
src = os.path.join(GXY_ROOT, "config/plugins", f)
dest = os.path.join(GXY_ROOT, "static/plugins", f)
dest_parent = os.path.abspath(os.path.join(dest, os.pardir))
if os.path.lexists(dest):
# We have to clear out the old staged or linked static to relink.
@@ -29,6 +29,8 @@ def stage_static(f):
if __name__ == "__main__":
# This is not awesome, but it's temporary, and it supports two-tier plugin static.
for f in glob.glob(os.path.join(GXY_ROOT, 'config/plugins/*/*/static')) + glob.glob(os.path.join(GXY_ROOT, 'config/plugins/*/*/*/static')):
f = os.path.relpath(f, os.path.join(GXY_ROOT, 'config/plugins'))
for f in glob.glob(os.path.join(GXY_ROOT, "config/plugins/*/*/static")) + glob.glob(
os.path.join(GXY_ROOT, "config/plugins/*/*/*/static")
):
f = os.path.relpath(f, os.path.join(GXY_ROOT, "config/plugins"))
stage_static(f)
scripts/release-diff.py
+16 -32
View File
@@ -45,8 +45,8 @@ def diff_files(old, new):
new_k = set(new_kv.keys())
added = []
for item in (new_k - old_k):
parent = '.'.join(item.split('.')[0:-1])
for item in new_k - old_k:
parent = ".".join(item.split(".")[0:-1])
if parent in new_k and parent not in old_k:
added.append(item)
else:
@@ -54,8 +54,8 @@ def diff_files(old, new):
added = set(added)
removed = []
for item in (old_k - new_k):
parent = '.'.join(item.split('.')[0:-1])
for item in old_k - new_k:
parent = ".".join(item.split(".")[0:-1])
if parent in old_k and parent not in new_k:
removed.append(item)
else:
@@ -85,7 +85,7 @@ def _report_dict(title, subheading, data, mapper):
def _indent(s, by=4):
whitespace = ' ' * by
whitespace = " " * by
s = s if isinstance(s, list) else s.splitlines()
return "\n".join((f"{whitespace}{line}" for line in s))
@@ -98,27 +98,19 @@ def report_diff(added, changed, removed, new_files):
print()
if added:
_report_dict(
"Added",
"The following configuration options are new",
added,
lambda x: f"- {x}"
)
_report_dict("Added", "The following configuration options are new", added, lambda x: f"- {x}")
if changed:
_report_dict(
"Changed",
"The following configuration options have been changed",
changed,
lambda x: f"- {x[0]} has changed from\n\n ::\n\n{_indent(x[1])}\n\n to\n\n ::\n\n{_indent(x[2])}\n\n"
lambda x: f"- {x[0]} has changed from\n\n ::\n\n{_indent(x[1])}\n\n to\n\n ::\n\n{_indent(x[2])}\n\n",
)
if removed:
_report_dict(
"Removed",
"The following configuration options have been completely removed",
removed,
lambda x: f"- {x}"
"Removed", "The following configuration options have been completely removed", removed, lambda x: f"- {x}"
)
if new_files:
@@ -133,9 +125,7 @@ def report_diff(added, changed, removed, new_files):
def load_at_time(path, revision=None):
if revision is not None:
return subprocess.check_output(
["git", "show", f"{revision}:{path}"], stderr=subprocess.STDOUT
)
return subprocess.check_output(["git", "show", f"{revision}:{path}"], stderr=subprocess.STDOUT)
else:
with open(path) as handle:
return handle.read()
@@ -156,19 +146,15 @@ def main(old_revision, new_revision=None):
for file in files_to_diff:
filename = file
if 'config_schema.yml' in file:
filename = 'config/galaxy.yml.sample:galaxy'
elif 'uwsgi_schema.yml' in file:
filename = 'config/galaxy.yml.sample:uwsgi'
if "config_schema.yml" in file:
filename = "config/galaxy.yml.sample:galaxy"
elif "uwsgi_schema.yml" in file:
filename = "config/galaxy.yml.sample:uwsgi"
real_path = Path(file).resolve().relative_to(Path.cwd())
try:
old_contents = yaml.load(
load_at_time(real_path, old_revision), Loader=MockOrderedLoader
)
new_contents = yaml.load(
load_at_time(real_path, new_revision), Loader=MockOrderedLoader
)
old_contents = yaml.load(load_at_time(real_path, old_revision), Loader=MockOrderedLoader)
new_contents = yaml.load(load_at_time(real_path, new_revision), Loader=MockOrderedLoader)
(a, r, c) = diff_files(old_contents, new_contents)
if a:
@@ -187,9 +173,7 @@ def main(old_revision, new_revision=None):
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Diff yaml configuration files between two points in time."
)
parser = argparse.ArgumentParser(description="Diff yaml configuration files between two points in time.")
parser.add_argument("old_revision", help="Old revision")
parser.add_argument(
"--new_revision",
scripts/resumable_upload.py
+33 -34
View File
@@ -6,30 +6,29 @@ import requests
from tusclient import client
from tusclient.storage import filestorage
UPLOAD_ENDPOINT = '/api/upload/resumable_upload'
SUBMISSION_ENDPOINT = '/api/tools/fetch'
CHUNK_SIZE = 10 ** 7
UPLOAD_ENDPOINT = "/api/upload/resumable_upload"
SUBMISSION_ENDPOINT = "/api/tools/fetch"
CHUNK_SIZE = 10**7
@click.command()
@click.option("--url", default='http://localhost:8080', help="URL of Galaxy instance")
@click.option("--url", default="http://localhost:8080", help="URL of Galaxy instance")
@click.option("--api_key", envvar="GALAXY_API_KEY", required=True, help="API key for Galaxy instance")
@click.option('--history_id', type=str, required=True, help="Target History ID")
@click.option('--file_type', default="auto", type=str, help="Galaxy file type to use")
@click.option('--dbkey', default="?", type=str, help="Genome Build for dataset")
@click.option('--filename', type=str, help="Filename to use in Galaxy history, if different from path")
@click.option('--storage', type=click.Path(), required=False, help="Store URLs to resume here")
@click.argument('path', type=click.Path())
def upload_file(url, path, api_key, history_id, file_type='auto', dbkey='?', filename=None, storage=None):
headers = {'x-api-key': api_key}
@click.option("--history_id", type=str, required=True, help="Target History ID")
@click.option("--file_type", default="auto", type=str, help="Galaxy file type to use")
@click.option("--dbkey", default="?", type=str, help="Genome Build for dataset")
@click.option("--filename", type=str, help="Filename to use in Galaxy history, if different from path")
@click.option("--storage", type=click.Path(), required=False, help="Store URLs to resume here")
@click.argument("path", type=click.Path())
def upload_file(url, path, api_key, history_id, file_type="auto", dbkey="?", filename=None, storage=None):
headers = {"x-api-key": api_key}
my_client = client.TusClient(f"{url}{UPLOAD_ENDPOINT}", headers=headers)
filename = filename or os.path.basename(path)
metadata = {
'filename': filename,
'history_id': history_id,
'file_type': file_type,
'dbkey': dbkey,
"filename": filename,
"history_id": history_id,
"file_type": file_type,
"dbkey": dbkey,
}
# Upload a file to a tus server.
@@ -40,26 +39,26 @@ def upload_file(url, path, api_key, history_id, file_type='auto', dbkey='?', fil
uploader.upload()
# Extract session from created upload URL
session_id = uploader.url.rsplit('/', 1)[1]
session_id = uploader.url.rsplit("/", 1)[1]
payload = {
'history_id': history_id,
'targets': json.dumps([
{
"destination": {"type": "hdas"},
"elements": [
{
"src": "files",
"ext": file_type,
"dbkey": dbkey,
"name": filename
}
]
}
]),
"history_id": history_id,
"targets": json.dumps(
[
{
"destination": {"type": "hdas"},
"elements": [{"src": "files", "ext": file_type, "dbkey": dbkey, "name": filename}],
}
]
),
}
response = requests.post(f"{url}{SUBMISSION_ENDPOINT}", data=payload, files={'files_0|file_data': json.dumps({"session_id": session_id})}, headers=headers)
response = requests.post(
f"{url}{SUBMISSION_ENDPOINT}",
data=payload,
files={"files_0|file_data": json.dumps({"session_id": session_id})},
headers=headers,
)
response.raise_for_status()
if __name__ == '__main__':
if __name__ == "__main__":
upload_file()
scripts/rst2html.py
+5 -5
View File
@@ -12,16 +12,16 @@ A minimal front end to the Docutils Publisher, producing HTML.
try:
import locale
locale.setlocale(locale.LC_ALL, '')
locale.setlocale(locale.LC_ALL, "")
except Exception:
pass
from docutils.core import (
default_description,
publish_cmdline
publish_cmdline,
)
description = ('Generates (X)HTML documents from standalone reStructuredText '
'sources. ' + default_description)
description = "Generates (X)HTML documents from standalone reStructuredText " "sources. " + default_description
publish_cmdline(writer_name='html', description=description)
publish_cmdline(writer_name="html", description=description)
scripts/runtime_stats.py
+65 -76
View File
@@ -40,13 +40,15 @@ import psycopg2
from sqlalchemy.engine import url
galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
sys.path.insert(1, os.path.join(galaxy_root, 'lib'))
sys.path.insert(1, os.path.join(galaxy_root, "lib"))
import galaxy.config
from galaxy.util.script import app_properties_from_args, populate_config_args
from galaxy.util.script import (
app_properties_from_args,
populate_config_args,
)
DATA_SOURCES = ('metrics', 'history')
DATA_SOURCES = ("metrics", "history")
METRICS_SQL = """
SELECT metric_value
FROM job_metric_numeric jmn
@@ -74,68 +76,54 @@ HISTORY_SQL = """
def parse_arguments():
parser = argparse.ArgumentParser(
description='Generate walltime statistics')
parser.add_argument('tool_id', help='Tool (by ID) to collect stats about')
parser.add_argument('--like',
action='store_true',
default=False,
help='Use SQL `LIKE` operator to find '
'a shed-installed tool using the tool\'s '
'"short" id')
parser = argparse.ArgumentParser(description="Generate walltime statistics")
parser.add_argument("tool_id", help="Tool (by ID) to collect stats about")
parser.add_argument(
"--like",
action="store_true",
default=False,
help="Use SQL `LIKE` operator to find " "a shed-installed tool using the tool's " '"short" id',
)
populate_config_args(parser)
parser.add_argument('-d', '--debug',
action='store_true',
default=False,
help='Print extra info')
parser.add_argument('-m', '--min',
type=int,
default=-1,
help='Ignore runtimes less than MIN seconds')
parser.add_argument('-M', '--max',
type=int,
default=-1,
help='Ignore runtimes greater than MAX seconds')
parser.add_argument('-u', '--user',
help='Return stats for only this user (id, email, '
'or username)')
parser.add_argument('-s', '--source',
default='metrics',
help='Runtime data source (SOURCES: %s)'
% ', '.join(DATA_SOURCES))
parser.add_argument("-d", "--debug", action="store_true", default=False, help="Print extra info")
parser.add_argument("-m", "--min", type=int, default=-1, help="Ignore runtimes less than MIN seconds")
parser.add_argument("-M", "--max", type=int, default=-1, help="Ignore runtimes greater than MAX seconds")
parser.add_argument("-u", "--user", help="Return stats for only this user (id, email, " "or username)")
parser.add_argument(
"-s", "--source", default="metrics", help="Runtime data source (SOURCES: %s)" % ", ".join(DATA_SOURCES)
)
args = parser.parse_args()
if args.like and '/' in args.tool_id:
print('ERROR: Do not use --like with a tool shed tool id (the tool '
'id should not contain `/` characters)')
if args.like and "/" in args.tool_id:
print("ERROR: Do not use --like with a tool shed tool id (the tool " "id should not contain `/` characters)")
sys.exit(2)
args.source = args.source.lower()
if args.source not in ('metrics', 'history'):
print('ERROR: Data source `%s` unknown, valid source are: %s'
% (args.source, ', '.join(DATA_SOURCES)))
if args.source not in ("metrics", "history"):
print("ERROR: Data source `%s` unknown, valid source are: %s" % (args.source, ", ".join(DATA_SOURCES)))
app_properties = app_properties_from_args(args)
config = galaxy.config.Configuration(**app_properties)
uri = args.config.get_database_url(config)
names = {'database': 'dbname', 'username': 'user'}
names = {"database": "dbname", "username": "user"}
args.connect_args = url.make_url(uri).translate_connect_args(**names)
if args.debug:
print('Got options:')
print("Got options:")
for i in vars(args).items():
print('%s: %s' % i)
print("%s: %s" % i)
return args
def query(tool_id=None, user=None, like=None, source='metrics',
connect_args=None, debug=False, min=-1, max=-1, **kwargs):
def query(
tool_id=None, user=None, like=None, source="metrics", connect_args=None, debug=False, min=-1, max=-1, **kwargs
):
connect_arg_str = ''
connect_arg_str = ""
for k, v in connect_args.items():
connect_arg_str += f'{k}={v}'
connect_arg_str += f"{k}={v}"
pc = psycopg2.connect(connect_arg_str)
cur = pc.cursor()
@@ -144,26 +132,26 @@ def query(tool_id=None, user=None, like=None, source='metrics',
try:
user_id = int(user)
except ValueError:
if '@' not in user:
field = 'username'
if "@" not in user:
field = "username"
else:
field = 'email'
sql = 'SELECT id FROM galaxy_user WHERE {} = {}'.format(field, '%s')
field = "email"
sql = "SELECT id FROM galaxy_user WHERE {} = {}".format(field, "%s")
cur.execute(sql, (user,))
if debug:
print('Executed:')
print("Executed:")
print(cur.query)
row = cur.fetchone()
if row:
user_id = row[0]
else:
print('Invalid user: %s' % user)
print("Invalid user: %s" % user)
sys.exit(1)
if like:
query_tool_id = '%%/%s/%%' % tool_id
elif '/' in tool_id and not re.match(r'\d+\.\d+', tool_id.split('/')[-1]):
query_tool_id = '%s%%' % tool_id
query_tool_id = "%%/%s/%%" % tool_id
elif "/" in tool_id and not re.match(r"\d+\.\d+", tool_id.split("/")[-1]):
query_tool_id = "%s%%" % tool_id
like = True
else:
query_tool_id = tool_id
@@ -181,7 +169,7 @@ def query(tool_id=None, user=None, like=None, source='metrics',
else:
user_clause = ""
if source == 'metrics':
if source == "metrics":
if min > 0 and max > 0:
time_clause = """AND metric_value > %s
AND metric_value < %s"""
@@ -196,56 +184,57 @@ def query(tool_id=None, user=None, like=None, source='metrics',
else:
time_clause = ""
sql = METRICS_SQL
elif source == 'history':
elif source == "history":
if min > 0 and max > 0:
time_clause = """WHERE ctimes[1] - ctimes[2] > interval %s
AND ctimes[1] - ctimes[2] < interval %s"""
sql_args.append('%s seconds' % min)
sql_args.append('%s seconds' % max)
sql_args.append("%s seconds" % min)
sql_args.append("%s seconds" % max)
elif min > 0:
time_clause = "WHERE ctimes[1] - ctimes[2] > interval %s"
sql_args.append('%s seconds' % min)
sql_args.append("%s seconds" % min)
elif max > 0:
time_clause = "WHERE ctimes[1] - ctimes[2] < interval %s"
sql_args.append('%s seconds' % max)
sql_args.append("%s seconds" % max)
else:
time_clause = ""
sql = HISTORY_SQL
sql = sql.format(tool_clause=tool_clause, user_clause=user_clause,
time_clause=time_clause)
sql = sql.format(tool_clause=tool_clause, user_clause=user_clause, time_clause=time_clause)
cur.execute(sql, sql_args)
if debug:
print('Executed:')
print("Executed:")
print(cur.query)
print('Query returned %d rows' % cur.rowcount)
print("Query returned %d rows" % cur.rowcount)
if source == 'metrics':
if source == "metrics":
times = numpy.array([r[0] for r in cur if r[0]])
elif source == 'history':
elif source == "history":
times = numpy.array([r[0].total_seconds() for r in cur if r[0]])
print('Collected %d times' % times.size)
print("Collected %d times" % times.size)
if times.size == 0:
return
if user:
print('Displaying statistics for user %s' % user)
print("Displaying statistics for user %s" % user)
stats = (('Mean runtime', numpy.mean(times)),
('Standard deviation', numpy.std(times)),
('Minimum runtime', times.min()),
('Maximum runtime', times.max()))
stats = (
("Mean runtime", numpy.mean(times)),
("Standard deviation", numpy.std(times)),
("Minimum runtime", times.min()),
("Maximum runtime", times.max()),
)
for name, seconds in stats:
hours, minutes = nice_times(seconds)
msg = name + ' is %0.0f seconds' % seconds
msg = name + " is %0.0f seconds" % seconds
if minutes:
msg += ' (=%0.2f minutes)' % minutes
msg += " (=%0.2f minutes)" % minutes
if hours:
msg += ' (=%0.2f hours)' % hours
msg += " (=%0.2f hours)" % hours
print(msg)
@@ -267,5 +256,5 @@ def main():
query(**vars(args))
if __name__ == '__main__':
if __name__ == "__main__":
main()
scripts/secret_decoder_ring.py
+12 -13
View File
@@ -7,22 +7,21 @@ import logging
import os
import sys
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib")))
from galaxy.security.idencoding import IdEncodingHelper
from galaxy.util import unicodify
from galaxy.util.script import app_properties_from_args, populate_config_args
from galaxy.util.script import (
app_properties_from_args,
populate_config_args,
)
logging.basicConfig()
log = logging.getLogger(__name__)
parser = argparse.ArgumentParser()
parser.add_argument('action', metavar='ACTION', type=str,
default=None,
help='decode|encode')
parser.add_argument('value', metavar='VALUE', type=str,
default=None,
help='value to encode or decode')
parser.add_argument("action", metavar="ACTION", type=str, default=None, help="decode|encode")
parser.add_argument("value", metavar="VALUE", type=str, default=None, help="value to encode or decode")
populate_config_args(parser)
args = parser.parse_args()
@@ -33,16 +32,16 @@ app_properties = app_properties_from_args(args)
if "id_secret" not in app_properties:
log.warning('No ID_SECRET specified. Please set the "id_secret" in your galaxy.yml.')
id_secret = app_properties.get('id_secret', 'dangerous_default')
id_secret = app_properties.get("id_secret", "dangerous_default")
security_helper = IdEncodingHelper(id_secret=id_secret)
# And get access to the models
# Login manager to manage current_user functionality
if args.action == 'decode':
sys.stdout.write(security_helper.decode_guid(args.value.lstrip('F')))
elif args.action == 'encode':
if args.action == "decode":
sys.stdout.write(security_helper.decode_guid(args.value.lstrip("F")))
elif args.action == "encode":
sys.stdout.write(unicodify(security_helper.encode_guid(args.value)))
else:
sys.stdout.write("Unknown argument")
sys.stdout.write('\n')
sys.stdout.write("\n")
scripts/set_dataset_sizes.py
+11 -8
View File
@@ -4,11 +4,14 @@ import argparse
import os
import sys
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib")))
import galaxy.config
from galaxy.objectstore import build_object_store_from_config
from galaxy.util.script import app_properties_from_args, populate_config_args
from galaxy.util.script import (
app_properties_from_args,
populate_config_args,
)
parser = argparse.ArgumentParser()
populate_config_args(parser)
@@ -24,16 +27,16 @@ def init():
return model, object_store
if __name__ == '__main__':
print('Loading Galaxy model...')
if __name__ == "__main__":
print("Loading Galaxy model...")
model, object_store = init()
sa_session = model.context.current
set = 0
dataset_count = sa_session.query(model.Dataset).count()
print('Processing %i datasets...' % dataset_count)
print("Processing %i datasets..." % dataset_count)
percent = 0
print('Completed %i%%' % percent, end=' ')
print("Completed %i%%" % percent, end=" ")
sys.stdout.flush()
for i, dataset in enumerate(sa_session.query(model.Dataset).enable_eagerloads(False).yield_per(1000)):
if dataset.total_size is None:
@@ -44,8 +47,8 @@ if __name__ == '__main__':
new_percent = int(float(i) / dataset_count * 100)
if new_percent != percent:
percent = new_percent
print('\rCompleted %i%%' % percent, end=' ')
print("\rCompleted %i%%" % percent, end=" ")
sys.stdout.flush()
sa_session.flush()
print('\rCompleted 100%')
print("\rCompleted 100%")
object_store.shutdown()
scripts/set_user_disk_usage.py
+28 -19
View File
@@ -4,28 +4,37 @@ import argparse
import os
import sys
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib")))
import galaxy.config
from galaxy.objectstore import build_object_store_from_config
from galaxy.util import nice_size
from galaxy.util.script import app_properties_from_args, populate_config_args
from galaxy.util.script import (
app_properties_from_args,
populate_config_args,
)
default_config = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'config/galaxy.ini'))
default_config = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "config/galaxy.ini"))
parser = argparse.ArgumentParser()
parser.add_argument('-u', '--username', dest='username', help='Username of user to update', default='all')
parser.add_argument('-e', '--email', dest='email', help='Email address of user to update', default='all')
parser.add_argument('--dry-run', dest='dryrun', help='Dry run (show changes but do not save to database)', action='store_true', default=False)
parser.add_argument("-u", "--username", dest="username", help="Username of user to update", default="all")
parser.add_argument("-e", "--email", dest="email", help="Email address of user to update", default="all")
parser.add_argument(
"--dry-run",
dest="dryrun",
help="Dry run (show changes but do not save to database)",
action="store_true",
default=False,
)
populate_config_args(parser)
args = parser.parse_args()
def init():
if args.username == 'all':
if args.username == "all":
args.username = None
if args.email == 'all':
if args.email == "all":
args.email = None
app_properties = app_properties_from_args(args)
@@ -38,7 +47,7 @@ def init():
def quotacheck(sa_session, users, engine):
sa_session.refresh(user)
current = user.get_disk_usage()
print(user.username, '<' + user.email + '>:', end=' ')
print(user.username, "<" + user.email + ">:", end=" ")
if not args.dryrun:
# Apply new disk usage
@@ -48,28 +57,28 @@ def quotacheck(sa_session, users, engine):
else:
new = user.calculate_disk_usage()
print('old usage:', nice_size(current), 'change:', end=' ')
print("old usage:", nice_size(current), "change:", end=" ")
if new in (current, None):
print('none')
print("none")
else:
if new > current:
print('+%s' % (nice_size(new - current)))
print("+%s" % (nice_size(new - current)))
else:
print('-%s' % (nice_size(current - new)))
print("-%s" % (nice_size(current - new)))
if __name__ == '__main__':
print('Loading Galaxy model...')
if __name__ == "__main__":
print("Loading Galaxy model...")
model, object_store, engine = init()
sa_session = model.context.current
if not args.username and not args.email:
user_count = sa_session.query(model.User).count()
print('Processing %i users...' % user_count)
print("Processing %i users..." % user_count)
for i, user in enumerate(sa_session.query(model.User).enable_eagerloads(False).yield_per(1000)):
print('%3i%%' % int(float(i) / user_count * 100), end=' ')
print("%3i%%" % int(float(i) / user_count * 100), end=" ")
quotacheck(sa_session, user, engine)
print('100% complete')
print("100% complete")
object_store.shutdown()
sys.exit(0)
elif args.username:
@@ -77,7 +86,7 @@ if __name__ == '__main__':
elif args.email:
user = sa_session.query(model.User).enable_eagerloads(False).filter_by(email=args.email).first()
if not user:
print('User not found')
print("User not found")
sys.exit(1)
object_store.shutdown()
quotacheck(sa_session, user, engine)
scripts/slideshow/build_slideshow.py
+7 -6
View File
@@ -2,7 +2,6 @@ import os
import string
import sys
SCRIPTS_DIRECTORY = os.path.dirname(__file__)
TEMPLATE_PATH = os.path.join(SCRIPTS_DIRECTORY, "slideshow_template.html")
TEMPLATE = string.Template(open(TEMPLATE_PATH).read())
@@ -13,13 +12,15 @@ def main(argv=None):
argv = sys.argv
title = argv[1]
markdown_source = argv[2]
output = os.path.splitext(markdown_source)[0] + '.html'
output = os.path.splitext(markdown_source)[0] + ".html"
with open(markdown_source) as s:
content = s.read()
html = TEMPLATE.safe_substitute(**{
'title': title,
'content': content,
})
html = TEMPLATE.safe_substitute(
**{
"title": title,
"content": content,
}
)
print(html)
open(output, "w").write(html)
scripts/summarize_timings.py
+1 -8
View File
@@ -5,7 +5,6 @@ from argparse import ArgumentParser
import numpy
DESCRIPTION = ""
TIMING_LINE_PATTERN = re.compile(r"\((\d+.\d+) ms\)")
@@ -36,13 +35,7 @@ def main(argv=None):
print(line.strip())
template = "Summary (ms) - Mean: %f, Median: %f, Max: %f, Min: %f, StdDev: %f"
message = template % (
numpy.mean(times),
numpy.median(times),
numpy.max(times),
numpy.min(times),
numpy.std(times)
)
message = template % (numpy.mean(times), numpy.median(times), numpy.max(times), numpy.min(times), numpy.std(times))
print(message)
scripts/tests_markdown.py
+99 -97
View File
@@ -8,15 +8,12 @@ import sys
import jinja2
from mir import html_report
DESCRIPTION = "Script to generate (potentially merged) HTML summary of Galaxy Test Performance"
templateLoader = jinja2.FileSystemLoader(searchpath="./scripts")
template_env = jinja2.Environment(loader=templateLoader)
TEMPLATE_FILE = "tests_markdown.tpl"
TEMPLATE_COMPARE_FILE = "tests_markdown_compare.tpl"
LINKS = [
{"href": "https://github.com/galaxyproject/galaxy", "title": "Galaxy"}
]
LINKS = [{"href": "https://github.com/galaxyproject/galaxy", "title": "Galaxy"}]
def main(argv=None):
@@ -71,20 +68,22 @@ def _merge_summarizes(raw_data_dicts):
all_labels = set()
for raw_data_dict in raw_data_dicts:
these_api_endpoints = raw_data_dict['raw_data']['api_endpoint_metrics']
these_api_endpoints = raw_data_dict["raw_data"]["api_endpoint_metrics"]
for api_endpoint in these_api_endpoints.keys():
all_api_endpoints.add(api_endpoint)
these_internal_endpoints = raw_data_dict['raw_data']['internals_metrics']
these_internal_endpoints = raw_data_dict["raw_data"]["internals_metrics"]
for internal_endpoint in these_internal_endpoints.keys():
all_internal_metrics.add(internal_endpoint)
these_tests = raw_data_dict['raw_data']['tests']
these_tests = raw_data_dict["raw_data"]["tests"]
for test in these_tests:
all_tests.add(test['nodeid'])
all_tests.add(test["nodeid"])
all_labels.add(raw_data_dict["label"])
for label in all_labels:
for api_endpoint in all_api_endpoints:
_ensure_has_dict_at_key(api_endpoints, api_endpoint,
_ensure_has_dict_at_key(
api_endpoints,
api_endpoint,
total_time={},
sql_time={},
sql_queries={},
@@ -92,7 +91,9 @@ def _merge_summarizes(raw_data_dicts):
api_endpoints[api_endpoint]["total_time"][label] = _empty_statistics()
api_endpoints[api_endpoint]["sql_time"][label] = _empty_statistics()
for internal_endpoint in all_internal_metrics:
_ensure_has_dict_at_key(internals, internal_endpoint,
_ensure_has_dict_at_key(
internals,
internal_endpoint,
total_time={},
)
internals[internal_endpoint]["total_time"][label] = _empty_statistics()
@@ -102,10 +103,10 @@ def _merge_summarizes(raw_data_dicts):
tests[test][label] = {"outcome": "absent"}
for raw_data_dict in raw_data_dicts:
ab_label = raw_data_dict['label']
these_api_endpoints = raw_data_dict['raw_data']['api_endpoint_metrics']
these_internals = raw_data_dict['raw_data']['internals_metrics']
these_tests = raw_data_dict['raw_data']['tests']
ab_label = raw_data_dict["label"]
these_api_endpoints = raw_data_dict["raw_data"]["api_endpoint_metrics"]
these_internals = raw_data_dict["raw_data"]["internals_metrics"]
these_tests = raw_data_dict["raw_data"]["tests"]
for api_endpoint, endpoint_metrics in these_api_endpoints.items():
api_endpoints[api_endpoint]["label"] = endpoint_metrics["label"]
@@ -114,16 +115,18 @@ def _merge_summarizes(raw_data_dicts):
for endpoint, endpoint_metrics in these_internals.items():
internals[endpoint]["label"] = endpoint_metrics["label"]
internals[endpoint]['total_time'][ab_label].update(endpoint_metrics["total_time"])
internals[endpoint]["total_time"][ab_label].update(endpoint_metrics["total_time"])
for test in these_tests:
tests[test["nodeid"]][ab_label]["outcome"] = test.get('outcome')
tests[test["nodeid"]][ab_label]["outcome"] = test.get("outcome")
return {"raw_data": {
"api_endpoint_metrics": api_endpoints,
"internals_metrics": internals,
"tests": tests,
}}
return {
"raw_data": {
"api_endpoint_metrics": api_endpoints,
"internals_metrics": internals,
"tests": tests,
}
}
def _prepare_raw_data(path):
@@ -138,11 +141,11 @@ def _prepare_raw_data(path):
def __inject_api_timing_summary_environment(environment):
for test in environment['raw_data']['tests']:
if 'metadata' not in test:
for test in environment["raw_data"]["tests"]:
if "metadata" not in test:
continue
if 'local_metrics' in test['metadata']:
if "local_metrics" in test["metadata"]:
__inject_api_timing_summary_test(test)
@@ -150,58 +153,62 @@ def __inject_api_timing_summary_across_tests(environment):
api_endpoints = {}
internals = {}
for test in environment['raw_data']['tests']:
if 'metadata' not in test:
for test in environment["raw_data"]["tests"]:
if "metadata" not in test:
continue
test_endpoints = test['api_endpoint_metrics']
test_endpoints = test["api_endpoint_metrics"]
for api_endpoint, endpoint_metrics in test_endpoints.items():
_ensure_has_dict_at_key(api_endpoints, api_endpoint,
total_time={'raw': []},
sql_time={'raw': []},
sql_queries={'raw': []},
_ensure_has_dict_at_key(
api_endpoints,
api_endpoint,
total_time={"raw": []},
sql_time={"raw": []},
sql_queries={"raw": []},
)
api_endpoints[api_endpoint]['label'] = endpoint_metrics['label']
api_endpoints[api_endpoint]['total_time']['raw'].extend(endpoint_metrics['total_time']['raw'])
api_endpoints[api_endpoint]['sql_time']['raw'].extend(endpoint_metrics['sql_time']['raw'])
api_endpoints[api_endpoint]['sql_queries']['raw'].extend(endpoint_metrics['sql_queries']['raw'])
api_endpoints[api_endpoint]["label"] = endpoint_metrics["label"]
api_endpoints[api_endpoint]["total_time"]["raw"].extend(endpoint_metrics["total_time"]["raw"])
api_endpoints[api_endpoint]["sql_time"]["raw"].extend(endpoint_metrics["sql_time"]["raw"])
api_endpoints[api_endpoint]["sql_queries"]["raw"].extend(endpoint_metrics["sql_queries"]["raw"])
test_endpoints = test['internals_metrics']
test_endpoints = test["internals_metrics"]
for api_endpoint, endpoint_metrics in test_endpoints.items():
_ensure_has_dict_at_key(internals, api_endpoint,
total_time={'raw': []},
_ensure_has_dict_at_key(
internals,
api_endpoint,
total_time={"raw": []},
)
internals[api_endpoint]['label'] = endpoint_metrics['label']
internals[api_endpoint]['total_time']['raw'].extend(endpoint_metrics['total_time']['raw'])
internals[api_endpoint]["label"] = endpoint_metrics["label"]
internals[api_endpoint]["total_time"]["raw"].extend(endpoint_metrics["total_time"]["raw"])
for endpoint_metrics in api_endpoints.values():
__inject_statistics(endpoint_metrics['total_time'])
__inject_statistics(endpoint_metrics['sql_time'])
__inject_statistics(endpoint_metrics['sql_queries'])
__inject_statistics(endpoint_metrics["total_time"])
__inject_statistics(endpoint_metrics["sql_time"])
__inject_statistics(endpoint_metrics["sql_queries"])
for endpoint_metrics in internals.values():
__inject_statistics(endpoint_metrics['total_time'])
__inject_statistics(endpoint_metrics["total_time"])
environment['raw_data']['api_endpoint_metrics'] = api_endpoints
environment['raw_data']['internals_metrics'] = internals
environment["raw_data"]["api_endpoint_metrics"] = api_endpoints
environment["raw_data"]["internals_metrics"] = internals
def __inject_raw_timings(environment):
all_timings = []
for test in environment['raw_data']['tests']:
if 'metadata' not in test or 'local_metrics' not in test['metadata']:
for test in environment["raw_data"]["tests"]:
if "metadata" not in test or "local_metrics" not in test["metadata"]:
continue
metrics = test['metadata']['local_metrics']
timing = metrics['timing']
metrics = test["metadata"]["local_metrics"]
timing = metrics["timing"]
for endpoint, timings in timing.items():
recording = timings[0].copy()
recording['endpoint'] = endpoint
recording["endpoint"] = endpoint
all_timings.append(recording)
environment['raw_data']['all_timings'] = all_timings
environment["raw_data"]["all_timings"] = all_timings
def _ensure_has_dict_at_key(the_dict, key, **kwd):
@@ -210,64 +217,62 @@ def _ensure_has_dict_at_key(the_dict, key, **kwd):
def __inject_api_timing_summary_test(test):
metrics = test['metadata']['local_metrics']
timing = metrics['timing']
counter = metrics['counter']
metrics = test["metadata"]["local_metrics"]
timing = metrics["timing"]
counter = metrics["counter"]
api_endpoints = {}
internal_timings = {}
def summarize_times(timings):
times = list(map(lambda t: t['time'], timings))
return __inject_statistics({
'raw': times,
})
times = list(map(lambda t: t["time"], timings))
return __inject_statistics(
{
"raw": times,
}
)
def summarize_counter(c):
counters = list(map(lambda t: t['n'], c))
return __inject_statistics({
'raw': counters,
})
counters = list(map(lambda t: t["n"], c))
return __inject_statistics(
{
"raw": counters,
}
)
for endpoint, timings in timing.items():
if not endpoint.startswith("api"):
continue
endpoint_summary = {
'total_time': summarize_times(timings),
'label': endpoint[len("api."):]
}
endpoint_summary = {"total_time": summarize_times(timings), "label": endpoint[len("api.") :]}
sql_times = "sql.%s" % endpoint
if sql_times in timing:
endpoint_summary['sql_time'] = summarize_times(timing[sql_times])
endpoint_summary["sql_time"] = summarize_times(timing[sql_times])
sql_queries = "sqlqueries.%s" % endpoint
if sql_queries in counter:
endpoint_summary['sql_queries'] = summarize_counter(counter[sql_queries])
endpoint_summary["sql_queries"] = summarize_counter(counter[sql_queries])
api_endpoints[endpoint] = endpoint_summary
for endpoint, timings in timing.items():
if not endpoint.startswith("internals"):
continue
internal_summary = {
'total_time': summarize_times(timings),
'label': endpoint[len("internals."):]
}
internal_summary = {"total_time": summarize_times(timings), "label": endpoint[len("internals.") :]}
internal_timings[endpoint] = internal_summary
test['api_endpoint_metrics'] = api_endpoints
test['internals_metrics'] = internal_timings
test["api_endpoint_metrics"] = api_endpoints
test["internals_metrics"] = internal_timings
def __inject_statistics(from_dict):
raw_values = from_dict['raw']
from_dict['sum'] = sum(raw_values)
from_dict['median'] = f"{statistics.median(raw_values):.2f}"
raw_values = from_dict["raw"]
from_dict["sum"] = sum(raw_values)
from_dict["median"] = f"{statistics.median(raw_values):.2f}"
if len(raw_values) > 1:
from_dict['stdev'] = f"{statistics.stdev(raw_values):.4f}"
from_dict["stdev"] = f"{statistics.stdev(raw_values):.4f}"
else:
from_dict['stdev'] = "n/a"
from_dict['mean'] = f"{statistics.mean(raw_values):.2f}"
from_dict['count'] = len(raw_values)
from_dict["stdev"] = "n/a"
from_dict["mean"] = f"{statistics.mean(raw_values):.2f}"
from_dict["count"] = len(raw_values)
return from_dict
@@ -275,29 +280,26 @@ def __inject_summary(environment):
total = 0
failures = 0
skips = 0
for test in environment['raw_data']['tests']:
for test in environment["raw_data"]["tests"]:
total += 1
status = test.get('outcome')
if status == 'failed':
status = test.get("outcome")
if status == "failed":
failures += 1
elif status == 'skipped':
elif status == "skipped":
skips += 1
environment['raw_data']['results'] = {
'total': total,
'failures': failures,
'skips': skips,
environment["raw_data"]["results"] = {
"total": total,
"failures": failures,
"skips": skips,
}
def _parser():
parser = argparse.ArgumentParser(description=DESCRIPTION)
parser.add_argument('input_path', metavar='INPUT', type=str, nargs="+",
help='structured input path (.json)')
parser.add_argument('--output_path', type=str, default="test.html",
help='output path (.html)')
parser.add_argument('--title', type=str, default="Test Performance Summary",
help='Performance Test Results')
parser.add_argument('--include_raw_metrics', action="store_true", default=False)
parser.add_argument("input_path", metavar="INPUT", type=str, nargs="+", help="structured input path (.json)")
parser.add_argument("--output_path", type=str, default="test.html", help="output path (.html)")
parser.add_argument("--title", type=str, default="Test Performance Summary", help="Performance Test Results")
parser.add_argument("--include_raw_metrics", action="store_true", default=False)
return parser
scripts/tool_shed/api/common.py
+72 -71
View File
@@ -7,35 +7,36 @@ from urllib.request import (
HTTPRedirectHandler,
install_opener,
Request,
urlopen
urlopen,
)
sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 'lib'))
sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, "lib"))
from galaxy import util
from tool_shed.util import hg_util
class HTTPRedirectWithDataHandler(HTTPRedirectHandler):
def __init__(self, method):
'''
"""
Upon first inspection, it would seem that this shouldn't be necessary, but for some reason
not having a constructor explicitly set the request method breaks PUT requests.
'''
self.valid_methods = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE']
self.redirect_codes = ['301', '302', '303', '307']
"""
self.valid_methods = ["GET", "HEAD", "POST", "PUT", "DELETE"]
self.redirect_codes = ["301", "302", "303", "307"]
self.method = method
def redirect_request(self, request, fp, code, msg, headers, new_url):
request_method = request.get_method()
if str(code) in self.redirect_codes and request_method in self.valid_methods:
new_url = new_url.replace(' ', '%20')
request = Request(new_url,
data=request.data,
headers=request.headers,
origin_req_host=request.get_origin_req_host(),
unverifiable=True)
new_url = new_url.replace(" ", "%20")
request = Request(
new_url,
data=request.data,
headers=request.headers,
origin_req_host=request.get_origin_req_host(),
unverifiable=True,
)
if self.method in self.valid_methods:
if request.get_method() != self.method:
request.get_method = lambda: self.method
@@ -50,7 +51,7 @@ def build_request_with_data(url, data, api_key, method):
opener = build_opener(http_redirect_with_data_handler)
install_opener(opener)
url = make_url(url, api_key=api_key, args=None)
request = Request(url, headers={'Content-Type': 'application/json'}, data=json.dumps(data))
request = Request(url, headers={"Content-Type": "application/json"}, data=json.dumps(data))
request_method = request.get_method()
if request_method != method:
request.get_method = lambda: method
@@ -63,7 +64,7 @@ def delete(api_key, url, data, return_formatted=True):
'data' will become the JSON payload read by the Tool Shed.
"""
try:
opener, request = build_request_with_data(url, data, api_key, 'DELETE')
opener, request = build_request_with_data(url, data, api_key, "DELETE")
delete_request = opener.open(request)
response = json.loads(delete_request.read())
except HTTPError as e:
@@ -72,10 +73,10 @@ def delete(api_key, url, data, return_formatted=True):
print(e.read(1024))
sys.exit(1)
else:
return dict(status='error', message=str(e.read(1024)))
return dict(status="error", message=str(e.read(1024)))
if return_formatted:
print('Response')
print('--------')
print("Response")
print("--------")
print(response)
else:
return response
@@ -91,33 +92,33 @@ def display(url, api_key=None, return_formatted=True):
print(e.read(1024))
sys.exit(1)
if isinstance(r, str):
print('error: %s' % r)
print("error: %s" % r)
return None
if not return_formatted:
return r
elif isinstance(r, list):
# Response is a collection as defined in the REST style.
print('Collection Members')
print('------------------')
print("Collection Members")
print("------------------")
for n, i in enumerate(r):
# All collection members should have a name in the response.
# url is optional
if 'url' in i:
print('#%d: %s' % (n + 1, i.pop('url')))
if 'name' in i:
print(' name: %s' % i.pop('name'))
if "url" in i:
print("#%d: %s" % (n + 1, i.pop("url")))
if "name" in i:
print(" name: %s" % i.pop("name"))
for k, v in i.items():
print(f' {k}: {v}')
print(f" {k}: {v}")
print()
print('%d element(s) in collection' % len(r))
print("%d element(s) in collection" % len(r))
elif isinstance(r, dict):
# Response is an element as defined in the REST style.
print('Member Information')
print('------------------')
print("Member Information")
print("------------------")
for k, v in r.items():
print(f'{k}: {v}')
print(f"{k}: {v}")
else:
print('response is unknown type: %s' % type(r))
print("response is unknown type: %s" % type(r))
def get(url, api_key=None):
@@ -131,11 +132,11 @@ def get(url, api_key=None):
def get_api_url(base, parts, params=None):
"""Compose and return a URL for the Tool Shed API."""
if 'api' in parts and parts.index('api') != 0:
parts.pop(parts.index('api'))
parts.insert(0, 'api')
elif 'api' not in parts:
parts.insert(0, 'api')
if "api" in parts and parts.index("api") != 0:
parts.pop(parts.index("api"))
parts.insert(0, "api")
elif "api" not in parts:
parts.insert(0, "api")
url = util.build_url(base, pathspec=parts, params=params)
return url
@@ -145,8 +146,8 @@ def get_latest_downloadable_changeset_revision_via_api(url, name, owner):
Return the latest downloadable changeset revision for the repository defined by the received
name and owner.
"""
error_message = ''
parts = ['api', 'repositories', 'get_ordered_installable_revisions']
error_message = ""
parts = ["api", "repositories", "get_ordered_installable_revisions"]
params = dict(name=name, owner=owner)
api_url = get_api_url(base=url, parts=parts, params=params)
changeset_revisions, error_message = json_from_url(api_url)
@@ -162,44 +163,44 @@ def get_repository_dict(url, repository_dict):
Send a request to the Tool Shed to get additional information about the repository defined
by the received repository_dict. Add the information to the repository_dict and return it.
"""
error_message = ''
error_message = ""
if not isinstance(repository_dict, dict):
error_message = 'Invalid repository_dict received: %s' % str(repository_dict)
error_message = "Invalid repository_dict received: %s" % str(repository_dict)
return None, error_message
repository_id = repository_dict.get('repository_id', None)
repository_id = repository_dict.get("repository_id", None)
if repository_id is None:
error_message = 'Invalid repository_dict does not contain a repository_id entry: %s' % str(repository_dict)
error_message = "Invalid repository_dict does not contain a repository_id entry: %s" % str(repository_dict)
return None, error_message
parts = ['api', 'repositories', repository_id]
parts = ["api", "repositories", repository_id]
api_url = get_api_url(base=url, parts=parts)
extended_dict, error_message = json_from_url(api_url)
if extended_dict is None or error_message:
return None, error_message
name = extended_dict.get('name', None)
owner = extended_dict.get('owner', None)
name = extended_dict.get("name", None)
owner = extended_dict.get("owner", None)
if name is not None and owner is not None:
name = str(name)
owner = str(owner)
latest_changeset_revision, error_message = get_latest_downloadable_changeset_revision_via_api(url, name, owner)
if latest_changeset_revision is None or error_message:
return None, error_message
extended_dict['latest_revision'] = str(latest_changeset_revision)
extended_dict["latest_revision"] = str(latest_changeset_revision)
return extended_dict, error_message
else:
error_message = 'Invalid extended_dict does not contain name or owner entries: %s' % str(extended_dict)
error_message = "Invalid extended_dict does not contain name or owner entries: %s" % str(extended_dict)
return None, error_message
def json_from_url(url):
"""Send a request to the Tool Shed via the Tool Shed API and handle the response."""
error_message = ''
error_message = ""
url_handle = urlopen(url)
url_contents = url_handle.read()
try:
parsed_json = json.loads(url_contents)
except Exception as e:
error_message = str(url_contents)
print('Error parsing JSON data in json_from_url():', e)
print("Error parsing JSON data in json_from_url():", e)
return None, error_message
return parsed_json, error_message
@@ -208,33 +209,33 @@ def make_url(url, api_key=None, args=None):
"""Adds the API Key to the URL if it's not already there."""
if args is None:
args = []
argsep = '&'
if '?' not in url:
argsep = '?'
argsep = "&"
if "?" not in url:
argsep = "?"
if api_key:
if '?key=' not in url and '&key=' not in url:
args.insert(0, ('key', api_key))
return url + argsep + '&'.join('='.join(t) for t in args)
if "?key=" not in url and "&key=" not in url:
args.insert(0, ("key", api_key))
return url + argsep + "&".join("=".join(t) for t in args)
def post(url, data, api_key=None):
"""Do the POST."""
try:
opener, request = build_request_with_data(url, data, api_key, 'POST')
opener, request = build_request_with_data(url, data, api_key, "POST")
post_request = opener.open(request)
return json.loads(post_request.read())
except HTTPError as e:
return dict(status='error', message=str(e.read(1024)))
return dict(status="error", message=str(e.read(1024)))
def put(url, data, api_key=None):
"""Do the PUT."""
try:
opener, request = build_request_with_data(url, data, api_key, 'PUT')
opener, request = build_request_with_data(url, data, api_key, "PUT")
put_request = opener.open(request)
return json.loads(put_request.read())
except HTTPError as e:
return dict(status='error', message=str(e.read(1024)))
return dict(status="error", message=str(e.read(1024)))
def submit(url, data, api_key=None, return_formatted=True):
@@ -250,24 +251,24 @@ def submit(url, data, api_key=None, return_formatted=True):
print(e.read(1024))
sys.exit(1)
else:
return dict(status='error', message=str(e.read(1024)))
return dict(status="error", message=str(e.read(1024)))
if not return_formatted:
return response
print('Response')
print('--------')
print("Response")
print("--------")
if isinstance(response, list):
# Currently the only implemented responses are lists of dicts, because submission creates
# some number of collection elements.
for i in response:
if isinstance(i, dict):
if 'url' in i:
print(i.pop('url'))
if "url" in i:
print(i.pop("url"))
else:
print('----')
if 'name' in i:
print(' name: %s' % i.pop('name'))
print("----")
if "name" in i:
print(" name: %s" % i.pop("name"))
for k, v in i.items():
print(f' {k}: {v}')
print(f" {k}: {v}")
else:
print(i)
else:
@@ -287,10 +288,10 @@ def update(api_key, url, data, return_formatted=True):
print(e.read(1024))
sys.exit(1)
else:
return dict(status='error', message=str(e.read(1024)))
return dict(status="error", message=str(e.read(1024)))
if return_formatted:
print('Response')
print('--------')
print("Response")
print("--------")
print(response)
else:
return response
scripts/tool_shed/api/create_categories.py
+32 -14
View File
@@ -20,24 +20,26 @@ available in the test public Tool Shed and create each of them in a local develo
import argparse
from common import get, submit
from common import (
get,
submit,
)
def main(options):
api_key = options.api
from_tool_shed = options.from_tool_shed.rstrip('/')
to_tool_shed = options.to_tool_shed.rstrip('/')
from_tool_shed = options.from_tool_shed.rstrip("/")
to_tool_shed = options.to_tool_shed.rstrip("/")
# Get the categories from the specified Tool Shed.
url = '%s/api/categories' % from_tool_shed
url = "%s/api/categories" % from_tool_shed
category_dicts = get(url)
create_response_dicts = []
for category_dict in category_dicts:
name = category_dict.get('name', None)
description = category_dict.get('description', None)
name = category_dict.get("name", None)
description = category_dict.get("description", None)
if name is not None and description is not None:
data = dict(name=name,
description=description)
url = '%s/api/categories' % to_tool_shed
data = dict(name=name, description=description)
url = "%s/api/categories" % to_tool_shed
try:
response = submit(url, data, api_key)
except Exception as e:
@@ -47,10 +49,26 @@ def main(options):
create_response_dicts.append(create_response_dict)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Retrieve a list of categories from a Tool Shed and create them in another Tool Shed.')
parser.add_argument("-a", "--api", dest="api", required=True, help="API Key for Tool Shed in which categories will be created")
parser.add_argument("-f", "--from_tool_shed", dest="from_tool_shed", required=True, help="URL of Tool Shed from which to retrieve the categories")
parser.add_argument("-t", "--to_tool_shed", dest="to_tool_shed", required=True, help="URL of Tool Shed in which to create the categories")
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Retrieve a list of categories from a Tool Shed and create them in another Tool Shed."
)
parser.add_argument(
"-a", "--api", dest="api", required=True, help="API Key for Tool Shed in which categories will be created"
)
parser.add_argument(
"-f",
"--from_tool_shed",
dest="from_tool_shed",
required=True,
help="URL of Tool Shed from which to retrieve the categories",
)
parser.add_argument(
"-t",
"--to_tool_shed",
dest="to_tool_shed",
required=True,
help="URL of Tool Shed in which to create the categories",
)
options = parser.parse_args()
main(options)
scripts/tool_shed/api/create_users.py
+29 -16
View File
@@ -21,26 +21,27 @@ are available in the test public Tool Shed and create each of them in a local de
import argparse
from common import get, submit
from common import (
get,
submit,
)
def main(options):
api_key = options.api
from_tool_shed = options.from_tool_shed.rstrip('/')
to_tool_shed = options.to_tool_shed.rstrip('/')
from_tool_shed = options.from_tool_shed.rstrip("/")
to_tool_shed = options.to_tool_shed.rstrip("/")
# Get the users from the specified Tool Shed.
url = '%s/api/users' % from_tool_shed
url = "%s/api/users" % from_tool_shed
user_dicts = get(url)
create_response_dicts = []
for user_dict in user_dicts:
username = user_dict.get('username', None)
username = user_dict.get("username", None)
if username is not None:
email = '%s@test.org' % username
password = 'testuser'
data = dict(email=email,
password=password,
username=username)
url = '%s/api/users' % to_tool_shed
email = "%s@test.org" % username
password = "testuser"
data = dict(email=email, password=password, username=username)
url = "%s/api/users" % to_tool_shed
try:
response = submit(url, data, api_key)
except Exception as e:
@@ -50,10 +51,22 @@ def main(options):
create_response_dicts.append(create_response_dict)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Retrieve a list of users from a Tool Shed and create them in another Tool Shed.')
parser.add_argument("-a", "--api", dest="api", required=True, help="API Key for Tool Shed in which users will be created")
parser.add_argument("-f", "--from_tool_shed", dest="from_tool_shed", required=True, help="URL of Tool Shed from which to retrieve the users")
parser.add_argument("-t", "--to_tool_shed", dest="to_tool_shed", required=True, help="URL of Tool Shed in which to create the users")
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Retrieve a list of users from a Tool Shed and create them in another Tool Shed."
)
parser.add_argument(
"-a", "--api", dest="api", required=True, help="API Key for Tool Shed in which users will be created"
)
parser.add_argument(
"-f",
"--from_tool_shed",
dest="from_tool_shed",
required=True,
help="URL of Tool Shed from which to retrieve the users",
)
parser.add_argument(
"-t", "--to_tool_shed", dest="to_tool_shed", required=True, help="URL of Tool Shed in which to create the users"
)
options = parser.parse_args()
main(options)
scripts/tool_shed/bootstrap_tool_shed/bootstrap_util.py
+46 -35
View File
@@ -5,9 +5,12 @@ import os
import sys
from configparser import ConfigParser
from sqlalchemy.exc import OperationalError, ProgrammingError
from sqlalchemy.exc import (
OperationalError,
ProgrammingError,
)
sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 'lib'))
sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, "lib"))
import tool_shed.webapp.model.mapping as tool_shed_model
from tool_shed.util import xml_util
@@ -16,21 +19,27 @@ from tool_shed.util import xml_util
def check_db(config_parser):
dburi = None
if config_parser.has_option('app:main', 'database_connection'):
dburi = config_parser.get('app:main', 'database_connection')
elif config_parser.has_option('app:main', 'database_file'):
db_file = config_parser.get('app:main', 'database_file')
if config_parser.has_option("app:main", "database_connection"):
dburi = config_parser.get("app:main", "database_connection")
elif config_parser.has_option("app:main", "database_file"):
db_file = config_parser.get("app:main", "database_file")
dburi = "sqlite:///%s?isolation_level=IMMEDIATE" % db_file
else:
sys.exit('The database configuration setting is missing from the tool_shed.ini file. Add this setting before attempting to bootstrap.')
sys.exit(
"The database configuration setting is missing from the tool_shed.ini file. Add this setting before attempting to bootstrap."
)
sa_session = None
database_exists_message = 'The database configured for this Tool Shed is not new, so bootstrapping is not allowed. '
database_exists_message += 'Create a new database that has not been migrated before attempting to bootstrap.'
database_exists_message = (
"The database configured for this Tool Shed is not new, so bootstrapping is not allowed. "
)
database_exists_message += "Create a new database that has not been migrated before attempting to bootstrap."
try:
model = tool_shed_model.init(config_parser.get('app:main', 'file_path'), dburi, engine_options={}, create_tables=False)
model = tool_shed_model.init(
config_parser.get("app:main", "file_path"), dburi, engine_options={}, create_tables=False
)
sa_session = model.context.current
sys.exit(database_exists_message)
except ProgrammingError:
@@ -40,7 +49,7 @@ def check_db(config_parser):
try:
if sa_session is not None:
result = sa_session.execute('SELECT version FROM migrate_version').first()
result = sa_session.execute("SELECT version FROM migrate_version").first()
if result[0] >= 2:
sys.exit(database_exists_message)
else:
@@ -48,14 +57,14 @@ def check_db(config_parser):
except ProgrammingError:
pass
if config_parser.has_option('app:main', 'hgweb_config_dir'):
if config_parser.has_option("app:main", "hgweb_config_dir"):
hgweb_config_parser = ConfigParser()
hgweb_dir = config_parser.get('app:main', 'hgweb_config_dir')
hgweb_config_file = os.path.join(hgweb_dir, 'hgweb.config')
hgweb_dir = config_parser.get("app:main", "hgweb_config_dir")
hgweb_config_file = os.path.join(hgweb_dir, "hgweb.config")
if not os.path.exists(hgweb_config_file):
sys.exit(0)
hgweb_config_parser.read(hgweb_config_file)
configured_repos = hgweb_config_parser.items('paths')
configured_repos = hgweb_config_parser.items("paths")
if len(configured_repos) >= 1:
message = "This Tool Shed's hgweb.config file contains entries, so bootstrapping is not allowed. Delete"
message += " the current hgweb.config file along with all associated repositories in the configured "
@@ -70,35 +79,37 @@ def check_db(config_parser):
def admin_user_info():
user_info_config = os.path.abspath(os.path.join(os.getcwd(), 'scripts/tool_shed/bootstrap_tool_shed', 'user_info.xml'))
user_info_config = os.path.abspath(
os.path.join(os.getcwd(), "scripts/tool_shed/bootstrap_tool_shed", "user_info.xml")
)
tree, error_message = xml_util.parse_xml(user_info_config)
username = None
email = None
password = None
if tree is None:
print("The XML file ", user_info_config, " seems to be invalid, using defaults.")
email = 'admin@test.org'
password = 'testuser'
username = 'admin'
email = "admin@test.org"
password = "testuser"
username = "admin"
else:
root = tree.getroot()
for elem in root:
if elem.tag == 'email':
if elem.tag == "email":
email = elem.text
elif elem.tag == 'password':
elif elem.tag == "password":
password = elem.text
elif elem.tag == 'username':
elif elem.tag == "username":
username = elem.text
return (username, email, password)
def get_local_tool_shed_url(config_parser):
port = '9009'
if config_parser.has_section('server:main'):
if config_parser.has_option('server:main', 'port'):
port = config_parser.get('server:main', 'port')
host = '127.0.0.1'
print(f'http://{host}:{port}')
port = "9009"
if config_parser.has_section("server:main"):
if config_parser.has_option("server:main", "port"):
port = config_parser.get("server:main", "port")
host = "127.0.0.1"
print(f"http://{host}:{port}")
return 0
@@ -110,22 +121,22 @@ def main(args):
else:
return 1
if args.method == 'check_db':
if args.method == "check_db":
return check_db(config_parser)
elif args.method == 'admin_user_info':
elif args.method == "admin_user_info":
(username, email, password) = admin_user_info()
print(f'{username}__SEP__{email}__SEP__{password}')
print(f"{username}__SEP__{email}__SEP__{password}")
return 0
elif args.method == 'get_url':
elif args.method == "get_url":
return get_local_tool_shed_url(config_parser)
else:
return 1
parser = optparse.OptionParser()
parser.add_option('-c', '--config_file', dest='config', action='store', default='config/tool_shed.yml.sample')
parser.add_option('-e', '--execute', dest='method', action='store', default='check_db')
parser.add_option("-c", "--config_file", dest="config", action="store", default="config/tool_shed.yml.sample")
parser.add_option("-e", "--execute", dest="method", action="store", default="check_db")
(args, options) = parser.parse_args()
if __name__ == '__main__':
if __name__ == "__main__":
sys.exit(main(args))
scripts/tool_shed/bootstrap_tool_shed/create_user_with_api_key.py
+20 -16
View File
@@ -6,18 +6,19 @@ import os
import sys
from configparser import ConfigParser
sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 'lib'))
sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, "lib"))
sys.path.insert(1, os.path.join(os.path.dirname(__file__)))
from bootstrap_util import admin_user_info # noqa: I100,I201
import tool_shed.webapp.config as tool_shed_config
from galaxy.security.idencoding import IdEncodingHelper
from galaxy.security.validate_user_input import (
validate_email_str,
validate_password_str,
validate_publicname_str
validate_publicname_str,
)
from tool_shed.webapp.model import mapping
from bootstrap_util import admin_user_info # noqa: I100,I201
log = logging.getLogger(__name__)
@@ -32,16 +33,15 @@ class BootstrapApplication:
self.config = config
if not self.config.database_connection:
self.config.database_connection = "sqlite:///%s?isolation_level=IMMEDIATE" % str(config.database)
print('Using database connection: ', self.config.database_connection)
print("Using database connection: ", self.config.database_connection)
# Setup the database engine and ORM
self.model = mapping.init(self.config.file_path,
self.config.database_connection,
engine_options={},
create_tables=False)
self.model = mapping.init(
self.config.file_path, self.config.database_connection, engine_options={}, create_tables=False
)
self.security = IdEncodingHelper(id_secret=self.config.id_secret)
self.hgweb_config_manager = self.model.hgweb_config_manager
self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir
print('Using hgweb.config file: ', self.hgweb_config_manager.hgweb_config)
print("Using hgweb.config file: ", self.hgweb_config_manager.hgweb_config)
@property
def sa_session(self):
@@ -82,18 +82,18 @@ def create_user(app):
def validate(email, password, username):
message = "\n".join([validate_email_str(email),
validate_password_str(password),
validate_publicname_str(username)]).rstrip()
message = "\n".join(
[validate_email_str(email), validate_password_str(password), validate_publicname_str(username)]
).rstrip()
return message
if __name__ == "__main__":
parser = optparse.OptionParser(description='Create a user with API key.')
parser.add_option('-c', dest='config', action='store', help='.ini file to retrieve toolshed configuration from')
parser = optparse.OptionParser(description="Create a user with API key.")
parser.add_option("-c", dest="config", action="store", help=".ini file to retrieve toolshed configuration from")
(args, options) = parser.parse_args()
ini_file = args.config
config_parser = ConfigParser({'here': os.getcwd()})
config_parser = ConfigParser({"here": os.getcwd()})
print("Reading ini file: ", ini_file)
config_parser.read(ini_file)
config_dict = {}
@@ -104,7 +104,11 @@ if __name__ == "__main__":
user = create_user(app)
if user is not None:
api_key = create_api_key(app, user)
print("Created new user with public username '", user.username, ". An API key was also created and associated with the user.")
print(
"Created new user with public username '",
user.username,
". An API key was also created and associated with the user.",
)
sys.exit(0)
else:
sys.exit("Problem creating a new user and an associated API key.")
scripts/tool_shed/build_ts_whoosh_index.py
+8 -9
View File
@@ -16,11 +16,11 @@ import logging
import os
import sys
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "lib")))
from galaxy.util.script import (
app_properties_from_args,
populate_config_args
populate_config_args,
)
from tool_shed.util.shed_index import build_index
from tool_shed.webapp import config as ts_config
@@ -30,12 +30,11 @@ log.addHandler(logging.StreamHandler(sys.stdout))
def parse_arguments():
parser = argparse.ArgumentParser(description='Build a disk-backed Toolshed repository index and tool index for searching.')
parser = argparse.ArgumentParser(
description="Build a disk-backed Toolshed repository index and tool index for searching."
)
populate_config_args(parser)
parser.add_argument('-d', '--debug',
action='store_true',
default=False,
help='Print extra info')
parser.add_argument("-d", "--debug", action="store_true", default=False, help="Print extra info")
args = parser.parse_args()
app_properties = app_properties_from_args(args)
config = ts_config.ToolShedAppConfiguration(**app_properties)
@@ -45,9 +44,9 @@ def parse_arguments():
args.file_path = config.file_path
if args.debug:
log.setLevel(logging.DEBUG)
log.debug('Full options:')
log.debug("Full options:")
for i in vars(args).items():
log.debug('%s: %s' % i)
log.debug("%s: %s" % i)
return args
scripts/tool_shed/deprecate_repositories_without_metadata.py
+77 -40
View File
@@ -7,21 +7,27 @@ import string
import sys
import textwrap
import time
from datetime import datetime, timedelta
from datetime import (
datetime,
timedelta,
)
from optparse import OptionParser
from time import strftime
import sqlalchemy as sa
from sqlalchemy import and_, distinct, false, not_
from sqlalchemy import (
and_,
distinct,
false,
not_,
)
sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib'))
sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "lib"))
import tool_shed.webapp.config as tool_shed_config
import tool_shed.webapp.model.mapping
from galaxy.util import (
build_url,
send_mail as galaxy_send_mail
)
from galaxy.util import build_url
from galaxy.util import send_mail as galaxy_send_mail
log = logging.getLogger()
log.setLevel(10)
@@ -30,23 +36,37 @@ assert sys.version_info[:2] >= (2, 6)
def build_citable_url(host, repository):
return build_url(host, pathspec=['view', repository.user.username, repository.name])
return build_url(host, pathspec=["view", repository.user.username, repository.name])
def main():
'''
"""
Script to deprecate any repositories that are older than n days, and have been empty since creation.
'''
"""
parser = OptionParser()
parser.add_option("-d", "--days", dest="days", action="store", type="int", help="number of days (14)", default=14)
parser.add_option("-i", "--info_only", action="store_true", dest="info_only", help="info about the requested action", default=False)
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="verbose mode, print the name of each repository", default=False)
parser.add_option(
"-i",
"--info_only",
action="store_true",
dest="info_only",
help="info about the requested action",
default=False,
)
parser.add_option(
"-v",
"--verbose",
action="store_true",
dest="verbose",
help="verbose mode, print the name of each repository",
default=False,
)
(options, args) = parser.parse_args()
try:
ini_file = args[0]
except IndexError:
sys.exit("Usage: python %s <tool shed .ini file> [options]" % sys.argv[0])
config_parser = configparser.ConfigParser({'here': os.getcwd()})
config_parser = configparser.ConfigParser({"here": os.getcwd()})
config_parser.read(ini_file)
config_dict = {}
for key, value in config_parser.items("app:main"):
@@ -66,32 +86,38 @@ def main():
def send_mail_to_owner(app, owner, email, repositories_deprecated, days=14):
'''
"""
Sends an email to the owner of the provided repository.
'''
smtp_server = app.config.get('smtp_server', None)
from_address = app.config.get('email_from', None)
"""
smtp_server = app.config.get("smtp_server", None)
from_address = app.config.get("email_from", None)
# Since there is no way to programmatically determine the URL for the tool shed from the .ini file, this method requires that
# an environment variable named TOOL_SHED_CANONICAL_URL be set, pointing to the tool shed that is being checked.
url = os.environ.get('TOOL_SHED_CANONICAL_URL', None)
url = os.environ.get("TOOL_SHED_CANONICAL_URL", None)
if None in [smtp_server, from_address]:
print('# Mail not configured, not sending email to repository owner.')
print("# Mail not configured, not sending email to repository owner.")
return
elif url is None:
print('# Environment variable TOOL_SHED_CANONICAL_URL not set, not sending email to repository owner.')
print("# Environment variable TOOL_SHED_CANONICAL_URL not set, not sending email to repository owner.")
return
subject = "Regarding your tool shed repositories at %s" % url
message_body_template = 'The tool shed automated repository checker has discovered that one or more of your repositories hosted ' + \
'at this tool shed url ${url} have remained empty for over ${days} days, so they have been marked as deprecated. If you have plans ' + \
'for these repositories, you can mark them as un-deprecated at any time.'
message_body_template = (
"The tool shed automated repository checker has discovered that one or more of your repositories hosted "
+ "at this tool shed url ${url} have remained empty for over ${days} days, so they have been marked as deprecated. If you have plans "
+ "for these repositories, you can mark them as un-deprecated at any time."
)
message_template = string.Template(message_body_template)
body = '\n'.join(textwrap.wrap(message_template.safe_substitute(days=days, url=url), width=95))
body += '\n\n'
body += 'Repositories that were deprecated:\n'
body += '\n'.join(build_citable_url(url, repository) for repository in repositories_deprecated)
body = "\n".join(textwrap.wrap(message_template.safe_substitute(days=days, url=url), width=95))
body += "\n\n"
body += "Repositories that were deprecated:\n"
body += "\n".join(build_citable_url(url, repository) for repository in repositories_deprecated)
try:
galaxy_send_mail(from_address, email, subject, body, app.config)
print("# An email has been sent to {}, the owner of {}.".format(owner, ', '.join(repository.name for repository in repositories_deprecated)))
print(
"# An email has been sent to {}, the owner of {}.".format(
owner, ", ".join(repository.name for repository in repositories_deprecated)
)
)
return True
except Exception as e:
print("# An error occurred attempting to send email: %s" % e)
@@ -106,8 +132,11 @@ def deprecate_repositories(app, cutoff_time, days=14, info_only=False, verbose=F
repository_ids_to_not_check = []
# Get a unique list of repository ids from the repository_metadata table. Any repository ID found in this table is not
# empty, and will not be checked.
metadata_records = app.sa_session.execute(sa.select([distinct(app.model.RepositoryMetadata.table.c.repository_id)],
from_obj=app.model.RepositoryMetadata.table))
metadata_records = app.sa_session.execute(
sa.select(
[distinct(app.model.RepositoryMetadata.table.c.repository_id)], from_obj=app.model.RepositoryMetadata.table
)
)
for metadata_record in metadata_records:
repository_ids_to_not_check.append(metadata_record.repository_id)
# Get the repositories that are A) not present in the above list, and b) older than the specified time.
@@ -129,39 +158,47 @@ def deprecate_repositories(app, cutoff_time, days=14, info_only=False, verbose=F
repository_ids = [row.id for row in query_result]
# Iterate through the list of repository ids for empty repositories and deprecate them unless info_only is set.
for repository_id in repository_ids:
repository = app.sa_session.query(app.model.Repository) \
.filter(app.model.Repository.table.c.id == repository_id).one()
repository = (
app.sa_session.query(app.model.Repository).filter(app.model.Repository.table.c.id == repository_id).one()
)
owner = repository.user
if info_only:
print(f'# Repository {repository.name} owned by {repository.user.username} would have been deprecated, but info_only was set.')
print(
f"# Repository {repository.name} owned by {repository.user.username} would have been deprecated, but info_only was set."
)
else:
if verbose:
print(f'# Deprecating repository {repository.name} owned by {owner.username}.')
print(f"# Deprecating repository {repository.name} owned by {owner.username}.")
if owner.username not in repositories_by_owner:
repositories_by_owner[owner.username] = dict(owner=owner, repositories=[])
repositories_by_owner[owner.username]['repositories'].append(repository)
repositories_by_owner[owner.username]["repositories"].append(repository)
repositories.append(repository)
# Send an email to each repository owner, listing the repositories that were deprecated.
for repository_owner in repositories_by_owner:
for repository in repositories_by_owner[repository_owner]['repositories']:
for repository in repositories_by_owner[repository_owner]["repositories"]:
repository.deprecated = True
app.sa_session.add(repository)
app.sa_session.flush()
owner = repositories_by_owner[repository_owner]['owner']
send_mail_to_owner(app, owner.username, owner.email, repositories_by_owner[repository_owner]['repositories'], days)
owner = repositories_by_owner[repository_owner]["owner"]
send_mail_to_owner(
app, owner.username, owner.email, repositories_by_owner[repository_owner]["repositories"], days
)
stop = time.time()
print('# Deprecated %d repositories.' % len(repositories))
print("# Deprecated %d repositories." % len(repositories))
print("# Elapsed time: ", stop - start)
print("####################################################################################")
class DeprecateRepositoriesApplication:
"""Encapsulates the state of a Universe application"""
def __init__(self, config):
if config.database_connection is False:
config.database_connection = "sqlite:///%s?isolation_level=IMMEDIATE" % config.database
# Setup the database engine and ORM
self.model = tool_shed.webapp.model.mapping.init(config.file_path, config.database_connection, engine_options={}, create_tables=False)
self.model = tool_shed.webapp.model.mapping.init(
config.file_path, config.database_connection, engine_options={}, create_tables=False
)
self.config = config
@property
scripts/tools/maf/check_loc_file.py
+5 -5
View File
@@ -15,12 +15,12 @@ def __main__():
index_location_file = sys.argv[1]
for i, line in enumerate(open(index_location_file)):
try:
if line.startswith('#'):
if line.startswith("#"):
continue
display_name, uid, indexed_for_species, species_exist, maf_files = line.rstrip().split('\t')
indexed_for_species = indexed_for_species.split(',')
species_exist = species_exist.split(',')
maf_files = maf_files.split(',')
display_name, uid, indexed_for_species, species_exist, maf_files = line.rstrip().split("\t")
indexed_for_species = indexed_for_species.split(",")
species_exist = species_exist.split(",")
maf_files = maf_files.split(",")
species_indexed_in_maf = []
species_found_in_maf = []
for maf_file in maf_files:
scripts/tools/re_escape_output.py
+9 -7
View File
@@ -11,21 +11,23 @@ import re
def __main__():
parser = optparse.OptionParser()
parser.add_option("-m", "--multiline", action="store_true", dest="multiline", default=False, help="Use Multiline Matching")
parser.add_option(
"-m", "--multiline", action="store_true", dest="multiline", default=False, help="Use Multiline Matching"
)
(options, args) = parser.parse_args()
input = open(args[0], 'rb')
input = open(args[0], "rb")
if len(args) > 1:
output = open(args[1], 'wb')
output = open(args[1], "wb")
else:
if options.multiline:
suffix = 're_match_multiline'
suffix = "re_match_multiline"
else:
suffix = 're_match'
output = open(f"{args[0]}.{suffix}", 'wb')
suffix = "re_match"
output = open(f"{args[0]}.{suffix}", "wb")
if options.multiline:
lines = [re.escape(input.read())]
else:
lines = ["%s\n" % re.escape(line.rstrip('\n\r')) for line in input]
lines = ["%s\n" % re.escape(line.rstrip("\n\r")) for line in input]
output.writelines(lines)
output.close()
scripts/update_shed_config_path.py
+41 -18
View File
@@ -3,22 +3,28 @@ import os
import sys
from configparser import ConfigParser
from sqlalchemy import create_engine, MetaData
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy import (
create_engine,
MetaData,
)
from sqlalchemy.orm import (
scoped_session,
sessionmaker,
)
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib")))
import galaxy.model.tool_shed_install.mapping as mapping
def main(opts, session, model):
'''
"""
Find all tool shed repositories with the bad path and update with the correct path.
'''
"""
for row in session.query(model.ToolShedRepository).all():
if 'shed_config_filename' in row.metadata_:
if row.metadata_['shed_config_filename'] == opts.bad_filename:
row.metadata_['shed_config_filename'] = opts.good_filename
if "shed_config_filename" in row.metadata_:
if row.metadata_["shed_config_filename"] == opts.bad_filename:
row.metadata_["shed_config_filename"] = opts.good_filename
session.add(row)
session.flush()
return 0
@@ -28,13 +34,13 @@ def create_database(config_file):
parser = ConfigParser()
parser.read(config_file)
# Determine which database connection to use.
database_connection = parser.get('app:main', 'install_database_connection')
database_connection = parser.get("app:main", "install_database_connection")
if database_connection is None:
database_connection = parser.get('app:main', 'database_connection')
database_connection = parser.get("app:main", "database_connection")
if database_connection is None:
database_connection = 'sqlite:///%s' % parser.get('app:main', 'database_file')
database_connection = "sqlite:///%s" % parser.get("app:main", "database_file")
if database_connection is None:
print('Unable to determine correct database connection.')
print("Unable to determine correct database connection.")
exit(1)
# Initialize the database connection.
@@ -45,15 +51,32 @@ def create_database(config_file):
return install_session, model
if __name__ == '__main__':
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--config_file', dest='config_file', required=True, help="The path to your Galaxy configuration .ini file.")
parser.add_argument('--from', dest='bad_filename', required=True, help="The old, invalid path to the shed_tool_conf.xml or migrated_tools_conf.xml file.")
parser.add_argument('--to', dest='good_filename', required=True, help="The updated path to the shed_tool_conf.xml or migrated_tools_conf.xml file.")
parser.add_argument('--force', dest='force', action='store_true', help="Use this flag to set the new path even if the file does not (yet) exist there.")
parser.add_argument(
"--config_file", dest="config_file", required=True, help="The path to your Galaxy configuration .ini file."
)
parser.add_argument(
"--from",
dest="bad_filename",
required=True,
help="The old, invalid path to the shed_tool_conf.xml or migrated_tools_conf.xml file.",
)
parser.add_argument(
"--to",
dest="good_filename",
required=True,
help="The updated path to the shed_tool_conf.xml or migrated_tools_conf.xml file.",
)
parser.add_argument(
"--force",
dest="force",
action="store_true",
help="Use this flag to set the new path even if the file does not (yet) exist there.",
)
opts = parser.parse_args()
if not os.path.exists(opts.good_filename) and not opts.force:
print('The file %s does not exist, use the --force option to proceed.' % opts.good_filename)
print("The file %s does not exist, use the --force option to proceed." % opts.good_filename)
exit(1)
session, model = create_database(opts.config_file)
exit(main(opts, session, model))