From 6dee708ae8cdad5ccc9f7ae095dae736b44a6091 Mon Sep 17 00:00:00 2001 From: Anastasia Tyryshkina Date: Fri, 1 Jun 2018 16:43:45 -0400 Subject: [PATCH 01/12] export another tsv file named datasets.tsv --- scripts/grt/export.py | 56 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/scripts/grt/export.py b/scripts/grt/export.py index 0daf8605bab..f119fcbbb14 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -32,7 +32,7 @@ def _init(args, need_app=False): config = galaxy.config.Configuration(**properties) object_store = build_object_store_from_config(config) if not config.database_connection: - logging.warning("The database connection is empty. If you are using the default value, please uncomment that in your galaxy.ini") + logging.warning("The database connection is empty. If you are using the default value, please uncomment that in your galaxy.yml") if need_app: config_file = config_file_from_args(args) @@ -205,6 +205,7 @@ def main(argv): annotate('galaxy_init', 'Loading Galaxy...') model, object_store, gxconfig, app = _init(args, need_app=config['grt']['share_toolbox']) + # Galaxy overrides our logging level. logging.getLogger().setLevel(getattr(logging, args.loglevel.upper())) sa_session = model.context.current @@ -277,6 +278,47 @@ def main(argv): handle_job.close() annotate('export_jobs_end') + annotate('export_datasets_start', 'Exporting Datasets') + handle_datasets = open(REPORT_BASE + '.datasets.tsv', 'w') + handle_datasets.write('\t'.join(('job_id', 'dataset_id', 'extension', 'file_size', 'name', 'param_name')) + '\n') + for offset_start in range(last_job_sent, end_job_id, args.batch_size): + logging.debug("Processing %s:%s", offset_start, min(end_job_id, offset_start + args.batch_size)) + for job in sa_session.query(model.Job) \ + .filter(model.Job.id > offset_start) \ + .filter(model.Job.id <= min(end_job_id, offset_start + args.batch_size)) \ + .all(): + + # If the tool is blacklisted, exclude everywhere + if job.tool_id in blacklisted_tools: + continue + + input_datasets=(job.input_datasets) + for input_dataset in input_datasets: + hda = input_dataset.dataset + dataset = hda.dataset + datasets=({'dataset_id': str(hda.dataset_id), + 'extension': hda.extension, + 'file_size': int(dataset.file_size), + 'name': hda.name, + 'param_name': input_dataset.name}) + + handle_datasets.write(str(job.id)) + handle_datasets.write('\t') + handle_datasets.write(datasets['dataset_id']) + handle_datasets.write('\t') + handle_datasets.write(datasets['extension']) + handle_datasets.write('\t') + handle_datasets.write(str(datasets['file_size'])) + handle_datasets.write('\t') + handle_datasets.write(datasets['name']) + handle_datasets.write('\t') + handle_datasets.write(datasets['param_name']) + handle_datasets.write('\n') + + + handle_datasets.close() + annotate('export_datasets_end') + annotate('export_metric_num_start', 'Exporting Metrics (Numeric)') handle_metric_num = open(REPORT_BASE + '.metric_num.tsv', 'w') handle_metric_num.write('\t'.join(('job_id', 'plugin', 'name', 'value')) + '\n') @@ -333,10 +375,10 @@ def main(argv): # Now on to outputs. with tarfile.open(REPORT_BASE + '.tar.gz', 'w:gz') as handle: - for name in ('jobs', 'metric_num', 'params'): + for name in ('jobs', 'metric_num', 'params', 'datasets'): handle.add(REPORT_BASE + '.' + name + '.tsv') - for name in ('jobs', 'metric_num', 'params'): + for name in ('jobs', 'metric_num', 'params', 'datasets'): os.unlink(REPORT_BASE + '.' + name + '.tsv') _times.append(('job_finish', time.time() - _start_time)) @@ -369,10 +411,10 @@ def main(argv): "tools": toolbox }, handle) - # Write our checkpoint file so we know where to start next time. - with open(CHECK_POINT_FILE, 'w') as handle: - handle.write(str(end_job_id)) + # # Write our checkpoint file so we know where to start next time. + # with open(CHECK_POINT_FILE, 'w') as handle: + # handle.write(str(end_job_id)) if __name__ == '__main__': - main(sys.argv) + main(sys.argv) \ No newline at end of file From 11575132dc4fba88751a5e2228956dfc1b75bbe2 Mon Sep 17 00:00:00 2001 From: Anastasia Tyryshkina Date: Fri, 1 Jun 2018 16:47:57 -0400 Subject: [PATCH 02/12] exports a dataset tsv --- scripts/grt/export.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/grt/export.py b/scripts/grt/export.py index f119fcbbb14..fdc6adfebdd 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -411,9 +411,9 @@ def main(argv): "tools": toolbox }, handle) - # # Write our checkpoint file so we know where to start next time. - # with open(CHECK_POINT_FILE, 'w') as handle: - # handle.write(str(end_job_id)) + # Write our checkpoint file so we know where to start next time. + with open(CHECK_POINT_FILE, 'w') as handle: + handle.write(str(end_job_id)) if __name__ == '__main__': From aa062f8546379bd35769314fb3470fde02445157 Mon Sep 17 00:00:00 2001 From: Anastasia Tyryshkina Date: Mon, 4 Jun 2018 11:53:14 -0400 Subject: [PATCH 03/12] fixed the issues with the last pull --- scripts/grt/export.py | 69 +++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 29 deletions(-) diff --git a/scripts/grt/export.py b/scripts/grt/export.py index fdc6adfebdd..26702ed349f 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -280,41 +280,52 @@ def main(argv): annotate('export_datasets_start', 'Exporting Datasets') handle_datasets = open(REPORT_BASE + '.datasets.tsv', 'w') - handle_datasets.write('\t'.join(('job_id', 'dataset_id', 'extension', 'file_size', 'name', 'param_name')) + '\n') + handle_datasets.write('\t'.join(('job_id', 'dataset_id', 'extension', 'file_size', 'param_name')) + '\n') for offset_start in range(last_job_sent, end_job_id, args.batch_size): logging.debug("Processing %s:%s", offset_start, min(end_job_id, offset_start + args.batch_size)) - for job in sa_session.query(model.Job) \ - .filter(model.Job.id > offset_start) \ - .filter(model.Job.id <= min(end_job_id, offset_start + args.batch_size)) \ - .all(): - # If the tool is blacklisted, exclude everywhere - if job.tool_id in blacklisted_tools: - continue + # three queries: JobToInputDatasetAssociation, HistoryDatasetAssociation, Dataset - input_datasets=(job.input_datasets) - for input_dataset in input_datasets: - hda = input_dataset.dataset - dataset = hda.dataset - datasets=({'dataset_id': str(hda.dataset_id), - 'extension': hda.extension, - 'file_size': int(dataset.file_size), - 'name': hda.name, - 'param_name': input_dataset.name}) + job_to_hda_ids=sa_session.query(model.JobToInputDatasetAssociation.job_id, model.JobToInputDatasetAssociation.dataset_id, + model.JobToInputDatasetAssociation.name) \ + .filter(model.JobToInputDatasetAssociation.job_id > offset_start) \ + .filter(model.JobToInputDatasetAssociation.job_id <= min(end_job_id, offset_start + args.batch_size)) \ + .all() - handle_datasets.write(str(job.id)) - handle_datasets.write('\t') - handle_datasets.write(datasets['dataset_id']) - handle_datasets.write('\t') - handle_datasets.write(datasets['extension']) - handle_datasets.write('\t') - handle_datasets.write(str(datasets['file_size'])) - handle_datasets.write('\t') - handle_datasets.write(datasets['name']) - handle_datasets.write('\t') - handle_datasets.write(datasets['param_name']) - handle_datasets.write('\n') + # put all of the hda_ids into a list + hda_ids=[i[1] for i in job_to_hda_ids] + hdas=sa_session.query(model.HistoryDatasetAssociation.id, model.HistoryDatasetAssociation.dataset_id, + model.HistoryDatasetAssociation.extension) \ + .filter(model.HistoryDatasetAssociation.id.in_(hda_ids)) \ + .all() + + # put all the dataset ids into a list + dataset_ids=[i[1] for i in hdas] + + # get the sizes of the datasets + datasets=sa_session.query(model.Dataset.id, model.Dataset.total_size) \ + .filter(model.HistoryDatasetAssociation.id.in_(dataset_ids)) \ + .all() + + # datasets to dictionay for easy search + hdas={i[0]:i[1:] for i in hdas} + datasets={i[0]:i[1:] for i in datasets} + + for job in job_to_hda_ids: + hda_id=job[1] + dataset_id=hdas[hda_id][0] + + handle_datasets.write(str(job[0])) + handle_datasets.write('\t') + handle_datasets.write(str(hda_id)) + handle_datasets.write('\t') + handle_datasets.write(hdas[hda_id][1]) # not sure if this results in a query, but this was the only way I managed to retrive it. + handle_datasets.write('\t') + handle_datasets.write(str(datasets[dataset_id][0])) + handle_datasets.write('\t') + handle_datasets.write(job[2]) + handle_datasets.write('\n') handle_datasets.close() annotate('export_datasets_end') From 0c852257bb9f91983a4afe063bf8d64692802e87 Mon Sep 17 00:00:00 2001 From: Anastasia Tyryshkina Date: Mon, 4 Jun 2018 12:39:55 -0400 Subject: [PATCH 04/12] delete comment thats no longer necessary --- scripts/grt/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/grt/export.py b/scripts/grt/export.py index 26702ed349f..929cd8fd8d3 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -320,7 +320,7 @@ def main(argv): handle_datasets.write('\t') handle_datasets.write(str(hda_id)) handle_datasets.write('\t') - handle_datasets.write(hdas[hda_id][1]) # not sure if this results in a query, but this was the only way I managed to retrive it. + handle_datasets.write(hdas[hda_id][1]) handle_datasets.write('\t') handle_datasets.write(str(datasets[dataset_id][0])) handle_datasets.write('\t') From 5b7055fc6f19e2a0489a92bb8f4762a1b6a1ece1 Mon Sep 17 00:00:00 2001 From: Anastasia Tyryshkina Date: Mon, 4 Jun 2018 12:56:47 -0400 Subject: [PATCH 05/12] respect blacklisted tools --- scripts/grt/export.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/grt/export.py b/scripts/grt/export.py index 929cd8fd8d3..823f0b11316 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -313,6 +313,14 @@ def main(argv): datasets={i[0]:i[1:] for i in datasets} for job in job_to_hda_ids: + + # No associated job + if job[0] not in job_tool_map: + continue + # If the tool is blacklisted, exclude everywhere + if job_tool_map[job[0]] in blacklisted_tools: + continue + hda_id=job[1] dataset_id=hdas[hda_id][0] @@ -320,7 +328,7 @@ def main(argv): handle_datasets.write('\t') handle_datasets.write(str(hda_id)) handle_datasets.write('\t') - handle_datasets.write(hdas[hda_id][1]) + handle_datasets.write(hdas[hda_id][1]) handle_datasets.write('\t') handle_datasets.write(str(datasets[dataset_id][0])) handle_datasets.write('\t') From 4630994fd0479e66357315fc7e2fa2c20367fb84 Mon Sep 17 00:00:00 2001 From: Anastasia Tyryshkina Date: Mon, 4 Jun 2018 13:33:59 -0400 Subject: [PATCH 06/12] get output datasets as well --- scripts/grt/export.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/scripts/grt/export.py b/scripts/grt/export.py index 823f0b11316..fb8f4990778 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -280,20 +280,30 @@ def main(argv): annotate('export_datasets_start', 'Exporting Datasets') handle_datasets = open(REPORT_BASE + '.datasets.tsv', 'w') - handle_datasets.write('\t'.join(('job_id', 'dataset_id', 'extension', 'file_size', 'param_name')) + '\n') + handle_datasets.write('\t'.join(('job_id', 'dataset_id', 'extension', 'file_size', 'param_name', 'type')) + '\n') for offset_start in range(last_job_sent, end_job_id, args.batch_size): logging.debug("Processing %s:%s", offset_start, min(end_job_id, offset_start + args.batch_size)) - # three queries: JobToInputDatasetAssociation, HistoryDatasetAssociation, Dataset + # four queries: JobToInputDatasetAssociation, JobToOutputDatasetAssociation, HistoryDatasetAssociation, Dataset - job_to_hda_ids=sa_session.query(model.JobToInputDatasetAssociation.job_id, model.JobToInputDatasetAssociation.dataset_id, + job_to_input_hda_ids=sa_session.query(model.JobToInputDatasetAssociation.job_id, model.JobToInputDatasetAssociation.dataset_id, model.JobToInputDatasetAssociation.name) \ .filter(model.JobToInputDatasetAssociation.job_id > offset_start) \ .filter(model.JobToInputDatasetAssociation.job_id <= min(end_job_id, offset_start + args.batch_size)) \ .all() + job_to_output_hda_ids=sa_session.query(model.JobToOutputDatasetAssociation.job_id, model.JobToOutputDatasetAssociation.dataset_id, + model.JobToOutputDatasetAssociation.name) \ + .filter(model.JobToOutputDatasetAssociation.job_id > offset_start) \ + .filter(model.JobToOutputDatasetAssociation.job_id <= min(end_job_id, offset_start + args.batch_size)) \ + .all() + + + # add type and concat + job_to_hda_ids=[[list(i),"input"] for i in job_to_input_hda_ids]+[[list(i),"output"] for i in job_to_output_hda_ids] + # put all of the hda_ids into a list - hda_ids=[i[1] for i in job_to_hda_ids] + hda_ids=[i[0][1] for i in job_to_hda_ids] hdas=sa_session.query(model.HistoryDatasetAssociation.id, model.HistoryDatasetAssociation.dataset_id, model.HistoryDatasetAssociation.extension) \ @@ -305,7 +315,7 @@ def main(argv): # get the sizes of the datasets datasets=sa_session.query(model.Dataset.id, model.Dataset.total_size) \ - .filter(model.HistoryDatasetAssociation.id.in_(dataset_ids)) \ + .filter(model.Dataset.id.in_(dataset_ids)) \ .all() # datasets to dictionay for easy search @@ -314,6 +324,9 @@ def main(argv): for job in job_to_hda_ids: + filetype=job[1] + job=job[0] + # No associated job if job[0] not in job_tool_map: continue @@ -333,6 +346,8 @@ def main(argv): handle_datasets.write(str(datasets[dataset_id][0])) handle_datasets.write('\t') handle_datasets.write(job[2]) + handle_datasets.write('\t') + handle_datasets.write(filetype) handle_datasets.write('\n') handle_datasets.close() From ba45c15fffcb9e54dda35759410675d78346394f Mon Sep 17 00:00:00 2001 From: Anastasia Tyryshkina Date: Mon, 4 Jun 2018 14:27:24 -0400 Subject: [PATCH 07/12] tried to fix some of stylization fails --- scripts/grt/export.py | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/scripts/grt/export.py b/scripts/grt/export.py index fb8f4990778..3b27b1ddf2f 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -205,7 +205,7 @@ def main(argv): annotate('galaxy_init', 'Loading Galaxy...') model, object_store, gxconfig, app = _init(args, need_app=config['grt']['share_toolbox']) - + # Galaxy overrides our logging level. logging.getLogger().setLevel(getattr(logging, args.loglevel.upper())) sa_session = model.context.current @@ -286,46 +286,47 @@ def main(argv): # four queries: JobToInputDatasetAssociation, JobToOutputDatasetAssociation, HistoryDatasetAssociation, Dataset - job_to_input_hda_ids=sa_session.query(model.JobToInputDatasetAssociation.job_id, model.JobToInputDatasetAssociation.dataset_id, + # /scripts/grt/export.py:291:17: E127 continuation line over-indented for visual indent + job_to_input_hda_ids = sa_session.query(model.JobToInputDatasetAssociation.job_id, model.JobToInputDatasetAssociation.dataset_id, model.JobToInputDatasetAssociation.name) \ .filter(model.JobToInputDatasetAssociation.job_id > offset_start) \ .filter(model.JobToInputDatasetAssociation.job_id <= min(end_job_id, offset_start + args.batch_size)) \ .all() - job_to_output_hda_ids=sa_session.query(model.JobToOutputDatasetAssociation.job_id, model.JobToOutputDatasetAssociation.dataset_id, + # ./scripts/grt/export.py:297:17: E127 continuation line over-indented for visual indent + job_to_output_hda_ids = sa_session.query(model.JobToOutputDatasetAssociation.job_id, model.JobToOutputDatasetAssociation.dataset_id, model.JobToOutputDatasetAssociation.name) \ .filter(model.JobToOutputDatasetAssociation.job_id > offset_start) \ .filter(model.JobToOutputDatasetAssociation.job_id <= min(end_job_id, offset_start + args.batch_size)) \ .all() - # add type and concat - job_to_hda_ids=[[list(i),"input"] for i in job_to_input_hda_ids]+[[list(i),"output"] for i in job_to_output_hda_ids] - - # put all of the hda_ids into a list - hda_ids=[i[0][1] for i in job_to_hda_ids] + job_to_hda_ids = [[list(i), "input"] for i in job_to_input_hda_ids] + [[list(i), "output"] for i in job_to_output_hda_ids] - hdas=sa_session.query(model.HistoryDatasetAssociation.id, model.HistoryDatasetAssociation.dataset_id, - model.HistoryDatasetAssociation.extension) \ - .filter(model.HistoryDatasetAssociation.id.in_(hda_ids)) \ - .all() + # put all of the hda_ids into a list + hda_ids = [i[0][1] for i in job_to_hda_ids] + + hdas = sa_session.query(model.HistoryDatasetAssociation.id, model.HistoryDatasetAssociation.dataset_id, + model.HistoryDatasetAssociation.extension) \ + .filter(model.HistoryDatasetAssociation.id.in_(hda_ids)) \ + .all() # put all the dataset ids into a list - dataset_ids=[i[1] for i in hdas] + dataset_ids = [i[1] for i in hdas] # get the sizes of the datasets - datasets=sa_session.query(model.Dataset.id, model.Dataset.total_size) \ + datasets = sa_session.query(model.Dataset.id, model.Dataset.total_size) \ .filter(model.Dataset.id.in_(dataset_ids)) \ .all() # datasets to dictionay for easy search - hdas={i[0]:i[1:] for i in hdas} - datasets={i[0]:i[1:] for i in datasets} + hdas = {i[0]:i[1: ] for i in hdas} + datasets = {i[0]:i[1: ] for i in datasets} for job in job_to_hda_ids: - filetype=job[1] - job=job[0] + filetype = job[1] + job = job[0] # No associated job if job[0] not in job_tool_map: @@ -334,8 +335,8 @@ def main(argv): if job_tool_map[job[0]] in blacklisted_tools: continue - hda_id=job[1] - dataset_id=hdas[hda_id][0] + hda_id = job[1] + dataset_id = hdas[hda_id][0] handle_datasets.write(str(job[0])) handle_datasets.write('\t') @@ -451,4 +452,4 @@ def main(argv): if __name__ == '__main__': - main(sys.argv) \ No newline at end of file + main(sys.argv) From 948d0767eb9b2e2a6207ead2456a6580e34eaf84 Mon Sep 17 00:00:00 2001 From: Anastasia Tyryshkina Date: Mon, 4 Jun 2018 14:34:09 -0400 Subject: [PATCH 08/12] more style fixes --- scripts/grt/export.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/scripts/grt/export.py b/scripts/grt/export.py index 3b27b1ddf2f..b1111ced3c1 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -288,17 +288,17 @@ def main(argv): # /scripts/grt/export.py:291:17: E127 continuation line over-indented for visual indent job_to_input_hda_ids = sa_session.query(model.JobToInputDatasetAssociation.job_id, model.JobToInputDatasetAssociation.dataset_id, - model.JobToInputDatasetAssociation.name) \ - .filter(model.JobToInputDatasetAssociation.job_id > offset_start) \ - .filter(model.JobToInputDatasetAssociation.job_id <= min(end_job_id, offset_start + args.batch_size)) \ - .all() + model.JobToInputDatasetAssociation.name) \ + .filter(model.JobToInputDatasetAssociation.job_id > offset_start) \ + .filter(model.JobToInputDatasetAssociation.job_id <= min(end_job_id, offset_start + args.batch_size)) \ + .all() # ./scripts/grt/export.py:297:17: E127 continuation line over-indented for visual indent job_to_output_hda_ids = sa_session.query(model.JobToOutputDatasetAssociation.job_id, model.JobToOutputDatasetAssociation.dataset_id, - model.JobToOutputDatasetAssociation.name) \ - .filter(model.JobToOutputDatasetAssociation.job_id > offset_start) \ - .filter(model.JobToOutputDatasetAssociation.job_id <= min(end_job_id, offset_start + args.batch_size)) \ - .all() + model.JobToOutputDatasetAssociation.name) \ + .filter(model.JobToOutputDatasetAssociation.job_id > offset_start) \ + .filter(model.JobToOutputDatasetAssociation.job_id <= min(end_job_id, offset_start + args.batch_size)) \ + .all() # add type and concat job_to_hda_ids = [[list(i), "input"] for i in job_to_input_hda_ids] + [[list(i), "output"] for i in job_to_output_hda_ids] @@ -316,12 +316,12 @@ def main(argv): # get the sizes of the datasets datasets = sa_session.query(model.Dataset.id, model.Dataset.total_size) \ - .filter(model.Dataset.id.in_(dataset_ids)) \ - .all() + .filter(model.Dataset.id.in_(dataset_ids)) \ + .all() # datasets to dictionay for easy search - hdas = {i[0]:i[1: ] for i in hdas} - datasets = {i[0]:i[1: ] for i in datasets} + hdas = {i[0]: i[1:] for i in hdas} + datasets = {i[0]: i[1:] for i in datasets} for job in job_to_hda_ids: @@ -342,7 +342,7 @@ def main(argv): handle_datasets.write('\t') handle_datasets.write(str(hda_id)) handle_datasets.write('\t') - handle_datasets.write(hdas[hda_id][1]) + handle_datasets.write(hdas[hda_id][1]) handle_datasets.write('\t') handle_datasets.write(str(datasets[dataset_id][0])) handle_datasets.write('\t') From 191a134bbc511c508d9ee0636030da2e2e68ea44 Mon Sep 17 00:00:00 2001 From: Anastasia Tyryshkina Date: Mon, 4 Jun 2018 15:57:22 -0400 Subject: [PATCH 09/12] got rid of things that are no longer necessary --- scripts/grt/export.py | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/scripts/grt/export.py b/scripts/grt/export.py index b1111ced3c1..7513c43b465 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -63,7 +63,6 @@ class Sanitization: # SA Stuff self.model = model self.sa_session = sa_session - self.filesize_cache = {} if 'tool_params' not in self.sanitization_config: self.sanitization_config['tool_params'] = {} @@ -99,37 +98,7 @@ class Sanitization: self.tool_id = tool_id return json.dumps(self._sanitize_value(unsanitized)) - def _file_dict(self, data): - key = '{src}-{id}'.format(**data) - if key in self.filesize_cache: - return self.filesize_cache[data] - if data['src'] == 'hda': - try: - dataset = self.sa_session.query(self.model.Dataset.id, self.model.Dataset.total_size) \ - .filter_by(id=data['id']) \ - .first() - if dataset and dataset[1]: - data['size'] = int(dataset[1]) - else: - data['size'] = None - except sa.orm.exc.NoResultFound: - data['size'] = None - - # Push to cache for later. - self.filesize_cache[data['id']] = data - return data - else: - logging.warning("Cannot handle {src} yet".format(data)) - return data - def _sanitize_dict(self, unsanitized_dict, path=""): - # if it is a file dictionary, handle specially. - if len(unsanitized_dict.keys()) == 2 and \ - 'id' in unsanitized_dict and \ - 'src' in unsanitized_dict and \ - unsanitized_dict['src'] in ('hda', 'ldda'): - return self._file_dict(unsanitized_dict) - return { k: self._sanitize_value(v, path=path + '.' + k) for (k, v) @@ -286,14 +255,12 @@ def main(argv): # four queries: JobToInputDatasetAssociation, JobToOutputDatasetAssociation, HistoryDatasetAssociation, Dataset - # /scripts/grt/export.py:291:17: E127 continuation line over-indented for visual indent job_to_input_hda_ids = sa_session.query(model.JobToInputDatasetAssociation.job_id, model.JobToInputDatasetAssociation.dataset_id, model.JobToInputDatasetAssociation.name) \ .filter(model.JobToInputDatasetAssociation.job_id > offset_start) \ .filter(model.JobToInputDatasetAssociation.job_id <= min(end_job_id, offset_start + args.batch_size)) \ .all() - # ./scripts/grt/export.py:297:17: E127 continuation line over-indented for visual indent job_to_output_hda_ids = sa_session.query(model.JobToOutputDatasetAssociation.job_id, model.JobToOutputDatasetAssociation.dataset_id, model.JobToOutputDatasetAssociation.name) \ .filter(model.JobToOutputDatasetAssociation.job_id > offset_start) \ From 6db9fd3d51b91804e24ed393468973ecfb659eef Mon Sep 17 00:00:00 2001 From: Anastasia Tyryshkina Date: Mon, 4 Jun 2018 16:01:32 -0400 Subject: [PATCH 10/12] style fix --- scripts/grt/export.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/grt/export.py b/scripts/grt/export.py index 7513c43b465..0cb92b31025 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -12,7 +12,6 @@ import tarfile import time from collections import defaultdict -import sqlalchemy as sa import yaml sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib'))) From dd4de3f3b28f1ddd92aaa87c4f2c00657ecbc457 Mon Sep 17 00:00:00 2001 From: Anastasia Tyryshkina Date: Tue, 5 Jun 2018 12:23:02 -0400 Subject: [PATCH 11/12] catch dataset_id=None --- scripts/grt/export.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/grt/export.py b/scripts/grt/export.py index 0cb92b31025..62e45df3dc0 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -302,6 +302,9 @@ def main(argv): continue hda_id = job[1] + # catch hda_id's where + if hda_id == None: + continue dataset_id = hdas[hda_id][0] handle_datasets.write(str(job[0])) From c37e3844543b1bd425cfd370830bdb235674706f Mon Sep 17 00:00:00 2001 From: Anastasia Tyryshkina Date: Tue, 5 Jun 2018 12:47:19 -0400 Subject: [PATCH 12/12] style fix --- scripts/grt/export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/grt/export.py b/scripts/grt/export.py index 62e45df3dc0..9bca8d929f7 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -302,8 +302,8 @@ def main(argv): continue hda_id = job[1] - # catch hda_id's where - if hda_id == None: + # catch hda_id's where + if hda_id is None: continue dataset_id = hdas[hda_id][0]