diff --git a/.github/workflows/db_indexes.yaml b/.github/workflows/db_indexes.yaml index b11d21bf025..64f46d667fa 100644 --- a/.github/workflows/db_indexes.yaml +++ b/.github/workflows/db_indexes.yaml @@ -11,9 +11,13 @@ jobs: matrix: python-version: ['3.7'] db: ['postgresql', 'sqlite'] + postgresql-version: ['13'] + include: + - db: postgresql + postgresql-version: '9.6' services: postgres: - image: postgres:13 + image: postgres:${{ matrix.postgresql-version }} env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres diff --git a/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue b/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue index a9fddc56612..fabf9dfeba7 100644 --- a/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue +++ b/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue @@ -250,9 +250,6 @@ export default { // logic from legacy code return !!(this.contains_file_or_folder && Galaxy.user); }, - allDatasets: function () { - return this.folderContents.filter((element) => element.type === "file"); - }, }, methods: { updateSearch: function (value) { @@ -271,7 +268,11 @@ export default { async getSelected() { if (this.isAllSelectedMode) { this.$emit("setBusy", true); - const selected = await this.services.getFilteredFolderContents(this.folder_id, this.unselected); + const selected = await this.services.getFilteredFolderContents( + this.folder_id, + this.unselected, + this.$parent.search_text + ); this.$emit("setBusy", false); return selected; } else { @@ -319,7 +320,6 @@ export default { if (isCollection) { new mod_import_collection.ImportCollectionModal({ selected: checkedItems, - allDatasets: this.allDatasets, }); } else { new mod_import_dataset.ImportDatasetModal({ diff --git a/client/src/components/Libraries/LibraryFolder/TopToolbar/add-datasets.js b/client/src/components/Libraries/LibraryFolder/TopToolbar/add-datasets.js index 9f3a2cc977c..79cb6ae1b93 100644 --- a/client/src/components/Libraries/LibraryFolder/TopToolbar/add-datasets.js +++ b/client/src/components/Libraries/LibraryFolder/TopToolbar/add-datasets.js @@ -710,6 +710,26 @@ var AddDatasets = Backbone.View.extend({ } }, + /** + * User clicked the checkbox in the table heading + * @param {context} event + */ + selectAll: function (event) { + var selected = event.target.checked; + var self = this; + // Iterate each checkbox + $(":checkbox", "#dataset_list tbody").each(function () { + this.checked = selected; + var $row = $(this).closest("tr"); + // Change color of selected/unselected + if (selected) { + self.makeDarkRow($row); + } else { + self.makeWhiteRow($row); + } + }); + }, + makeDarkRow: function ($row) { $row.addClass("table-primary"); }, diff --git a/client/src/components/Libraries/LibraryFolder/TopToolbar/import-to-history/import-collection.js b/client/src/components/Libraries/LibraryFolder/TopToolbar/import-to-history/import-collection.js index b1325b50c9c..6529dc50e7b 100644 --- a/client/src/components/Libraries/LibraryFolder/TopToolbar/import-to-history/import-collection.js +++ b/client/src/components/Libraries/LibraryFolder/TopToolbar/import-to-history/import-collection.js @@ -42,36 +42,40 @@ var ImportCollectionModal = Backbone.View.extend({ showCollectionSelect: function (e) { const Galaxy = getGalaxyInstance(); var checked_items = this.findCheckedItems(); - var template = this.templateCollectionSelectModal(); + if (checked_items.length === 0) { + Toast.info("You must select some datasets first."); + } else { + var template = this.templateCollectionSelectModal(); - var promise = this.fetchUserHistories(); - promise - .done(() => { - this.modal = Galaxy.modal; - this.modal.show({ - closing_events: true, - title: "Create History Collection from Datasets", - body: template({ - selected_datasets: checked_items.dataset_ids.length, - histories: this.histories.models, - }), - buttons: { - Continue: () => { - this.showCollectionBuilder(checked_items.dataset_ids); + var promise = this.fetchUserHistories(); + promise + .done(() => { + this.modal = Galaxy.modal; + this.modal.show({ + closing_events: true, + title: "Create History Collection from Datasets", + body: template({ + selected_datasets: checked_items.dataset_ids.length, + histories: this.histories.models, + }), + buttons: { + Continue: () => { + this.showCollectionBuilder(checked_items.dataset_ids); + }, + Close: () => { + Galaxy.modal.hide(); + }, }, - Close: () => { - Galaxy.modal.hide(); - }, - }, + }); + }) + .fail((model, response) => { + if (typeof response.responseJSON !== "undefined") { + Toast.error(response.responseJSON.err_msg); + } else { + Toast.error("An error occurred."); + } }); - }) - .fail((model, response) => { - if (typeof response.responseJSON !== "undefined") { - Toast.error(response.responseJSON.err_msg); - } else { - Toast.error("An error occurred."); - } - }); + } }, /** * Note: The collection creation process expects ldda_ids as ids @@ -79,21 +83,17 @@ var ImportCollectionModal = Backbone.View.extend({ * The code below overwrites the id with ldda_id for this reason. */ showCollectionBuilder: function (checked_items) { - let collection_elements = []; - const elements_source = this.modal.$('input[type="radio"]:checked').val(); - if (elements_source === "selection") { - for (let i = checked_items.length - 1; i >= 0; i--) { - const collection_item = {}; - const dataset = checked_items[i]; - collection_item.id = dataset.ldda_id; - collection_item.name = dataset.name; - collection_item.deleted = dataset.deleted; - collection_item.state = dataset.state; - collection_elements.push(collection_item); - } - } else if (elements_source === "folder") { - collection_elements = this.options.allDatasets; + const collection_elements = []; + for (let i = checked_items.length - 1; i >= 0; i--) { + const collection_item = {}; + const dataset = checked_items[i]; + collection_item.id = dataset.ldda_id; + collection_item.name = dataset.name; + collection_item.deleted = dataset.deleted; + collection_item.state = dataset.state; + collection_elements.push(collection_item); } + const new_history_name = this.modal.$("input[name=history_name]").val(); if (new_history_name !== "") { this.createNewHistory(new_history_name) @@ -198,25 +198,6 @@ var ImportCollectionModal = Backbone.View.extend({ templateCollectionSelectModal: function () { return _.template( `
-
-

Which datasets?

-
-
- disabled <% } else { %> checked <% } %> /> - -
-
- checked <% } %> > - -
-
-

Collection type

diff --git a/client/src/components/Libraries/LibraryFolder/services.js b/client/src/components/Libraries/LibraryFolder/services.js index 4225df9be57..34f3f0e47ee 100644 --- a/client/src/components/Libraries/LibraryFolder/services.js +++ b/client/src/components/Libraries/LibraryFolder/services.js @@ -8,8 +8,11 @@ export class Services { } async getFolderContents(id, include_deleted, limit, offset, search_text = false) { - search_text = search_text ? `&search_text=${encodeURI(search_text.trim())}` : ""; - const url = `${this.root}api/folders/${id}/contents?include_deleted=${include_deleted}&limit=${limit}&offset=${offset}${search_text}`; + const url = `${ + this.root + }api/folders/${id}/contents?include_deleted=${include_deleted}&limit=${limit}&offset=${offset}${this.getSearchQuery( + search_text + )}`; try { const response = await axios.get(url); return response.data; @@ -18,13 +21,17 @@ export class Services { } } - async getFilteredFolderContents(id, excluded) { - const contents = await axios.get(`${this.root}api/folders/${id}/contents`); + async getFilteredFolderContents(id, excluded, search_text) { + const contents = await axios.get(`${this.root}api/folders/${id}/contents?${this.getSearchQuery(search_text)}`); return contents.data.folder_contents.filter((item) => { return !excluded.some((exc) => exc.id === item.id); }); } + getSearchQuery(search_text) { + return search_text ? `&search_text=${encodeURI(search_text.trim())}` : ""; + } + updateFolder(item, onSucess, onError) { const url = `${this.root}api/folders/${item.id}`; try { diff --git a/lib/galaxy/managers/markdown_parse.py b/lib/galaxy/managers/markdown_parse.py index f4f658c878c..1ca312deb8b 100644 --- a/lib/galaxy/managers/markdown_parse.py +++ b/lib/galaxy/managers/markdown_parse.py @@ -33,9 +33,9 @@ VALID_ARGUMENTS = { "tool_stdout": ["step", "job_id"], "generate_galaxy_version": [], "generate_time": [], - "invocation_time": ["invocation_id"], "visualization": DYNAMIC_ARGUMENTS, # Invocation Flavored Markdown + "invocation_time": ["invocation_id"], "invocation_outputs": [], "invocation_inputs": [], } diff --git a/lib/galaxy/managers/markdown_util.py b/lib/galaxy/managers/markdown_util.py index 839263418e9..d0b36cba754 100644 --- a/lib/galaxy/managers/markdown_util.py +++ b/lib/galaxy/managers/markdown_util.py @@ -685,8 +685,8 @@ history_dataset_collection_display(input={}) return (f"workflow_display(workflow_id={invocation.workflow.stored_workflow.id})\n", False) if container == "history_link": return (f"history_link(history_id={invocation.history.id})\n", False) - if container == "invocation_date": - return (f"invocation_date(invocation_id={invocation.id})\n", False) + if container == "invocation_time": + return (f"invocation_time(invocation_id={invocation.id})\n", False) ref_object_type = None output_match = re.search(OUTPUT_LABEL_PATTERN, line) input_match = re.search(INPUT_LABEL_PATTERN, line) diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py index c39df81bc9d..024f549643f 100644 --- a/lib/galaxy/metadata/set_metadata.py +++ b/lib/galaxy/metadata/set_metadata.py @@ -139,6 +139,7 @@ def set_metadata_portable(): version_string = "" export_store = None + final_job_state = Job.states.OK if extended_metadata_collection: tool_dict = metadata_params["tool"] stdio_exit_code_dicts, stdio_regex_dicts = tool_dict["stdio_exit_codes"], tool_dict["stdio_regexes"] @@ -198,7 +199,7 @@ def set_metadata_portable(): if os.path.exists(COMMAND_VERSION_FILENAME): version_string = open(COMMAND_VERSION_FILENAME).read() - job_context = ExpressionContext(dict(stdout=tool_stdout, stderr=tool_stderr)) + expression_context = ExpressionContext(dict(stdout=tool_stdout, stderr=tool_stderr)) # Load outputs. export_store = store.DirectoryModelExportStore('metadata/outputs_populated', serialize_dataset_objects=True, for_edit=True, strip_metadata_files=False, serialize_jobs=False) @@ -208,6 +209,27 @@ def set_metadata_portable(): # Remove in 21.09, this should only happen for jobs that started on <= 20.09 and finish now import_model_store = None + job_context = SessionlessJobContext( + metadata_params, + tool_provided_metadata, + object_store, + export_store, + import_model_store, + os.path.join(tool_job_working_directory, "working"), + final_job_state=final_job_state, + ) + + unnamed_id_to_path = {} + for unnamed_output_dict in job_context.tool_provided_metadata.get_unnamed_outputs(): + destination = unnamed_output_dict["destination"] + elements = unnamed_output_dict["elements"] + destination_type = destination["type"] + if destination_type == 'hdas': + for element in elements: + filename = element.get('filename') + if filename: + unnamed_id_to_path[element['object_id']] = os.path.join(job_context.job_working_directory, filename) + for output_name, output_dict in outputs.items(): dataset_instance_id = output_dict["id"] klass = getattr(galaxy.model, output_dict.get('model_class', 'HistoryDatasetAssociation')) @@ -232,7 +254,7 @@ def set_metadata_portable(): # Same block as below... set_meta_kwds = stringify_dictionary_keys(json.load(open(filename_kwds))) # load kwds; need to ensure our keywords are not unicode try: - dataset.dataset.external_filename = dataset_filename_override + dataset.dataset.external_filename = unnamed_id_to_path.get(dataset_instance_id, dataset_filename_override) store_by = output_dict.get("object_store_store_by", legacy_object_store_store_by) extra_files_dir_name = f"dataset_{getattr(dataset.dataset, store_by)}_files" files_path = os.path.abspath(os.path.join(tool_job_working_directory, "working", extra_files_dir_name)) @@ -248,14 +270,17 @@ def set_metadata_portable(): setattr(dataset.metadata, metadata_name, metadata_file_override) if output_dict.get("validate", False): set_validated_state(dataset) - set_meta(dataset, file_dict) + if dataset_instance_id not in unnamed_id_to_path: + # We're going to run through set_metadata in collect_dynamic_outputs with more contextual metadata, + # so skip set_meta here. + set_meta(dataset, file_dict) if extended_metadata_collection: meta = tool_provided_metadata.get_dataset_meta(output_name, dataset.dataset.id, dataset.dataset.uuid) if meta: - context = ExpressionContext(meta, job_context) + context = ExpressionContext(meta, expression_context) else: - context = job_context + context = expression_context # Lazy and unattached # if getattr(dataset, "hidden_beneath_collection_instance", None): @@ -314,17 +339,6 @@ def set_metadata_portable(): if extended_metadata_collection: # discover extra outputs... - - job_context = SessionlessJobContext( - metadata_params, - tool_provided_metadata, - object_store, - export_store, - import_model_store, - os.path.join(tool_job_working_directory, "working"), - final_job_state=final_job_state, - ) - output_collections = {} for name, output_collection in metadata_params["output_collections"].items(): output_collections[name] = import_model_store.sa_session.query(HistoryDatasetCollectionAssociation).find(output_collection["id"]) diff --git a/lib/galaxy/model/migrate/triggers/update_audit_table.py b/lib/galaxy/model/migrate/triggers/update_audit_table.py index d3340f8b727..29bf9cfd76a 100644 --- a/lib/galaxy/model/migrate/triggers/update_audit_table.py +++ b/lib/galaxy/model/migrate/triggers/update_audit_table.py @@ -83,7 +83,7 @@ def _postgres_install(engine): $BODY$ """ - def statement_trigger_def(source_table, id_field, operation, when="AFTER"): + def statement_trigger_def(source_table, id_field, operation, when="AFTER", function_keyword="FUNCTION"): fn = f"{fn_prefix}_{id_field}" # Postgres supports many triggers per operation/table so the label can @@ -95,10 +95,10 @@ def _postgres_install(engine): CREATE TRIGGER {trigger_name} {when} {operation} ON {source_table} REFERENCING NEW TABLE AS new_table - FOR EACH STATEMENT EXECUTE FUNCTION {fn}(); + FOR EACH STATEMENT EXECUTE {function_keyword} {fn}(); """ - def row_trigger_def(source_table, id_field, operation, when="AFTER"): + def row_trigger_def(source_table, id_field, operation, when="AFTER", function_keyword="FUNCTION"): fn = f"{fn_prefix}_{id_field}" label = f"history_audit_by_{id_field}" @@ -107,20 +107,24 @@ def _postgres_install(engine): return f""" CREATE TRIGGER {trigger_name} {when} {operation} ON {source_table} - FOR EACH ROW EXECUTE FUNCTION {fn}(); + FOR EACH ROW EXECUTE {function_keyword} {fn}(); """ # pick row or statement triggers depending on postgres version version = engine.dialect.server_version_info[0] trigger_fn = statement_trigger_fn if version > 10 else row_trigger_fn trigger_def = statement_trigger_def if version > 10 else row_trigger_def + # In the syntax of CREATE TRIGGER, the keywords FUNCTION and PROCEDURE are equivalent, + # but the referenced function must in any case be a function, not a procedure. + # The use of the keyword PROCEDURE here is historical and deprecated (https://www.postgresql.org/docs/11/sql-createtrigger.html). + function_keyword = "FUNCTION" if version > 10 else "PROCEDURE" for id_field in ["history_id", "id"]: sql.append(trigger_fn(id_field)) for source_table, id_field in trigger_config.items(): for operation in ["UPDATE", "INSERT"]: - sql.append(trigger_def(source_table, id_field, operation)) + sql.append(trigger_def(source_table, id_field, operation, function_keyword=function_keyword)) return sql diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 66846a8df04..c5310a5009f 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -184,12 +184,12 @@ def _bam_to_sam(local_name, temp_name): temp_local = tempfile.NamedTemporaryFile(suffix='.sam', prefix='local_bam_converted_to_sam_') with tempfile.NamedTemporaryFile(suffix='.sam', prefix='history_bam_converted_to_sam_', delete=False) as temp: try: - pysam.view('-h', f'-o{temp_local.name}', local_name) + pysam.view('-h', '--no-PG', '-o', temp_local.name, local_name, catch_stdout=False) except Exception as e: msg = f"Converting local (test-data) BAM to SAM failed: {unicodify(e)}" raise Exception(msg) try: - pysam.view('-h', f'-o{temp.name}', temp_name) + pysam.view('-h', '--no-PG', '-o', temp.name, temp_name, catch_stdout=False) except Exception as e: msg = f"Converting history BAM to SAM failed: {unicodify(e)}" raise Exception(msg) diff --git a/lib/galaxy_test/api/test_tools_upload.py b/lib/galaxy_test/api/test_tools_upload.py index d94c318aa9d..50c39616d63 100644 --- a/lib/galaxy_test/api/test_tools_upload.py +++ b/lib/galaxy_test/api/test_tools_upload.py @@ -322,6 +322,28 @@ class ToolsUploadTestCase(ApiTestCase): assert output0["state"] == "ok" assert output1["state"] == "error" + @uses_test_history(require_new=False) + def test_fetch_bam_file_from_url_with_extension_set(self, history_id): + destination = {"type": "hdas"} + targets = [{ + "destination": destination, + "items": [ + { + "src": "url", + "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bam", + "ext": "bam" + }, + ] + }] + payload = { + "history_id": history_id, + "targets": json.dumps(targets), + } + fetch_response = self.dataset_populator.fetch(payload) + self._assert_status_code_is(fetch_response, 200) + outputs = fetch_response.json()["outputs"] + self.dataset_populator.get_history_dataset_details(history_id, dataset=outputs[0], assert_ok=True) + @skip_without_datatype("velvet") def test_composite_datatype(self): with self.dataset_populator.test_history() as history_id: diff --git a/test/functional/tools/sam_to_bam.xml b/test/functional/tools/sam_to_bam.xml index b01152a7670..3420ed73ec9 100644 --- a/test/functional/tools/sam_to_bam.xml +++ b/test/functional/tools/sam_to_bam.xml @@ -11,7 +11,7 @@ cat '$input1' > '$out_file1' - + diff --git a/test/unit/workflows/test_workflow_markdown.py b/test/unit/workflows/test_workflow_markdown.py index 742d557b451..262b78650c5 100644 --- a/test/unit/workflows/test_workflow_markdown.py +++ b/test/unit/workflows/test_workflow_markdown.py @@ -53,6 +53,18 @@ history_dataset_peek(input=input1) assert "```galaxy\nhistory_dataset_peek(history_dataset_id=567)\n```" in galaxy_markdown +def test_invocation_time(): + workflow_markdown = """ +And outputs... + +```galaxy +invocation_time() +``` +""" + galaxy_markdown = resolved_markdown(workflow_markdown) + assert "```galaxy\ninvocation_time(invocation_id=44)\n```" in galaxy_markdown + + def test_output_reference_mapping(): workflow_markdown = """ And outputs... @@ -78,6 +90,7 @@ def example_invocation(trans): invocation = model.WorkflowInvocation() workflow = yaml_to_model(TEST_WORKFLOW_YAML) workflow.id = 342 + invocation.id = 44 invocation.workflow = workflow # TODO: fix this to use workflow id and eliminate hack.