opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'release_21.05' into dev

Browse Source
This commit is contained in:
mvdbeek
2021-06-03 10:23:52 +02:00
parent 17078f5011 90dff9af5b
commit bf78f40bf6
13 changed files with 162 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/db_indexes.yaml
+5 -1
View File
@@ -11,9 +11,13 @@ jobs:
matrix:
python-version: ['3.7']
db: ['postgresql', 'sqlite']
postgresql-version: ['13']
include:
- db: postgresql
postgresql-version: '9.6'
services:
postgres:
image: postgres:13
image: postgres:${{ matrix.postgresql-version }}
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue
+5 -5
View File
@@ -250,9 +250,6 @@ export default {
// logic from legacy code
return !!(this.contains_file_or_folder && Galaxy.user);
},
allDatasets: function () {
return this.folderContents.filter((element) => element.type === "file");
},
},
methods: {
updateSearch: function (value) {
@@ -271,7 +268,11 @@ export default {
async getSelected() {
if (this.isAllSelectedMode) {
this.$emit("setBusy", true);
const selected = await this.services.getFilteredFolderContents(this.folder_id, this.unselected);
const selected = await this.services.getFilteredFolderContents(
this.folder_id,
this.unselected,
this.$parent.search_text
);
this.$emit("setBusy", false);
return selected;
} else {
@@ -319,7 +320,6 @@ export default {
if (isCollection) {
new mod_import_collection.ImportCollectionModal({
selected: checkedItems,
allDatasets: this.allDatasets,
});
} else {
new mod_import_dataset.ImportDatasetModal({
client/src/components/Libraries/LibraryFolder/TopToolbar/add-datasets.js
+20
View File
@@ -710,6 +710,26 @@ var AddDatasets = Backbone.View.extend({
}
},
/**
* User clicked the checkbox in the table heading
* @param {context} event
*/
selectAll: function (event) {
var selected = event.target.checked;
var self = this;
// Iterate each checkbox
$(":checkbox", "#dataset_list tbody").each(function () {
this.checked = selected;
var $row = $(this).closest("tr");
// Change color of selected/unselected
if (selected) {
self.makeDarkRow($row);
} else {
self.makeWhiteRow($row);
}
});
},
makeDarkRow: function ($row) {
$row.addClass("table-primary");
},
client/src/components/Libraries/LibraryFolder/TopToolbar/import-to-history/import-collection.js
+41 -60
View File
@@ -42,36 +42,40 @@ var ImportCollectionModal = Backbone.View.extend({
showCollectionSelect: function (e) {
const Galaxy = getGalaxyInstance();
var checked_items = this.findCheckedItems();
var template = this.templateCollectionSelectModal();
if (checked_items.length === 0) {
Toast.info("You must select some datasets first.");
} else {
var template = this.templateCollectionSelectModal();
var promise = this.fetchUserHistories();
promise
.done(() => {
this.modal = Galaxy.modal;
this.modal.show({
closing_events: true,
title: "Create History Collection from Datasets",
body: template({
selected_datasets: checked_items.dataset_ids.length,
histories: this.histories.models,
}),
buttons: {
Continue: () => {
this.showCollectionBuilder(checked_items.dataset_ids);
var promise = this.fetchUserHistories();
promise
.done(() => {
this.modal = Galaxy.modal;
this.modal.show({
closing_events: true,
title: "Create History Collection from Datasets",
body: template({
selected_datasets: checked_items.dataset_ids.length,
histories: this.histories.models,
}),
buttons: {
Continue: () => {
this.showCollectionBuilder(checked_items.dataset_ids);
},
Close: () => {
Galaxy.modal.hide();
},
},
Close: () => {
Galaxy.modal.hide();
},
},
});
})
.fail((model, response) => {
if (typeof response.responseJSON !== "undefined") {
Toast.error(response.responseJSON.err_msg);
} else {
Toast.error("An error occurred.");
}
});
})
.fail((model, response) => {
if (typeof response.responseJSON !== "undefined") {
Toast.error(response.responseJSON.err_msg);
} else {
Toast.error("An error occurred.");
}
});
}
},
/**
* Note: The collection creation process expects ldda_ids as ids
@@ -79,21 +83,17 @@ var ImportCollectionModal = Backbone.View.extend({
* The code below overwrites the id with ldda_id for this reason.
*/
showCollectionBuilder: function (checked_items) {
let collection_elements = [];
const elements_source = this.modal.$('input[type="radio"]:checked').val();
if (elements_source === "selection") {
for (let i = checked_items.length - 1; i >= 0; i--) {
const collection_item = {};
const dataset = checked_items[i];
collection_item.id = dataset.ldda_id;
collection_item.name = dataset.name;
collection_item.deleted = dataset.deleted;
collection_item.state = dataset.state;
collection_elements.push(collection_item);
}
} else if (elements_source === "folder") {
collection_elements = this.options.allDatasets;
const collection_elements = [];
for (let i = checked_items.length - 1; i >= 0; i--) {
const collection_item = {};
const dataset = checked_items[i];
collection_item.id = dataset.ldda_id;
collection_item.name = dataset.name;
collection_item.deleted = dataset.deleted;
collection_item.state = dataset.state;
collection_elements.push(collection_item);
}
const new_history_name = this.modal.$("input[name=history_name]").val();
if (new_history_name !== "") {
this.createNewHistory(new_history_name)
@@ -198,25 +198,6 @@ var ImportCollectionModal = Backbone.View.extend({
templateCollectionSelectModal: function () {
return _.template(
`<div> <!-- elements selection -->
<div class="library-modal-item">
<h4>Which datasets?</h4>
<form class="form-inline">
<div class="form-check form-check-inline">
<input class="form-check-input" type="radio" name="radio_elements" id="selection_radio" value="selection"
<% if (!selected_datasets) { %> disabled <% } else { %> checked <% } %> />
<label class="form-check-label" for="selection_radio">
current selection
<% if (selected_datasets) { %>
(<%- selected_datasets %>)
<% } %>
</label>
</div>
<div class="form-check form-check-inline">
<input class="form-check-input" type="radio" name="radio_elements" id="folder_radio" value="folder" <% if (!selected_datasets) { %> checked <% } %> >
<label class="form-check-label" for="folder_radio">all datasets in current folder</label>
</div>
</form>
</div>
<!-- type selection -->
<div class="library-modal-item">
<h4>Collection type</h4>
client/src/components/Libraries/LibraryFolder/services.js
+11 -4
View File
@@ -8,8 +8,11 @@ export class Services {
}
async getFolderContents(id, include_deleted, limit, offset, search_text = false) {
search_text = search_text ? `&search_text=${encodeURI(search_text.trim())}` : "";
const url = `${this.root}api/folders/${id}/contents?include_deleted=${include_deleted}&limit=${limit}&offset=${offset}${search_text}`;
const url = `${
this.root
}api/folders/${id}/contents?include_deleted=${include_deleted}&limit=${limit}&offset=${offset}${this.getSearchQuery(
search_text
)}`;
try {
const response = await axios.get(url);
return response.data;
@@ -18,13 +21,17 @@ export class Services {
}
}
async getFilteredFolderContents(id, excluded) {
const contents = await axios.get(`${this.root}api/folders/${id}/contents`);
async getFilteredFolderContents(id, excluded, search_text) {
const contents = await axios.get(`${this.root}api/folders/${id}/contents?${this.getSearchQuery(search_text)}`);
return contents.data.folder_contents.filter((item) => {
return !excluded.some((exc) => exc.id === item.id);
});
}
getSearchQuery(search_text) {
return search_text ? `&search_text=${encodeURI(search_text.trim())}` : "";
}
updateFolder(item, onSucess, onError) {
const url = `${this.root}api/folders/${item.id}`;
try {
lib/galaxy/managers/markdown_parse.py
+1 -1
View File
@@ -33,9 +33,9 @@ VALID_ARGUMENTS = {
"tool_stdout": ["step", "job_id"],
"generate_galaxy_version": [],
"generate_time": [],
"invocation_time": ["invocation_id"],
"visualization": DYNAMIC_ARGUMENTS,
# Invocation Flavored Markdown
"invocation_time": ["invocation_id"],
"invocation_outputs": [],
"invocation_inputs": [],
}
lib/galaxy/managers/markdown_util.py
+2 -2
View File
@@ -685,8 +685,8 @@ history_dataset_collection_display(input={})
return (f"workflow_display(workflow_id={invocation.workflow.stored_workflow.id})\n", False)
if container == "history_link":
return (f"history_link(history_id={invocation.history.id})\n", False)
if container == "invocation_date":
return (f"invocation_date(invocation_id={invocation.id})\n", False)
if container == "invocation_time":
return (f"invocation_time(invocation_id={invocation.id})\n", False)
ref_object_type = None
output_match = re.search(OUTPUT_LABEL_PATTERN, line)
input_match = re.search(INPUT_LABEL_PATTERN, line)
lib/galaxy/metadata/set_metadata.py
+30 -16
View File
@@ -139,6 +139,7 @@ def set_metadata_portable():
version_string = ""
export_store = None
final_job_state = Job.states.OK
if extended_metadata_collection:
tool_dict = metadata_params["tool"]
stdio_exit_code_dicts, stdio_regex_dicts = tool_dict["stdio_exit_codes"], tool_dict["stdio_regexes"]
@@ -198,7 +199,7 @@ def set_metadata_portable():
if os.path.exists(COMMAND_VERSION_FILENAME):
version_string = open(COMMAND_VERSION_FILENAME).read()
job_context = ExpressionContext(dict(stdout=tool_stdout, stderr=tool_stderr))
expression_context = ExpressionContext(dict(stdout=tool_stdout, stderr=tool_stderr))
# Load outputs.
export_store = store.DirectoryModelExportStore('metadata/outputs_populated', serialize_dataset_objects=True, for_edit=True, strip_metadata_files=False, serialize_jobs=False)
@@ -208,6 +209,27 @@ def set_metadata_portable():
# Remove in 21.09, this should only happen for jobs that started on <= 20.09 and finish now
import_model_store = None
job_context = SessionlessJobContext(
metadata_params,
tool_provided_metadata,
object_store,
export_store,
import_model_store,
os.path.join(tool_job_working_directory, "working"),
final_job_state=final_job_state,
)
unnamed_id_to_path = {}
for unnamed_output_dict in job_context.tool_provided_metadata.get_unnamed_outputs():
destination = unnamed_output_dict["destination"]
elements = unnamed_output_dict["elements"]
destination_type = destination["type"]
if destination_type == 'hdas':
for element in elements:
filename = element.get('filename')
if filename:
unnamed_id_to_path[element['object_id']] = os.path.join(job_context.job_working_directory, filename)
for output_name, output_dict in outputs.items():
dataset_instance_id = output_dict["id"]
klass = getattr(galaxy.model, output_dict.get('model_class', 'HistoryDatasetAssociation'))
@@ -232,7 +254,7 @@ def set_metadata_portable():
# Same block as below...
set_meta_kwds = stringify_dictionary_keys(json.load(open(filename_kwds))) # load kwds; need to ensure our keywords are not unicode
try:
dataset.dataset.external_filename = dataset_filename_override
dataset.dataset.external_filename = unnamed_id_to_path.get(dataset_instance_id, dataset_filename_override)
store_by = output_dict.get("object_store_store_by", legacy_object_store_store_by)
extra_files_dir_name = f"dataset_{getattr(dataset.dataset, store_by)}_files"
files_path = os.path.abspath(os.path.join(tool_job_working_directory, "working", extra_files_dir_name))
@@ -248,14 +270,17 @@ def set_metadata_portable():
setattr(dataset.metadata, metadata_name, metadata_file_override)
if output_dict.get("validate", False):
set_validated_state(dataset)
set_meta(dataset, file_dict)
if dataset_instance_id not in unnamed_id_to_path:
# We're going to run through set_metadata in collect_dynamic_outputs with more contextual metadata,
# so skip set_meta here.
set_meta(dataset, file_dict)
if extended_metadata_collection:
meta = tool_provided_metadata.get_dataset_meta(output_name, dataset.dataset.id, dataset.dataset.uuid)
if meta:
context = ExpressionContext(meta, job_context)
context = ExpressionContext(meta, expression_context)
else:
context = job_context
context = expression_context
# Lazy and unattached
# if getattr(dataset, "hidden_beneath_collection_instance", None):
@@ -314,17 +339,6 @@ def set_metadata_portable():
if extended_metadata_collection:
# discover extra outputs...
job_context = SessionlessJobContext(
metadata_params,
tool_provided_metadata,
object_store,
export_store,
import_model_store,
os.path.join(tool_job_working_directory, "working"),
final_job_state=final_job_state,
)
output_collections = {}
for name, output_collection in metadata_params["output_collections"].items():
output_collections[name] = import_model_store.sa_session.query(HistoryDatasetCollectionAssociation).find(output_collection["id"])
lib/galaxy/model/migrate/triggers/update_audit_table.py
+9 -5
View File
@@ -83,7 +83,7 @@ def _postgres_install(engine):
$BODY$
"""
def statement_trigger_def(source_table, id_field, operation, when="AFTER"):
def statement_trigger_def(source_table, id_field, operation, when="AFTER", function_keyword="FUNCTION"):
fn = f"{fn_prefix}_{id_field}"
# Postgres supports many triggers per operation/table so the label can
@@ -95,10 +95,10 @@ def _postgres_install(engine):
CREATE TRIGGER {trigger_name}
{when} {operation} ON {source_table}
REFERENCING NEW TABLE AS new_table
FOR EACH STATEMENT EXECUTE FUNCTION {fn}();
FOR EACH STATEMENT EXECUTE {function_keyword} {fn}();
"""
def row_trigger_def(source_table, id_field, operation, when="AFTER"):
def row_trigger_def(source_table, id_field, operation, when="AFTER", function_keyword="FUNCTION"):
fn = f"{fn_prefix}_{id_field}"
label = f"history_audit_by_{id_field}"
@@ -107,20 +107,24 @@ def _postgres_install(engine):
return f"""
CREATE TRIGGER {trigger_name}
{when} {operation} ON {source_table}
FOR EACH ROW EXECUTE FUNCTION {fn}();
FOR EACH ROW EXECUTE {function_keyword} {fn}();
"""
# pick row or statement triggers depending on postgres version
version = engine.dialect.server_version_info[0]
trigger_fn = statement_trigger_fn if version > 10 else row_trigger_fn
trigger_def = statement_trigger_def if version > 10 else row_trigger_def
# In the syntax of CREATE TRIGGER, the keywords FUNCTION and PROCEDURE are equivalent,
# but the referenced function must in any case be a function, not a procedure.
# The use of the keyword PROCEDURE here is historical and deprecated (https://www.postgresql.org/docs/11/sql-createtrigger.html).
function_keyword = "FUNCTION" if version > 10 else "PROCEDURE"
for id_field in ["history_id", "id"]:
sql.append(trigger_fn(id_field))
for source_table, id_field in trigger_config.items():
for operation in ["UPDATE", "INSERT"]:
sql.append(trigger_def(source_table, id_field, operation))
sql.append(trigger_def(source_table, id_field, operation, function_keyword=function_keyword))
return sql
lib/galaxy/tool_util/verify/__init__.py
+2 -2
View File
@@ -184,12 +184,12 @@ def _bam_to_sam(local_name, temp_name):
temp_local = tempfile.NamedTemporaryFile(suffix='.sam', prefix='local_bam_converted_to_sam_')
with tempfile.NamedTemporaryFile(suffix='.sam', prefix='history_bam_converted_to_sam_', delete=False) as temp:
try:
pysam.view('-h', f'-o{temp_local.name}', local_name)
pysam.view('-h', '--no-PG', '-o', temp_local.name, local_name, catch_stdout=False)
except Exception as e:
msg = f"Converting local (test-data) BAM to SAM failed: {unicodify(e)}"
raise Exception(msg)
try:
pysam.view('-h', f'-o{temp.name}', temp_name)
pysam.view('-h', '--no-PG', '-o', temp.name, temp_name, catch_stdout=False)
except Exception as e:
msg = f"Converting history BAM to SAM failed: {unicodify(e)}"
raise Exception(msg)
lib/galaxy_test/api/test_tools_upload.py
+22
View File
@@ -322,6 +322,28 @@ class ToolsUploadTestCase(ApiTestCase):
assert output0["state"] == "ok"
assert output1["state"] == "error"
@uses_test_history(require_new=False)
def test_fetch_bam_file_from_url_with_extension_set(self, history_id):
destination = {"type": "hdas"}
targets = [{
"destination": destination,
"items": [
{
"src": "url",
"url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bam",
"ext": "bam"
},
]
}]
payload = {
"history_id": history_id,
"targets": json.dumps(targets),
}
fetch_response = self.dataset_populator.fetch(payload)
self._assert_status_code_is(fetch_response, 200)
outputs = fetch_response.json()["outputs"]
self.dataset_populator.get_history_dataset_details(history_id, dataset=outputs[0], assert_ok=True)
@skip_without_datatype("velvet")
def test_composite_datatype(self):
with self.dataset_populator.test_history() as history_id:
test/functional/tools/sam_to_bam.xml
+1 -1
View File
@@ -11,7 +11,7 @@ cat '$input1' > '$out_file1'
<tests>
<test>
<param name="input1" value="sam_with_header.sam" ftype="sam"/>
<output name="out_file1" file="bam_from_sam.bam"/>
<output name="out_file1" file="bam_from_sam.bam" ftype="bam"/>
</test>
</tests>
<help>
test/unit/workflows/test_workflow_markdown.py
+13
View File
@@ -53,6 +53,18 @@ history_dataset_peek(input=input1)
assert "```galaxy\nhistory_dataset_peek(history_dataset_id=567)\n```" in galaxy_markdown
def test_invocation_time():
workflow_markdown = """
And outputs...
```galaxy
invocation_time()
```
"""
galaxy_markdown = resolved_markdown(workflow_markdown)
assert "```galaxy\ninvocation_time(invocation_id=44)\n```" in galaxy_markdown
def test_output_reference_mapping():
workflow_markdown = """
And outputs...
@@ -78,6 +90,7 @@ def example_invocation(trans):
invocation = model.WorkflowInvocation()
workflow = yaml_to_model(TEST_WORKFLOW_YAML)
workflow.id = 342
invocation.id = 44
invocation.workflow = workflow
# TODO: fix this to use workflow id and eliminate hack.