From dc74249d10c2ae2badea6b068877e150c906a61c Mon Sep 17 00:00:00 2001 From: vahid Date: Tue, 19 Sep 2017 17:05:37 -0700 Subject: [PATCH 0001/1016] Added checksum column to dataset table and required migration script. --- lib/galaxy/model/mapping.py | 3 +- ...36_add_checksum_column_to_dataset_table.py | 37 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 lib/galaxy/model/migrate/versions/0136_add_checksum_column_to_dataset_table.py diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index d4f0e0c245e..f70834478e9 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -161,7 +161,8 @@ model.Dataset.table = Table( Column("_extra_files_path", TEXT), Column('file_size', Numeric(15, 0)), Column('total_size', Numeric(15, 0)), - Column('uuid', UUIDType())) + Column('uuid', UUIDType()), + Column('checksum', TEXT, index=True)) # hda read access permission given by a user to a specific site (gen. for external display applications) model.HistoryDatasetAssociationDisplayAtAuthorization.table = Table( diff --git a/lib/galaxy/model/migrate/versions/0136_add_checksum_column_to_dataset_table.py b/lib/galaxy/model/migrate/versions/0136_add_checksum_column_to_dataset_table.py new file mode 100644 index 00000000000..e3592c91570 --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0136_add_checksum_column_to_dataset_table.py @@ -0,0 +1,37 @@ +""" +Migration script to add 'checksum' column to the dataset table. +""" +from __future__ import print_function + +import logging + +from sqlalchemy import Column, MetaData, Table, TEXT + +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) +checksum_column = Column("checksum", TEXT) + + +def upgrade(migrate_engine): + print(__doc__) + metadata = MetaData() + metadata.bind = migrate_engine + metadata.reflect() + try: + dataset_table = Table("dataset", metadata, autoload=True) + checksum_column.create(dataset_table) + assert checksum_column is dataset_table.c.checksum + except Exception: + log.exception("Adding `checksum` column to the `dataset` table failed.") + + +def downgrade(migrate_engine): + metadata = MetaData() + metadata.bind = migrate_engine + metadata.reflect() + try: + dataset_table = Table("dataset", metadata, autoload=True) + dataset_table.c.checksum.drop() + log.debug("Dropped `checksum` column from the `dataset` table.") + except Exception: + log.exception("Dropping `checksum` column from the `dataset` table failed.") From 5821f2cc7253e4781107de23b044802a4551a19a Mon Sep 17 00:00:00 2001 From: vahid Date: Tue, 19 Sep 2017 20:47:28 -0700 Subject: [PATCH 0002/1016] Removed the attribute set to index the checksum column. --- lib/galaxy/model/mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index f70834478e9..f35d4363719 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -162,7 +162,7 @@ model.Dataset.table = Table( Column('file_size', Numeric(15, 0)), Column('total_size', Numeric(15, 0)), Column('uuid', UUIDType()), - Column('checksum', TEXT, index=True)) + Column('checksum', TEXT)) # hda read access permission given by a user to a specific site (gen. for external display applications) model.HistoryDatasetAssociationDisplayAtAuthorization.table = Table( From 5dcb26955d90612279d3cd99a5e6305a0296e52b Mon Sep 17 00:00:00 2001 From: vahid Date: Tue, 19 Sep 2017 23:14:24 -0700 Subject: [PATCH 0003/1016] Calculated & stored a md5-based checksum for a locally uploaded dataset. --- lib/galaxy/model/__init__.py | 8 +++++--- lib/galaxy/tools/actions/upload_common.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 5f60a75dd4d..b79fc279d73 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1756,7 +1756,8 @@ class Dataset(StorableObject): object_store = None # This get initialized in mapping.py (method init) by app.py engine = None - def __init__(self, id=None, state=None, external_filename=None, extra_files_path=None, file_size=None, purgable=True, uuid=None): + def __init__(self, id=None, state=None, external_filename=None, extra_files_path=None, file_size=None, + purgable=True, uuid=None, checksum=None): super(Dataset, self).__init__(id=id) self.state = state self.deleted = False @@ -1770,6 +1771,7 @@ class Dataset(StorableObject): self.uuid = uuid4() else: self.uuid = UUID(str(uuid)) + self.checksum = checksum def in_ready_state(self): return self.state in self.ready_states @@ -1918,7 +1920,7 @@ class DatasetInstance(object): permitted_actions = Dataset.permitted_actions def __init__(self, id=None, hid=None, name=None, info=None, blurb=None, peek=None, tool_version=None, extension=None, - dbkey=None, metadata=None, history=None, dataset=None, deleted=False, designation=None, + dbkey=None, metadata=None, history=None, dataset=None, deleted=False, designation=None, checksum=None, parent_id=None, validation_errors=None, visible=True, create_dataset=False, sa_session=None, extended_metadata=None, flush=True): self.name = name or "Unnamed dataset" @@ -1940,7 +1942,7 @@ class DatasetInstance(object): # Relationships if not dataset and create_dataset: # Had to pass the sqlalchemy session in order to create a new dataset - dataset = Dataset(state=Dataset.states.NEW) + dataset = Dataset(state=Dataset.states.NEW, checksum=checksum) if flush: sa_session.add(dataset) sa_session.flush() diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index 09d6e988a99..a5f4f20bbb5 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -5,6 +5,7 @@ import shlex import socket import subprocess import tempfile +import hashlib from cgi import FieldStorage from json import dumps @@ -220,6 +221,7 @@ def __new_history_upload(trans, uploaded_dataset, history=None, state=None): hda = trans.app.model.HistoryDatasetAssociation(name=uploaded_dataset.name, extension=uploaded_dataset.file_type, dbkey=uploaded_dataset.dbkey, + checksum=uploaded_dataset.checksum, history=history, create_dataset=True, sa_session=trans.sa_session) @@ -337,6 +339,7 @@ def get_uploaded_datasets(trans, cntrller, params, precreated_datasets, dataset_ uploaded_datasets.extend(dataset_upload_input.get_uploaded_datasets(trans, params)) for uploaded_dataset in uploaded_datasets: data = get_precreated_dataset(precreated_datasets, uploaded_dataset.name) + uploaded_dataset.checksum = __get_checksum(uploaded_dataset.path) if not data: data = new_upload(trans, cntrller, uploaded_dataset, library_bunch=library_bunch, history=history) else: @@ -374,6 +377,22 @@ def get_uploaded_datasets(trans, cntrller, params, precreated_datasets, dataset_ return uploaded_datasets +def __get_checksum(filename): + """ + This function calculates an md5-based checksum for a given file. + It reads small chunks of the file in memory because a file could + potentially be large, and reading as a whole to memory could be + suboptimal or not possible due to memory limit. + :param filename: absolute path of a file + :return: md5-based checksum + """ + md5 = hashlib.md5() + with open(filename, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + md5.update(chunk) + return md5.hexdigest() + + def create_paramfile(trans, uploaded_datasets): """ Create the upload tool's JSON "param" file. From 6460f6fde2a9eaf8c9ca21bdd65e3f2d5c490a31 Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 20 Sep 2017 10:07:47 -0700 Subject: [PATCH 0004/1016] Updated a comment. --- lib/galaxy/tools/actions/upload_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index a5f4f20bbb5..c5c463305a1 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -383,7 +383,7 @@ def __get_checksum(filename): It reads small chunks of the file in memory because a file could potentially be large, and reading as a whole to memory could be suboptimal or not possible due to memory limit. - :param filename: absolute path of a file + :param filename: absolute path to a file :return: md5-based checksum """ md5 = hashlib.md5() From 58c2ce6b5b7441e85bc056b159478a933a515f47 Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 20 Sep 2017 11:04:17 -0700 Subject: [PATCH 0005/1016] Sorted imports. --- lib/galaxy/tools/actions/upload_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index c5c463305a1..f89795dd523 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -1,3 +1,4 @@ +import hashlib import ipaddress import logging import os @@ -5,7 +6,6 @@ import shlex import socket import subprocess import tempfile -import hashlib from cgi import FieldStorage from json import dumps From b6f2748b6c3f48761397af722df635f293755b96 Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 20 Sep 2017 17:39:54 -0700 Subject: [PATCH 0006/1016] Updated migration script to create a checksum table. --- ...36_add_checksum_column_to_dataset_table.py | 37 ------------------ .../versions/0136_add_checksum_table.py | 39 +++++++++++++++++++ 2 files changed, 39 insertions(+), 37 deletions(-) delete mode 100644 lib/galaxy/model/migrate/versions/0136_add_checksum_column_to_dataset_table.py create mode 100644 lib/galaxy/model/migrate/versions/0136_add_checksum_table.py diff --git a/lib/galaxy/model/migrate/versions/0136_add_checksum_column_to_dataset_table.py b/lib/galaxy/model/migrate/versions/0136_add_checksum_column_to_dataset_table.py deleted file mode 100644 index e3592c91570..00000000000 --- a/lib/galaxy/model/migrate/versions/0136_add_checksum_column_to_dataset_table.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -Migration script to add 'checksum' column to the dataset table. -""" -from __future__ import print_function - -import logging - -from sqlalchemy import Column, MetaData, Table, TEXT - -log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) -checksum_column = Column("checksum", TEXT) - - -def upgrade(migrate_engine): - print(__doc__) - metadata = MetaData() - metadata.bind = migrate_engine - metadata.reflect() - try: - dataset_table = Table("dataset", metadata, autoload=True) - checksum_column.create(dataset_table) - assert checksum_column is dataset_table.c.checksum - except Exception: - log.exception("Adding `checksum` column to the `dataset` table failed.") - - -def downgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - metadata.reflect() - try: - dataset_table = Table("dataset", metadata, autoload=True) - dataset_table.c.checksum.drop() - log.debug("Dropped `checksum` column from the `dataset` table.") - except Exception: - log.exception("Dropping `checksum` column from the `dataset` table failed.") diff --git a/lib/galaxy/model/migrate/versions/0136_add_checksum_table.py b/lib/galaxy/model/migrate/versions/0136_add_checksum_table.py new file mode 100644 index 00000000000..cd860cfb7b3 --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0136_add_checksum_table.py @@ -0,0 +1,39 @@ +""" +Migration script to add 'dataset_checksum' table. +""" +from __future__ import print_function + +import logging + +from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table, TEXT + +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) + +metadata = MetaData() + +checksum_table = Table("dataset_checksum", metadata, + Column("id", Integer, primary_key=True), + Column("dataset_id", Integer, ForeignKey("dataset.id"), index=True), + Column("hash_function", TEXT), + Column("hash_value", TEXT)) + + +def upgrade(migrate_engine): + print(__doc__) + metadata.bind = migrate_engine + metadata.reflect() + try: + checksum_table.create() + except Exception: + log.exception("Adding `dataset_checksum` table failed.") + + +def downgrade(migrate_engine): + metadata.bind = migrate_engine + metadata.reflect() + try: + checksum_table.drop() + log.debug("Dropped `dataset_checksum` table.") + except Exception: + log.exception("Dropping `dataset_checksum` table failed.") From f58127a6db0a35a353f71df170bad8859b87f974 Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 20 Sep 2017 17:40:46 -0700 Subject: [PATCH 0007/1016] Updated mapping to create a checksum table. --- lib/galaxy/model/mapping.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index f35d4363719..41f9938a90a 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -161,8 +161,15 @@ model.Dataset.table = Table( Column("_extra_files_path", TEXT), Column('file_size', Numeric(15, 0)), Column('total_size', Numeric(15, 0)), - Column('uuid', UUIDType()), - Column('checksum', TEXT)) + Column('uuid', UUIDType())) + +model.DatasetChecksum.table = Table( + "dataset_checksum", metadata, + Column("id", Integer, primary_key=True), + Column("dataset_id", Integer, ForeignKey("dataset.id"), index=True), + Column("hash_function", TEXT), + Column("hash_value", TEXT) +) # hda read access permission given by a user to a specific site (gen. for external display applications) model.HistoryDatasetAssociationDisplayAtAuthorization.table = Table( @@ -1661,6 +1668,10 @@ simple_mapping(model.Dataset, backref='datasets') ) +mapper(model.DatasetChecksum, model.DatasetChecksum.table, properties=dict( + user=relation(model.Dataset) +)) + mapper(model.HistoryDatasetAssociationDisplayAtAuthorization, model.HistoryDatasetAssociationDisplayAtAuthorization.table, properties=dict( history_dataset_association=relation(model.HistoryDatasetAssociation), user=relation(model.User) From 4dd6b7c5f56c8a9ad66f3150cb385f017065d596 Mon Sep 17 00:00:00 2001 From: vahid Date: Mon, 25 Sep 2017 12:55:20 -0700 Subject: [PATCH 0008/1016] Removed previous method for computing/uploading checksum. --- lib/galaxy/model/__init__.py | 7 +++---- lib/galaxy/tools/actions/upload_common.py | 19 ------------------- 2 files changed, 3 insertions(+), 23 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index b79fc279d73..8293fd6ac50 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1757,7 +1757,7 @@ class Dataset(StorableObject): engine = None def __init__(self, id=None, state=None, external_filename=None, extra_files_path=None, file_size=None, - purgable=True, uuid=None, checksum=None): + purgable=True, uuid=None): super(Dataset, self).__init__(id=id) self.state = state self.deleted = False @@ -1771,7 +1771,6 @@ class Dataset(StorableObject): self.uuid = uuid4() else: self.uuid = UUID(str(uuid)) - self.checksum = checksum def in_ready_state(self): return self.state in self.ready_states @@ -1920,7 +1919,7 @@ class DatasetInstance(object): permitted_actions = Dataset.permitted_actions def __init__(self, id=None, hid=None, name=None, info=None, blurb=None, peek=None, tool_version=None, extension=None, - dbkey=None, metadata=None, history=None, dataset=None, deleted=False, designation=None, checksum=None, + dbkey=None, metadata=None, history=None, dataset=None, deleted=False, designation=None, parent_id=None, validation_errors=None, visible=True, create_dataset=False, sa_session=None, extended_metadata=None, flush=True): self.name = name or "Unnamed dataset" @@ -1942,7 +1941,7 @@ class DatasetInstance(object): # Relationships if not dataset and create_dataset: # Had to pass the sqlalchemy session in order to create a new dataset - dataset = Dataset(state=Dataset.states.NEW, checksum=checksum) + dataset = Dataset(state=Dataset.states.NEW) if flush: sa_session.add(dataset) sa_session.flush() diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index f89795dd523..09d6e988a99 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -1,4 +1,3 @@ -import hashlib import ipaddress import logging import os @@ -221,7 +220,6 @@ def __new_history_upload(trans, uploaded_dataset, history=None, state=None): hda = trans.app.model.HistoryDatasetAssociation(name=uploaded_dataset.name, extension=uploaded_dataset.file_type, dbkey=uploaded_dataset.dbkey, - checksum=uploaded_dataset.checksum, history=history, create_dataset=True, sa_session=trans.sa_session) @@ -339,7 +337,6 @@ def get_uploaded_datasets(trans, cntrller, params, precreated_datasets, dataset_ uploaded_datasets.extend(dataset_upload_input.get_uploaded_datasets(trans, params)) for uploaded_dataset in uploaded_datasets: data = get_precreated_dataset(precreated_datasets, uploaded_dataset.name) - uploaded_dataset.checksum = __get_checksum(uploaded_dataset.path) if not data: data = new_upload(trans, cntrller, uploaded_dataset, library_bunch=library_bunch, history=history) else: @@ -377,22 +374,6 @@ def get_uploaded_datasets(trans, cntrller, params, precreated_datasets, dataset_ return uploaded_datasets -def __get_checksum(filename): - """ - This function calculates an md5-based checksum for a given file. - It reads small chunks of the file in memory because a file could - potentially be large, and reading as a whole to memory could be - suboptimal or not possible due to memory limit. - :param filename: absolute path to a file - :return: md5-based checksum - """ - md5 = hashlib.md5() - with open(filename, "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - md5.update(chunk) - return md5.hexdigest() - - def create_paramfile(trans, uploaded_datasets): """ Create the upload tool's JSON "param" file. From f736e7b0e653518df018f8a585388bae4283ebea Mon Sep 17 00:00:00 2001 From: vahid Date: Mon, 25 Sep 2017 12:59:25 -0700 Subject: [PATCH 0009/1016] Added checksum class, and functions in dataset to compute/set checksum. --- lib/galaxy/model/__init__.py | 96 ++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 8293fd6ac50..b446522ba02 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6,6 +6,7 @@ the relationship cardinalities are obvious (e.g. prefer Dataset to Data) """ import codecs import errno +import hashlib import json import logging import numbers @@ -1911,6 +1912,101 @@ class Dataset(StorableObject): return True return False + def set_checksum(self, trans, hash_function, hash_value): + """ + Sets the checksum of this dataset. + :param trans: Galaxy trans. + :param hash_function: Sets the hash function used + to compute the given hash value. + :param hash_value: Sets the hash value + :return: void + """ + checksum = DatasetChecksum(self.id, hash_function, hash_value) + trans.sa_session.add(checksum) + trans.sa_session.flush() + + def compute_checksum(self, trans, hash_function, filename): + """ + Computes a checksum value for this dataset. + :param trans: Galaxy trans. + :param hash_function: Sets the hash function to be + used for computing the hash value. Value should be + a key from `DatasetChecksum.hash_functions` dictionary. + :param filename: absolute path to the dataset file. + :return: void + """ + checksum = DatasetChecksum(self.id) + checksum.compute_checksum(hash_function, filename) + trans.sa_session.add(checksum) + trans.sa_session.flush() + + +class DatasetChecksum(object): + hash_functions = Bunch(MD5='md5', + SHA1='sha-1', + SHA256='SHA-256', + SHA512='SHA-512') + + _buffer_size = 4096 # in bytes + + def __init__(self, + dataset_id, + hash_function=None, + hash_value=None): + self.dataset_id = dataset_id + if hash_function is not None and hash_function not in self.hash_functions.values(): + raise ValueError('Invalid hash function received. Hash function should match "hash_functions"') + self.hash_function = hash_function + self.hash_value = hash_value + + def set_checksum(self, hash_function, hash_value): + """ + Sets the checksum of the dataset referenced by + `dataset_id`. This function should be used when + the dataset checksum is computed by a third-party. + :param hash_function: is the hash function + which is used to compute the hash value. Its + value should match a value from `hash_functions`. + :param hash_value: is a string (hex) hash value. + :return: void + """ + if hash_function not in self.hash_functions.values(): + raise ValueError('Invalid hash function received. Hash function should match "hash_functions"') + self.hash_function = hash_function + self.hash_value = hash_value + + def compute_checksum(self, hash_function, filename): + """ + Computes a checksum value for the dataset referenced + by `dataset_id` based on the chosen hash function. + :param hash_function: Sets the hash function to be used + for computing the checksum. Its value should be a value + from `hash_functions` dictionary. + :param filename: Sets the absolute path to the dataset file. + :return: void + """ + if hash_function not in self.hash_functions.values(): + raise ValueError('Invalid hash function received. Hash function should match "hash_functions"') + self.hash_function = hash_function + self.hash_value = self.__compute_checksum(filename) + + def __compute_checksum(self, filename): + if self.hash_function == self.hash_functions.MD5: + hf = hashlib.md5() + elif self.hash_function == self.hash_functions.SHA1: + hf = hashlib.sha1() + elif self.hash_function == self.hash_functions.SHA256: + hf = hashlib.sha256() + elif self.hash_function == self.hash_functions.SHA512: + hf = hashlib.sha512() + else: + raise TypeError("Invalid checksum hash function") + + with open(filename, "rb") as f: + for chunk in iter(lambda: f.read(self._buffer_size), b""): + hf.update(chunk) + return hf.hexdigest() + class DatasetInstance(object): """A base class for all 'dataset instances', HDAs, LDAs, etc""" From 62b0d05bc58bd56d675bad1b911c84443e90c53e Mon Sep 17 00:00:00 2001 From: vahid Date: Mon, 25 Sep 2017 13:01:09 -0700 Subject: [PATCH 0010/1016] Removed a line break unnecessary for the checksum functionality. --- lib/galaxy/model/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index b446522ba02..5e0a02296f2 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1757,8 +1757,7 @@ class Dataset(StorableObject): object_store = None # This get initialized in mapping.py (method init) by app.py engine = None - def __init__(self, id=None, state=None, external_filename=None, extra_files_path=None, file_size=None, - purgable=True, uuid=None): + def __init__(self, id=None, state=None, external_filename=None, extra_files_path=None, file_size=None, purgable=True, uuid=None): super(Dataset, self).__init__(id=id) self.state = state self.deleted = False From eb00a443a8b3ad2067db56d2dcbc8bdd7c7ab26b Mon Sep 17 00:00:00 2001 From: Dave B Date: Wed, 29 Aug 2018 10:16:40 -0400 Subject: [PATCH 0011/1016] Fix pagination of repositories in category. --- .../scripts/mvc/toolshed/repositories-view.js | 49 +++++++++++++------ lib/tool_shed/util/metadata_util.py | 5 +- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/client/galaxy/scripts/mvc/toolshed/repositories-view.js b/client/galaxy/scripts/mvc/toolshed/repositories-view.js index ef92e7751cc..7f90321fe60 100644 --- a/client/galaxy/scripts/mvc/toolshed/repositories-view.js +++ b/client/galaxy/scripts/mvc/toolshed/repositories-view.js @@ -30,19 +30,29 @@ var ToolShedCategoryContentsView = Backbone.View.extend({ direction: {owner: 'asc', description: 'asc', name: 'asc'}}; sorting.class[this.params.sort_key] = (this.params.sort_order == 'desc' ? 'fa-sort-down' : 'fa-sort-up'); sorting.direction[this.params.sort_key] = (this.params.sort_order == 'asc' ? 'desc' : 'asc'); - console.log(this.model.models[0]); var current_page = this.params.page; var previous_page = Math.max(this.params.page - 1, 1); var pages = parseInt((this.model.models[0].get('repository_count')) % 25) + 1; var next_page = Math.min(parseInt(this.params.page) + 1, pages); + var pagination_params = { + page: this.params.page, + next: next_page, + previous: previous_page, + pages: pages}; + if (pages > 5) { + var page = parseInt(this.params.page); + var page_slice = Array(); + var slice_start = Math.max(page - 2, 2); + var i = slice_start; + var slice_end = Math.min(Math.min(page + 2, slice_start + 4), pages); + for (i = slice_start; i <= slice_end; i++) { + page_slice.push(i); + } + pagination_params['page_slice'] = page_slice; + } this.$el.html( category_contents_template({ - page_navigation: page_navigation_template({ - page: this.params.page, - next: next_page, - previous: previous_page, - pages: pages - }), + page_navigation: page_navigation_template(pagination_params), category: this.model.models[0], tool_shed: this.model.tool_shed, queue: toolshed_util.queueLength(), @@ -109,15 +119,26 @@ var ToolShedCategoryContentsView = Backbone.View.extend({ '', '<% } %>', '<% if (pages > 5) { %>', - '<%= page %>', + '<% if (page != 1) { %>', + '1', + '<% if (page != 2) { %>', '…', - '<%= page %>', - '<%= page %>', - '<%= page %>', - '<%= page %>', - '<%= page %>', + '<% } %>', + '<% } else { %>', + '1', + '<% } %>', + '<% _.each(page_slice, function(i) { %>', + '<% if (i == page) { %>', + '<%= i %>', + '<% } else { %>', + '<%= i %>', + '<% } %>', + '<% }); %>', + '<% var last_pages = [pages - 2, pages - 1, pages]; %>', + '<% if (last_pages.indexOf(parseInt(page)) == -1) { %>', '…', - '<%= page %>', + '<%= pages %>', + '<% } %>', '<% } else { %>', '<% for (i = 1; i <= pages; i++) { %>', '<% if (i == page) { %>', diff --git a/lib/tool_shed/util/metadata_util.py b/lib/tool_shed/util/metadata_util.py index 622cf1fc728..9e10a51cc03 100644 --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -229,9 +229,8 @@ def get_repository_metadata_by_changeset_revision(app, id, changeset_revision): if len(all_metadata_records) > 1: # Delete all records older than the last one updated. for repository_metadata in all_metadata_records[1:]: - print(repository_metadata) - # sa_session.delete(repository_metadata) - # sa_session.flush() + sa_session.delete(repository_metadata) + sa_session.flush() return all_metadata_records[0] elif all_metadata_records: return all_metadata_records[0] From 2bf70a0693fe3640bf4eaae51ee864b5fc1145bc Mon Sep 17 00:00:00 2001 From: Dave B Date: Wed, 29 Aug 2018 14:38:46 -0400 Subject: [PATCH 0012/1016] Fix sorting. --- client/galaxy/scripts/mvc/toolshed/repositories-view.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/galaxy/scripts/mvc/toolshed/repositories-view.js b/client/galaxy/scripts/mvc/toolshed/repositories-view.js index 7f90321fe60..3dcd4d2173f 100644 --- a/client/galaxy/scripts/mvc/toolshed/repositories-view.js +++ b/client/galaxy/scripts/mvc/toolshed/repositories-view.js @@ -16,7 +16,7 @@ var ToolShedCategoryContentsView = Backbone.View.extend({ this.model = new toolshed_model.CategoryCollection(); this.listenTo(this.model, "sync", this.render); var shed = params.tool_shed.replace(/\//g, "%2f"); - this.model.url += `?tool_shed_url=${shed}&category_id=${params.category_id}&sort_key=${params.sort_key}&sort=${params.sort_order}&page=${params.page}`; + this.model.url += `?tool_shed_url=${shed}&category_id=${params.category_id}&sort_key=${params.sort_key}&sort_order=${params.sort_order}&page=${params.page}`; this.model.tool_shed = shed; this.model.category = params.category_id; this.model.fetch(); From 7003b3c155ecff6644204d22c22cf97023a7eaab Mon Sep 17 00:00:00 2001 From: Dave B Date: Wed, 29 Aug 2018 14:51:28 -0400 Subject: [PATCH 0013/1016] Fix linting. --- .../migrate/versions/0026_add_numeric_revision_column.py | 2 +- lib/tool_shed/util/repository_util.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/webapps/tool_shed/model/migrate/versions/0026_add_numeric_revision_column.py b/lib/galaxy/webapps/tool_shed/model/migrate/versions/0026_add_numeric_revision_column.py index 05478f0e1f9..f9839b9ae66 100644 --- a/lib/galaxy/webapps/tool_shed/model/migrate/versions/0026_add_numeric_revision_column.py +++ b/lib/galaxy/webapps/tool_shed/model/migrate/versions/0026_add_numeric_revision_column.py @@ -4,7 +4,7 @@ from __future__ import print_function import logging import sys -from sqlalchemy import Integer, Column, MetaData, Table +from sqlalchemy import Column, Integer, MetaData, Table log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) diff --git a/lib/tool_shed/util/repository_util.py b/lib/tool_shed/util/repository_util.py index e7a461aad84..98f508fd6e7 100644 --- a/lib/tool_shed/util/repository_util.py +++ b/lib/tool_shed/util/repository_util.py @@ -1,4 +1,3 @@ -import time import logging import os import re @@ -496,6 +495,7 @@ def get_repo_info_tuple_contents(repo_info_tuple): description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = repo_info_tuple return description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies + def get_repositories_by_category(app, category_id, installable=False, sort_order='asc', sort_key='name', page=None, per_page=25): sa_session = app.model.context.current query = sa_session.query(app.model.Repository) \ @@ -521,12 +521,9 @@ def get_repositories_by_category(app, category_id, installable=False, sort_order repository_dict = repository.to_dict(value_mapper=default_value_mapper) repository_dict['metadata'] = {} for changeset, changehash in repository.installable_revisions(app): - starttime = time.time() encoded_id = app.security.encode_id(repository.id) metadata = metadata_util.get_repository_metadata_by_changeset_revision(app, encoded_id, changehash) repository_dict['metadata']['%s:%s' % (changeset, changehash)] = metadata.to_dict(value_mapper=default_value_mapper) - finish = time.time() - duration = finish - starttime if installable: if len(repository.installable_revisions(app)): repositories.append(repository_dict) From 845b26194fc995908f961d3883c69ed825a9baf4 Mon Sep 17 00:00:00 2001 From: Dave B Date: Wed, 29 Aug 2018 15:45:00 -0400 Subject: [PATCH 0014/1016] Update docstring. --- lib/galaxy/webapps/galaxy/api/toolshed.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/api/toolshed.py b/lib/galaxy/webapps/galaxy/api/toolshed.py index 89458d845a2..05eae89abef 100644 --- a/lib/galaxy/webapps/galaxy/api/toolshed.py +++ b/lib/galaxy/webapps/galaxy/api/toolshed.py @@ -187,6 +187,9 @@ class ToolShedController(BaseAPIController): :param tool_shed_url: the url of the toolshed to get repositories from :param category_id: the category to get repositories from + :param sort_key: the field by which the repositories should be sorted + :param sort_order: ascending or descending sort + :param page: the page number to return """ sort_order = kwd.get('sort_order', 'asc') sort_key = kwd.get('sort_key', 'name') From be703fe44fa75c16323d359b39eee78173714dfa Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 6 Dec 2018 15:49:00 -0500 Subject: [PATCH 0015/1016] Fix up workflow output logic -- using_workflow_outputs should only count 'tool' steps. Additionally, remove 'output' stars/PJAs from *all* step types when rectifying. --- .../scripts/mvc/workflow/workflow-manager.js | 70 +++++++++---------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/client/galaxy/scripts/mvc/workflow/workflow-manager.js b/client/galaxy/scripts/mvc/workflow/workflow-manager.js index 826caf8f78e..97fab0fded8 100644 --- a/client/galaxy/scripts/mvc/workflow/workflow-manager.js +++ b/client/galaxy/scripts/mvc/workflow/workflow-manager.js @@ -92,10 +92,10 @@ class Workflow { var using_workflow_outputs = false; var has_existing_pjas = false; $.each(this.nodes, (k, node) => { - if (node.workflow_outputs && node.workflow_outputs.length > 0) { + if (node.type === "tool" && node.workflow_outputs && node.workflow_outputs.length > 0) { using_workflow_outputs = true; } - $.each(node.post_job_actions, (pja_id, pja) => { + $.each(node.post_job_actions, (pja_using_workflow_outputsid, pja) => { if (pja.action_type === "HideDatasetAction") { has_existing_pjas = true; } @@ -104,43 +104,41 @@ class Workflow { if (using_workflow_outputs !== false || has_existing_pjas !== false) { // Using workflow outputs, or has existing pjas. Remove all PJAs and recreate based on outputs. $.each(this.nodes, (k, node) => { - if (node.type === "tool") { - var node_changed = false; - if (node.post_job_actions === null) { - node.post_job_actions = {}; - node_changed = true; + var node_changed = false; + if (node.post_job_actions === null) { + node.post_job_actions = {}; + node_changed = true; + } + var pjas_to_rem = []; + $.each(node.post_job_actions, (pja_id, pja) => { + if (pja.action_type == "HideDatasetAction") { + pjas_to_rem.push(pja_id); } - var pjas_to_rem = []; - $.each(node.post_job_actions, (pja_id, pja) => { - if (pja.action_type == "HideDatasetAction") { - pjas_to_rem.push(pja_id); + }); + if (pjas_to_rem.length > 0) { + $.each(pjas_to_rem, (i, pja_name) => { + node_changed = true; + delete node.post_job_actions[pja_name]; + }); + } + if (using_workflow_outputs) { + $.each(node.output_terminals, (ot_id, ot) => { + var create_pja = !node.isWorkflowOutput(ot.name); + if (create_pja === true) { + node_changed = true; + var pja = { + action_type: "HideDatasetAction", + output_name: ot.name, + action_arguments: {} + }; + node.post_job_actions[`HideDatasetAction${ot.name}`] = null; + node.post_job_actions[`HideDatasetAction${ot.name}`] = pja; } }); - if (pjas_to_rem.length > 0) { - $.each(pjas_to_rem, (i, pja_name) => { - node_changed = true; - delete node.post_job_actions[pja_name]; - }); - } - if (using_workflow_outputs) { - $.each(node.output_terminals, (ot_id, ot) => { - var create_pja = !node.isWorkflowOutput(ot.name); - if (create_pja === true) { - node_changed = true; - var pja = { - action_type: "HideDatasetAction", - output_name: ot.name, - action_arguments: {} - }; - node.post_job_actions[`HideDatasetAction${ot.name}`] = null; - node.post_job_actions[`HideDatasetAction${ot.name}`] = pja; - } - }); - } - // lastly, if this is the active node, and we made changes, reload the display at right. - if (this.active_node == node && node_changed === true) { - this.reload_active_node(); - } + } + // lastly, if this is the active node, and we made changes, reload the display at right. + if (this.active_node == node && node_changed === true) { + this.reload_active_node(); } }); } From c3af6488fa9d11d302db05a215faa153adcad291 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 12 Dec 2018 12:41:20 -0500 Subject: [PATCH 0016/1016] fix typo, client format --- client/galaxy/scripts/layout/menu.js | 6 +++--- client/galaxy/scripts/mvc/workflow/workflow-manager.js | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/galaxy/scripts/layout/menu.js b/client/galaxy/scripts/layout/menu.js index e8c22996f8c..2dc1b410985 100644 --- a/client/galaxy/scripts/layout/menu.js +++ b/client/galaxy/scripts/layout/menu.js @@ -221,9 +221,9 @@ var Collection = Backbone.Collection.extend({ } if (options.helpsite_url) { helpTab.menu.unshift({ - title: _l("Galaxy Help"), - url: options.helpsite_url, - target: "_blank" + title: _l("Galaxy Help"), + url: options.helpsite_url, + target: "_blank" }); } this.add(helpTab); diff --git a/client/galaxy/scripts/mvc/workflow/workflow-manager.js b/client/galaxy/scripts/mvc/workflow/workflow-manager.js index 97fab0fded8..2fe71ead7c3 100644 --- a/client/galaxy/scripts/mvc/workflow/workflow-manager.js +++ b/client/galaxy/scripts/mvc/workflow/workflow-manager.js @@ -95,7 +95,7 @@ class Workflow { if (node.type === "tool" && node.workflow_outputs && node.workflow_outputs.length > 0) { using_workflow_outputs = true; } - $.each(node.post_job_actions, (pja_using_workflow_outputsid, pja) => { + $.each(node.post_job_actions, (pja_id, pja) => { if (pja.action_type === "HideDatasetAction") { has_existing_pjas = true; } From 782b18c5acc708e2b9aeb878282dded29f870f7a Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Mon, 14 Jan 2019 11:01:04 -0500 Subject: [PATCH 0017/1016] Set newly added nodes as workflow outputs. --- client/galaxy/scripts/mvc/workflow/workflow-view.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/client/galaxy/scripts/mvc/workflow/workflow-view.js b/client/galaxy/scripts/mvc/workflow/workflow-view.js index f06a78db5ed..1bb90ada625 100644 --- a/client/galaxy/scripts/mvc/workflow/workflow-view.js +++ b/client/galaxy/scripts/mvc/workflow/workflow-view.js @@ -673,6 +673,16 @@ export default Backbone.View.extend({ success: function(data) { node.init_field_data(data); node.update_field_data(data); + // Post init/update, for new modules we want to default to + // nodes being outputs + // TODO: Overhaul the handling of all this when we modernize + // the editor, replace callout image manipulation with a simple + // class toggle, etc. + $.each(node.output_terminals, (ot_id, ot) => { + node.addWorkflowOutput(ot.name); + var callout = $(node.element).find(`.callout.${ot.name.replace(/(?=[()])/g, "\\")}`); + callout.find("img").attr("src", `${Galaxy.root}static/images/fugue/asterisk-small.png`); + }); self.workflow.activate_node(node); } }); From b6d5675f5da1e91aece2df4af5d037072e897185 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 15 Jan 2019 12:21:30 +0100 Subject: [PATCH 0018/1016] Update galaxy.yml.sample for prefix config --- config/galaxy.yml.sample | 13 +++++++++++-- config/reports.yml.sample | 8 ++++++++ config/tool_shed.yml.sample | 8 ++++++++ doc/source/admin/galaxy_options.rst | 5 +++-- lib/galaxy/webapps/config_manage.py | 10 ++++++++++ lib/galaxy/webapps/galaxy/config_schema.yml | 4 ++-- 6 files changed, 42 insertions(+), 6 deletions(-) diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample index f2f30ce7bee..593d6179faf 100644 --- a/config/galaxy.yml.sample +++ b/config/galaxy.yml.sample @@ -66,6 +66,14 @@ uwsgi: # Reports, etc.) that you are loading. module: galaxy.webapps.galaxy.buildapp:uwsgi_app() + # Mount the web application (e.g. Galaxy, Reports, etc.) at the given + # prefix. Cannot be used together with 'module:' above. + #mount: /galaxy=galaxy.webapps.galaxy.buildapp:uwsgi_app() + + # Make uWSGI rewrite PATH_INFO and SCRIPT_NAME according to mount- + # points. Set this to true if a prefix is used. + manage-script-name: false + # It is usually a good idea to set this to ``true`` if processes is # greater than 1. thunder-lock: false @@ -91,8 +99,9 @@ uwsgi: galaxy: # If running behind a proxy server and Galaxy is served from a - # subdirectory, enable the proxy-prefix filter and set the prefix in - # the [filter:proxy-prefix] section above. + # subdirectory, enable the proxy-prefix filter, uncomment the 'mount: + # /galaxy=galaxy.webapps.galaxy.buildapp:uwsgi_app()' line and enable + # `manage-script-name` in the uwsgi config section. #filter-with: proxy-prefix # If proxy-prefix is enabled and you're running more than one Galaxy diff --git a/config/reports.yml.sample b/config/reports.yml.sample index aa6fc445e21..3f426e10135 100644 --- a/config/reports.yml.sample +++ b/config/reports.yml.sample @@ -48,6 +48,14 @@ uwsgi: # Reports, etc.) that you are loading. module: galaxy.webapps.reports.buildapp:uwsgi_app() + # Mount the web application (e.g. Galaxy, Reports, etc.) at the given + # prefix. Cannot be used together with 'module:' above. + #mount: /galaxy=galaxy.webapps.galaxy.buildapp:uwsgi_app() + + # Make uWSGI rewrite PATH_INFO and SCRIPT_NAME according to mount- + # points. Set this to true if a prefix is used. + manage-script-name: false + # It is usually a good idea to set this to ``true`` if processes is # greater than 1. thunder-lock: false diff --git a/config/tool_shed.yml.sample b/config/tool_shed.yml.sample index ea86936c839..619729600f8 100644 --- a/config/tool_shed.yml.sample +++ b/config/tool_shed.yml.sample @@ -48,6 +48,14 @@ uwsgi: # Reports, etc.) that you are loading. module: galaxy.webapps.tool_shed.buildapp:uwsgi_app() + # Mount the web application (e.g. Galaxy, Reports, etc.) at the given + # prefix. Cannot be used together with 'module:' above. + #mount: /galaxy=galaxy.webapps.galaxy.buildapp:uwsgi_app() + + # Make uWSGI rewrite PATH_INFO and SCRIPT_NAME according to mount- + # points. Set this to true if a prefix is used. + manage-script-name: false + # It is usually a good idea to set this to ``true`` if processes is # greater than 1. thunder-lock: false diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index c9a2b47dc69..45230831bbc 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -4,8 +4,9 @@ :Description: If running behind a proxy server and Galaxy is served from a - subdirectory, enable the proxy-prefix filter and set the prefix in - the [filter:proxy-prefix] section above. + subdirectory, enable the proxy-prefix filter, uncomment the + 'mount: /galaxy=galaxy.webapps.galaxy.buildapp:uwsgi_app()' line + and enable `manage-script-name` in the uwsgi config section. :Default: ``proxy-prefix`` :Type: str diff --git a/lib/galaxy/webapps/config_manage.py b/lib/galaxy/webapps/config_manage.py index 3bd5dbc3dbe..be7f5e4a075 100644 --- a/lib/galaxy/webapps/config_manage.py +++ b/lib/galaxy/webapps/config_manage.py @@ -111,6 +111,16 @@ UWSGI_OPTIONS = OrderedDict([ 'default': '$uwsgi_module', 'type': 'str', }), + ('#mount', { + 'desc': """Mount the web application (e.g. Galaxy, Reports, etc.) at the given prefix. Cannot be used together with 'module:' above.""", + 'default': '/galaxy=galaxy.webapps.galaxy.buildapp:uwsgi_app()', + 'type': 'str', + }), + ('manage-script-name', { + 'desc': """Make uWSGI rewrite PATH_INFO and SCRIPT_NAME according to mount-points. Set this to true if a prefix is used.""", + 'default': False, + 'type': 'bool', + }), ('thunder-lock', { 'desc': """It is usually a good idea to set this to ``true`` if processes is greater than 1.""", 'default': False, diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index ad44ea8d7c5..873e00bd0cc 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -40,8 +40,8 @@ mapping: required: false desc: | If running behind a proxy server and Galaxy is served from a subdirectory, - enable the proxy-prefix filter and set the prefix in the - [filter:proxy-prefix] section above. + enable the proxy-prefix filter, uncomment the 'mount: /galaxy=galaxy.webapps.galaxy.buildapp:uwsgi_app()' + line and enable `manage-script-name` in the uwsgi config section. cookie_path: type: str From 30d10985433035d062ada03beb960e63c03031dc Mon Sep 17 00:00:00 2001 From: Dave B Date: Mon, 21 Jan 2019 10:05:04 -0500 Subject: [PATCH 0019/1016] Add converters between tabular and CSV. --- config/datatypes_conf.xml.sample | 14 ++++-- .../datatypes/converters/csv_to_tabular.xml | 12 +++++ .../datatypes/converters/tabular_csv.py | 44 +++++++++++++++++++ .../datatypes/converters/tabular_to_csv.xml | 12 +++++ 4 files changed, 78 insertions(+), 4 deletions(-) create mode 100644 lib/galaxy/datatypes/converters/csv_to_tabular.xml create mode 100644 lib/galaxy/datatypes/converters/tabular_csv.py create mode 100644 lib/galaxy/datatypes/converters/tabular_to_csv.xml diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 70ab3228653..8ca2e47859d 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -72,8 +72,12 @@ - - + + + + + + @@ -96,7 +100,7 @@ - + @@ -304,7 +308,9 @@ - + + + diff --git a/lib/galaxy/datatypes/converters/csv_to_tabular.xml b/lib/galaxy/datatypes/converters/csv_to_tabular.xml new file mode 100644 index 00000000000..4a00b67e406 --- /dev/null +++ b/lib/galaxy/datatypes/converters/csv_to_tabular.xml @@ -0,0 +1,12 @@ + + + python '$__tool_directory__/tabular_csv.py' -o '$tabular' -i '$csv' + + + + + + + + + diff --git a/lib/galaxy/datatypes/converters/tabular_csv.py b/lib/galaxy/datatypes/converters/tabular_csv.py new file mode 100644 index 00000000000..53b3148f5a8 --- /dev/null +++ b/lib/galaxy/datatypes/converters/tabular_csv.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +""" +Uses the python csv library to convert to and from tabular + +usage: %prog [--from-tabular] -i in_file -o out_file +""" + +import argparse +import os.path + +import csv + + +def main(): + usage = "Usage: %prog [options]" + parser = argparse.ArgumentParser(usage=usage) + parser.add_argument('-f', '--from-tabular', action='store_true', default=False, dest='fromtabular') + parser.add_argument('-i', '--input', type=str, dest='input') + parser.add_argument('-o', '--output', type=str, dest='output') + args = parser.parse_args() + input_fname = args.input + output_fname = args.output + if args.fromtabular: + convert_to_csv(input_fname, output_fname) + else: + convert_to_tsv(input_fname, output_fname) + +def convert_to_tsv(input_fname, output_fname): + with open(input_fname, 'rb') as csvfile: + with open(output_fname, 'wb') as ofh: + reader = csv.reader(csvfile) + for line in reader: + ofh.write('\t'.join(line) + '\n') + +def convert_to_csv(input_fname, output_fname): + with open(input_fname, 'rb') as tabfile: + with open(output_fname, 'wb') as ofh: + writer = csv.writer(ofh, delimiter=',') + for line in tabfile.readlines(): + writer.writerow(line.strip().split('\t')) + + +if __name__ == "__main__": + main() diff --git a/lib/galaxy/datatypes/converters/tabular_to_csv.xml b/lib/galaxy/datatypes/converters/tabular_to_csv.xml new file mode 100644 index 00000000000..2c853109184 --- /dev/null +++ b/lib/galaxy/datatypes/converters/tabular_to_csv.xml @@ -0,0 +1,12 @@ + + + python '$__tool_directory__/tabular_csv.py' --from-tabular -i '$tabular' -o '$csv' + + + + + + + + + From f0ac15266aac6f8c3adcfce38e8aa49208e6ae87 Mon Sep 17 00:00:00 2001 From: Dave B Date: Mon, 21 Jan 2019 10:33:31 -0500 Subject: [PATCH 0020/1016] Fix imports, E302s --- lib/galaxy/datatypes/converters/tabular_csv.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/datatypes/converters/tabular_csv.py b/lib/galaxy/datatypes/converters/tabular_csv.py index 53b3148f5a8..0b18e1571e5 100644 --- a/lib/galaxy/datatypes/converters/tabular_csv.py +++ b/lib/galaxy/datatypes/converters/tabular_csv.py @@ -6,8 +6,6 @@ usage: %prog [--from-tabular] -i in_file -o out_file """ import argparse -import os.path - import csv @@ -25,6 +23,7 @@ def main(): else: convert_to_tsv(input_fname, output_fname) + def convert_to_tsv(input_fname, output_fname): with open(input_fname, 'rb') as csvfile: with open(output_fname, 'wb') as ofh: @@ -32,6 +31,7 @@ def convert_to_tsv(input_fname, output_fname): for line in reader: ofh.write('\t'.join(line) + '\n') + def convert_to_csv(input_fname, output_fname): with open(input_fname, 'rb') as tabfile: with open(output_fname, 'wb') as ofh: From 33a74cae2def618e9fe32d1a8b1355d00a39a525 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 22 Jan 2019 15:15:55 -0500 Subject: [PATCH 0021/1016] Fix hotdata method access; this still needs a lot of love/refactoring, but this makes it work again? --- .../components/RuleCollectionBuilder.vue | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/client/galaxy/scripts/components/RuleCollectionBuilder.vue b/client/galaxy/scripts/components/RuleCollectionBuilder.vue index 7136d150a54..6d527df80e3 100644 --- a/client/galaxy/scripts/components/RuleCollectionBuilder.vue +++ b/client/galaxy/scripts/components/RuleCollectionBuilder.vue @@ -1257,8 +1257,9 @@ export default { return targets; }, colHeaders() { - const data = this.hotData["data"]; - const columns = this.hotData["columns"]; + let hotData = this.hotData(); + const data = hotData["data"]; + const columns = hotData["columns"]; return RuleDefs.colHeadersFor(data, columns); }, colHeadersDisplay() { @@ -1684,7 +1685,8 @@ export default { return identifierColumns; }, buildRequestElements(createDatasetDescription, createSubcollectionDescription, subElementProp) { - const data = this.hotData["data"]; + let hotData = this.hotData(); + const data = hotData["data"]; const identifierColumns = this.identifierColumns(); if (identifierColumns.length < 1) { console.log("Error but this shouldn't have happened, create button should have been disabled."); @@ -1791,8 +1793,9 @@ export default { return elementsByName; }, creationElementsFromDatasets() { - const sources = this.hotData["sources"]; - const data = this.hotData["data"]; + let hotData = this.hotData(); + const sources = hotData["sources"]; + const data = hotData["data"]; const mappingAsDict = this.mappingAsDict; const elementsByCollectionName = this.buildRequestElements( @@ -1811,7 +1814,8 @@ export default { }, creationElementsForFetch() { // fetch elements for HDCA - const data = this.hotData["data"]; + let hotData = this.hotData(); + const data = hotData["data"]; const mappingAsDict = this.mappingAsDict; const elementsByCollectionName = this.buildRequestElements( @@ -1830,7 +1834,8 @@ export default { }, creationDatasetsForFetch() { // fetch elements for HDAs if not collection information specified. - const data = this.hotData["data"]; + let hotData = this.hotData(); + const data = hotData["data"]; const mappingAsDict = this.mappingAsDict; const datasets = []; From a64a5cdc5e1ace5981452026c4aa2857fbfd729b Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 22 Jan 2019 17:12:38 -0500 Subject: [PATCH 0022/1016] Prefer const, none of these are reassigned. --- .../scripts/components/RuleCollectionBuilder.vue | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/client/galaxy/scripts/components/RuleCollectionBuilder.vue b/client/galaxy/scripts/components/RuleCollectionBuilder.vue index 6d527df80e3..2d9a8028969 100644 --- a/client/galaxy/scripts/components/RuleCollectionBuilder.vue +++ b/client/galaxy/scripts/components/RuleCollectionBuilder.vue @@ -1257,7 +1257,7 @@ export default { return targets; }, colHeaders() { - let hotData = this.hotData(); + const hotData = this.hotData(); const data = hotData["data"]; const columns = hotData["columns"]; return RuleDefs.colHeadersFor(data, columns); @@ -1685,7 +1685,7 @@ export default { return identifierColumns; }, buildRequestElements(createDatasetDescription, createSubcollectionDescription, subElementProp) { - let hotData = this.hotData(); + const hotData = this.hotData(); const data = hotData["data"]; const identifierColumns = this.identifierColumns(); if (identifierColumns.length < 1) { @@ -1793,7 +1793,7 @@ export default { return elementsByName; }, creationElementsFromDatasets() { - let hotData = this.hotData(); + const hotData = this.hotData(); const sources = hotData["sources"]; const data = hotData["data"]; const mappingAsDict = this.mappingAsDict; @@ -1814,7 +1814,7 @@ export default { }, creationElementsForFetch() { // fetch elements for HDCA - let hotData = this.hotData(); + const hotData = this.hotData(); const data = hotData["data"]; const mappingAsDict = this.mappingAsDict; @@ -1834,7 +1834,7 @@ export default { }, creationDatasetsForFetch() { // fetch elements for HDAs if not collection information specified. - let hotData = this.hotData(); + const hotData = this.hotData(); const data = hotData["data"]; const mappingAsDict = this.mappingAsDict; From 3c9f263c69c657f8a644fceb778a7ff36c888d0a Mon Sep 17 00:00:00 2001 From: Mason Date: Wed, 23 Jan 2019 13:20:29 -0800 Subject: [PATCH 0023/1016] Restored hotData to computed, but removed unnecessary side-effects --- .../components/RuleCollectionBuilder.vue | 95 +++++++++---------- .../scripts/mvc/rules/rule-definitions.js | 6 +- 2 files changed, 48 insertions(+), 53 deletions(-) diff --git a/client/galaxy/scripts/components/RuleCollectionBuilder.vue b/client/galaxy/scripts/components/RuleCollectionBuilder.vue index 2d9a8028969..05a548841fb 100644 --- a/client/galaxy/scripts/components/RuleCollectionBuilder.vue +++ b/client/galaxy/scripts/components/RuleCollectionBuilder.vue @@ -448,7 +448,7 @@ []); + columns = []; + } else if (this.elementsType == "collection_contents") { + const collection = this.initialElements.slice(); + if (collection) { + const obj = this.populateElementsFromCollectionDescription( + collection.elements, + collection.collection_type + ); + data = obj.data; + sources = obj.sources; + columns = []; + } else { + data = []; + sources = []; + columns = []; + } + } else { + data = this.initialElements.slice(); + sources = data.map(el => null); + columns = []; + if (this.initialElements) { + this.initialElements[0].forEach(() => columns.push("new")); + } + } + return RuleDefs.applyRules(data, sources, columns, this.rules); + }, + colHeadersPerRule() { + return this.hotData.colHeadersPerRule; } }, methods: { @@ -1413,42 +1449,6 @@ export default { this.rules.push(rule); } }, - hotData() { - let data, sources, columns; - if ( - this.elementsType == "datasets" || - this.elementsType == "library_datasets" || - this.elementsType == "ftp" - ) { - data = this.initialElements.map(el => []); - sources = this.initialElements.slice(); - columns = []; - } else if (this.elementsType == "collection_contents") { - const collection = this.initialElements; - if (collection) { - const obj = this.populateElementsFromCollectionDescription( - collection.elements, - collection.collection_type - ); - data = obj.data; - sources = obj.sources; - columns = []; - } else { - data = []; - sources = []; - columns = []; - } - } else { - data = this.initialElements.slice(); - sources = data.map(el => null); - columns = []; - if (this.initialElements) { - this.initialElements[0].forEach(() => columns.push("new")); - } - } - this.colHeadersPerRule = []; - return RuleDefs.applyRules(data, sources, columns, this.rules, this.colHeadersPerRule); - }, viewSource() { this.resetSource(); this.ruleView = "source"; @@ -1685,8 +1685,7 @@ export default { return identifierColumns; }, buildRequestElements(createDatasetDescription, createSubcollectionDescription, subElementProp) { - const hotData = this.hotData(); - const data = hotData["data"]; + const data = this.hotData.data; const identifierColumns = this.identifierColumns(); if (identifierColumns.length < 1) { console.log("Error but this shouldn't have happened, create button should have been disabled."); @@ -1793,9 +1792,7 @@ export default { return elementsByName; }, creationElementsFromDatasets() { - const hotData = this.hotData(); - const sources = hotData["sources"]; - const data = hotData["data"]; + const { sources, data } = this.hotData; const mappingAsDict = this.mappingAsDict; const elementsByCollectionName = this.buildRequestElements( @@ -1814,8 +1811,7 @@ export default { }, creationElementsForFetch() { // fetch elements for HDCA - const hotData = this.hotData(); - const data = hotData["data"]; + const data = this.hotData.data; const mappingAsDict = this.mappingAsDict; const elementsByCollectionName = this.buildRequestElements( @@ -1834,8 +1830,7 @@ export default { }, creationDatasetsForFetch() { // fetch elements for HDAs if not collection information specified. - const hotData = this.hotData(); - const data = hotData["data"]; + const data = this.hotData.data; const mappingAsDict = this.mappingAsDict; const datasets = []; diff --git a/client/galaxy/scripts/mvc/rules/rule-definitions.js b/client/galaxy/scripts/mvc/rules/rule-definitions.js index 4b410cb2e55..c0cb365f121 100644 --- a/client/galaxy/scripts/mvc/rules/rule-definitions.js +++ b/client/galaxy/scripts/mvc/rules/rule-definitions.js @@ -861,8 +861,8 @@ const colHeadersFor = function(data, columns) { } }; -const applyRules = function(data, sources, columns, rules, colHeadersPerRule) { - colHeadersPerRule = colHeadersPerRule || []; +const applyRules = function(data, sources, columns, rules, headersPerRule = []) { + let colHeadersPerRule = Array.from(headersPerRule); let hasRuleError = false; for (var ruleIndex in rules) { const ruleHeaders = colHeadersFor(data, columns); @@ -889,7 +889,7 @@ const applyRules = function(data, sources, columns, rules, colHeadersPerRule) { columns = res.columns || columns; } } - return { data, sources, columns }; + return { data, sources, columns, colHeadersPerRule }; }; export default { From 205cdd0954ffdf34efc2c364ec1b62079e4a27d8 Mon Sep 17 00:00:00 2001 From: Mason Date: Wed, 23 Jan 2019 14:38:45 -0800 Subject: [PATCH 0024/1016] Removed slice method on non-array property --- client/galaxy/scripts/components/RuleCollectionBuilder.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/galaxy/scripts/components/RuleCollectionBuilder.vue b/client/galaxy/scripts/components/RuleCollectionBuilder.vue index 05a548841fb..1a83bdbe670 100644 --- a/client/galaxy/scripts/components/RuleCollectionBuilder.vue +++ b/client/galaxy/scripts/components/RuleCollectionBuilder.vue @@ -1385,7 +1385,7 @@ export default { data = sources.map(el => []); columns = []; } else if (this.elementsType == "collection_contents") { - const collection = this.initialElements.slice(); + const collection = this.initialElements; if (collection) { const obj = this.populateElementsFromCollectionDescription( collection.elements, From 569ea9ea36e15afcbd174b50ae2de79029c4bc0a Mon Sep 17 00:00:00 2001 From: Mason Date: Wed, 23 Jan 2019 14:56:53 -0800 Subject: [PATCH 0025/1016] Removed commented code in RuleCollectionBuilder --- client/galaxy/scripts/components/RuleCollectionBuilder.vue | 1 - 1 file changed, 1 deletion(-) diff --git a/client/galaxy/scripts/components/RuleCollectionBuilder.vue b/client/galaxy/scripts/components/RuleCollectionBuilder.vue index 1a83bdbe670..47818d25224 100644 --- a/client/galaxy/scripts/components/RuleCollectionBuilder.vue +++ b/client/galaxy/scripts/components/RuleCollectionBuilder.vue @@ -1029,7 +1029,6 @@ export default { } return { rules: rules, - // colHeadersPerRule: [], mapping: mapping, state: "build", // 'build', 'error', 'wait', ruleView: "normal", // 'normal' or 'source' From 70aba0f1ba84d236124527bd8fa80aba164ba940 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 25 Jan 2019 12:56:49 -0500 Subject: [PATCH 0026/1016] Add a Galaxy IE for cellxgene. --- config/datatypes_conf.xml.sample | 3 +- .../config/allowed_images.yml.sample | 5 ++ .../cellxgene/config/cellxgene.ini.sample | 46 +++++++++++++++++++ .../cellxgene/config/cellxgene.xml | 18 ++++++++ .../cellxgene/docker/Dockerfile | 15 ++++++ .../cellxgene/docker/run_cellxgene.sh | 3 ++ .../cellxgene/templates/cellxgene.mako | 38 +++++++++++++++ lib/galaxy/datatypes/binary.py | 5 ++ 8 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 config/plugins/interactive_environments/cellxgene/config/allowed_images.yml.sample create mode 100644 config/plugins/interactive_environments/cellxgene/config/cellxgene.ini.sample create mode 100644 config/plugins/interactive_environments/cellxgene/config/cellxgene.xml create mode 100644 config/plugins/interactive_environments/cellxgene/docker/Dockerfile create mode 100755 config/plugins/interactive_environments/cellxgene/docker/run_cellxgene.sh create mode 100644 config/plugins/interactive_environments/cellxgene/templates/cellxgene.mako diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 70ab3228653..29cc4b1cdea 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -96,7 +96,7 @@ - + @@ -139,6 +139,7 @@ + diff --git a/config/plugins/interactive_environments/cellxgene/config/allowed_images.yml.sample b/config/plugins/interactive_environments/cellxgene/config/allowed_images.yml.sample new file mode 100644 index 00000000000..0cc66c3d838 --- /dev/null +++ b/config/plugins/interactive_environments/cellxgene/config/allowed_images.yml.sample @@ -0,0 +1,5 @@ +--- +- + image: cellxgene-galaxy-ie + description: | + An interactive data explorer for single-cell transcriptomics datasets diff --git a/config/plugins/interactive_environments/cellxgene/config/cellxgene.ini.sample b/config/plugins/interactive_environments/cellxgene/config/cellxgene.ini.sample new file mode 100644 index 00000000000..9347211cc48 --- /dev/null +++ b/config/plugins/interactive_environments/cellxgene/config/cellxgene.ini.sample @@ -0,0 +1,46 @@ +[main] +# Following options are ignored if using the Galaxy dynamic proxy but +# are useful if mapping a range of ports for environment consumption. +#password_auth = False +#ssl = False + +[docker] +# Command to launch docker container. For example `sudo docker` or `docker-lxc`. +# If you need to use a command like `sg` you can do that here, just be sure to +# wrap all of the docker portion in single quotes. E.g. `sg 'docker' 'docker {docker_args}'` +# +# It is recommended that you use command_inject if you need to inject +# additional parameters. This command string is re-used for a `docker inspect` +# command and will likely cause errors if it is extensively modified, past the +# usual group/sudo changes. +#command = docker {docker_args} + +# The image argument was moved to "allowed_images.yml.sample" + +# Additional arguments that are passed to the `docker run` command. +command_inject = --sig-proxy=true -e DEBUG=false -e KILL_MODE=True -e DEFAULT_CONTAINER_RUNTIME=120 + +# URL to access the Galaxy API with from the spawn Docker containter, if empty +# this falls back to galaxy.ini's galaxy_infrastructure_url and finally to the +# Docker host of the spawned container if that is also not set. +#galaxy_url = + +# The Docker hostname. It can be useful to run the Docker daemon on a different +# host than Galaxy. +#docker_hostname = localhost + +# Try to set the tempdirectory to world execute - this can fix the issue +# where 'sudo docker' is not able to mount the folder otherwise. +# "finalize namespace chdir to /import permission denied" +#wx_tempdir = False + +# Overwride the IE tempdirectory. This can be useful if you regular tempdir is +# located on an NFS share, which does not work well as Docker volume. In this case +# you can have a shared sshfs share which you can use as temporary directory to +# share data between the IE and Galaxy. +#docker_galaxy_temp_dir = None + +# If your Docker container exposes more then one port, Galaxy needs to know to +# which ports it needs to connect. With this option you can specify the port number +# inside your container to which Galaxy should connect. +docker_connect_port = 80 diff --git a/config/plugins/interactive_environments/cellxgene/config/cellxgene.xml b/config/plugins/interactive_environments/cellxgene/config/cellxgene.xml new file mode 100644 index 00000000000..1761bbe71f4 --- /dev/null +++ b/config/plugins/interactive_environments/cellxgene/config/cellxgene.xml @@ -0,0 +1,18 @@ + + + + + + HistoryDatasetAssociation + binary.Anndata + + dataset_id + + + + dataset_id + + cellxgene.mako + diff --git a/config/plugins/interactive_environments/cellxgene/docker/Dockerfile b/config/plugins/interactive_environments/cellxgene/docker/Dockerfile new file mode 100644 index 00000000000..f72207e417d --- /dev/null +++ b/config/plugins/interactive_environments/cellxgene/docker/Dockerfile @@ -0,0 +1,15 @@ +FROM ubuntu:bionic + +ENV LC_ALL=C.UTF-8 +ENV LANG=C.UTF-8 + +RUN apt-get update && \ + apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev + +RUN pip3 install cellxgene + +ADD ./run_cellxgene.sh / + +ENTRYPOINT ["/run_cellxgene.sh"] + +EXPOSE 80 diff --git a/config/plugins/interactive_environments/cellxgene/docker/run_cellxgene.sh b/config/plugins/interactive_environments/cellxgene/docker/run_cellxgene.sh new file mode 100755 index 00000000000..0759da7af8e --- /dev/null +++ b/config/plugins/interactive_environments/cellxgene/docker/run_cellxgene.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +cellxgene launch --host 0.0.0.0 --port 80 /input/file.* diff --git a/config/plugins/interactive_environments/cellxgene/templates/cellxgene.mako b/config/plugins/interactive_environments/cellxgene/templates/cellxgene.mako new file mode 100644 index 00000000000..12ebf7de088 --- /dev/null +++ b/config/plugins/interactive_environments/cellxgene/templates/cellxgene.mako @@ -0,0 +1,38 @@ +<%namespace name="ie" file="ie.mako" /> +<% + ie_request.load_deploy_config() + + # Make the dataset available inside the container + data_file = ie_request.volume('/input/file.' + hda.ext, hda.file_name, mode='ro') + + ie_request.launch( + image = trans.request.params.get('image_tag', None), + volumes = [data_file] + ) + + url = ie_request.url_template('${PROXY_URL}') +%> + + + ${ ie.load_default_js() } + + + +
+
+ + diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index b03e2618c0d..db93474c86d 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -830,6 +830,11 @@ class Loom(H5): except Exception as e: log.warning('%s, set_meta Exception: %s', self, e) +class Anndata(H5): + """ + Class describing an anndata file: http://anndata.rtfd.io + """ + file_ext = 'h5ad' class GmxBinary(Binary): """ From 92cc7afa3400493b2037fc97fa0f413a9344b046 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 25 Jan 2019 14:17:39 -0500 Subject: [PATCH 0027/1016] Add keepalive to cellxgene GIE. This uses the same approach as in ethercalc, so I moved the `keepAlive` method into the `interactive_environments` package for reuse. --- .../galaxy.interactive_environments.js | 36 +++++++++++++++++++ .../cellxgene/templates/cellxgene.mako | 1 + .../ethercalc/static/js/ethercalc.js | 36 ------------------- .../ethercalc/templates/ethercalc.mako | 2 +- 4 files changed, 38 insertions(+), 37 deletions(-) diff --git a/client/galaxy/scripts/galaxy.interactive_environments.js b/client/galaxy/scripts/galaxy.interactive_environments.js index 430bf463a1d..01451e4f924 100644 --- a/client/galaxy/scripts/galaxy.interactive_environments.js +++ b/client/galaxy/scripts/galaxy.interactive_environments.js @@ -210,3 +210,39 @@ export function load_when_ready(url, success_callback) { var spin_state = make_spin_state("IE container readiness"); spin_until(url, true, messages, success_callback, spin_state); } + +/** + * Keep the container alive by pinging `notebookAccessURL` every 10 seconds. + * If the user leaves this site this function is not constantly pinging the + * container, the container will terminate itself. + */ +export function keepAlive(notebookAccessURL){ + + var request_count = 0; + var interval = window.setInterval(function(){ + $.ajax({ + url: notebookAccessURL, + xhrFields: { + withCredentials: true + }, + type: "GET", + timeout: 500, + success: function(){ + console.log("Connected to IE, returning"); + }, + error: function(jqxhr, status, error){ + request_count++; + console.log("Request " + request_count); + if(request_count > 30){ + window.clearInterval(interval); + IES.clear_main_area(); + toastr.error( + "Could not connect to IE, contact your administrator", + "Error", + {'closeButton': true, 'timeOut': 20000, 'tapToDismiss': false} + ); + } + } + }); + }, 10000); +} diff --git a/config/plugins/interactive_environments/cellxgene/templates/cellxgene.mako b/config/plugins/interactive_environments/cellxgene/templates/cellxgene.mako index 12ebf7de088..3d91ae1c9b5 100644 --- a/config/plugins/interactive_environments/cellxgene/templates/cellxgene.mako +++ b/config/plugins/interactive_environments/cellxgene/templates/cellxgene.mako @@ -25,6 +25,7 @@ requirejs(['galaxy.interactive_environments'], function (IES) { $( document ).ready(function() { + IES.keepAlive(url); IES.test_ie_availability(url, function() { IES.append_notebook(url); }); diff --git a/config/plugins/interactive_environments/ethercalc/static/js/ethercalc.js b/config/plugins/interactive_environments/ethercalc/static/js/ethercalc.js index a08f05d5126..bcf564666ae 100644 --- a/config/plugins/interactive_environments/ethercalc/static/js/ethercalc.js +++ b/config/plugins/interactive_environments/ethercalc/static/js/ethercalc.js @@ -7,39 +7,3 @@ function load_notebook(url){ IES.append_notebook(url); }); } - -function keep_alive(notebook_access_url){ - /** - * This is needed to keep the container alive. If the user leaves this site - * this function is not constantly pinging the container, the container will - * terminate itself. - */ - - var request_count = 0; - var interval = window.setInterval(function(){ - $.ajax({ - url: notebook_access_url, - xhrFields: { - withCredentials: true - }, - type: "GET", - timeout: 500, - success: function(){ - console.log("Connected to IE, returning"); - }, - error: function(jqxhr, status, error){ - request_count++; - console.log("Request " + request_count); - if(request_count > 30){ - window.clearInterval(interval); - IES.clear_main_area(); - toastr.error( - "Could not connect to IE, contact your administrator", - "Error", - {'closeButton': true, 'timeOut': 20000, 'tapToDismiss': false} - ); - } - } - }); - }, 10000); -} diff --git a/config/plugins/interactive_environments/ethercalc/templates/ethercalc.mako b/config/plugins/interactive_environments/ethercalc/templates/ethercalc.mako index 28c3fa0d58b..5f2e83bd63a 100644 --- a/config/plugins/interactive_environments/ethercalc/templates/ethercalc.mako +++ b/config/plugins/interactive_environments/ethercalc/templates/ethercalc.mako @@ -28,7 +28,7 @@ ${ ie.plugin_require_config() } // Keep container running and load IE requirejs(['galaxy.interactive_environments', 'plugin/ethercalc'], function(IES){ window.IES = IES; - keep_alive(url); + IES.keepAlive(url); IES.load_when_ready(ie_readiness_url, function(){ load_notebook(url); }); From ca9b182253dd71e138b0f2cb38dc97e0a8c504de Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 25 Jan 2019 14:33:00 -0500 Subject: [PATCH 0028/1016] Reference cellxgene container in quay.io --- .../cellxgene/config/allowed_images.yml.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/plugins/interactive_environments/cellxgene/config/allowed_images.yml.sample b/config/plugins/interactive_environments/cellxgene/config/allowed_images.yml.sample index 0cc66c3d838..8e2f66fe314 100644 --- a/config/plugins/interactive_environments/cellxgene/config/allowed_images.yml.sample +++ b/config/plugins/interactive_environments/cellxgene/config/allowed_images.yml.sample @@ -1,5 +1,5 @@ --- - - image: cellxgene-galaxy-ie + image: quay.io/galaxy/cellxgene-galaxy-ie description: | An interactive data explorer for single-cell transcriptomics datasets From 7f6252dff3bf1bd39e756f60cff888fc8faaa6d0 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 25 Jan 2019 14:43:14 -0500 Subject: [PATCH 0029/1016] Pep 8 fix. --- lib/galaxy/datatypes/binary.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index db93474c86d..af4119daac3 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -830,12 +830,14 @@ class Loom(H5): except Exception as e: log.warning('%s, set_meta Exception: %s', self, e) + class Anndata(H5): """ Class describing an anndata file: http://anndata.rtfd.io """ file_ext = 'h5ad' + class GmxBinary(Binary): """ Base class for GROMACS binary files - xtc, trr, cpt From 9231812833b790ea29d3f7ea9231930f5f609726 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 25 Jan 2019 14:55:57 -0500 Subject: [PATCH 0030/1016] Add sniffer for h5ad, use tagged image --- .../cellxgene/config/allowed_images.yml.sample | 2 +- lib/galaxy/datatypes/binary.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/config/plugins/interactive_environments/cellxgene/config/allowed_images.yml.sample b/config/plugins/interactive_environments/cellxgene/config/allowed_images.yml.sample index 8e2f66fe314..743bf2b3aff 100644 --- a/config/plugins/interactive_environments/cellxgene/config/allowed_images.yml.sample +++ b/config/plugins/interactive_environments/cellxgene/config/allowed_images.yml.sample @@ -1,5 +1,5 @@ --- - - image: quay.io/galaxy/cellxgene-galaxy-ie + image: quay.io/galaxy/cellxgene-galaxy-ie:2019.01.25 description: | An interactive data explorer for single-cell transcriptomics datasets diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index af4119daac3..59f81c196b5 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -830,14 +830,23 @@ class Loom(H5): except Exception as e: log.warning('%s, set_meta Exception: %s', self, e) - + class Anndata(H5): """ Class describing an anndata file: http://anndata.rtfd.io """ file_ext = 'h5ad' - + def sniff(self, filename): + if super(Anndata, self).sniff(filename): + try: + with h5py.File(filename) as anndata_file: + return all( attr in f for attr in ['X', 'obs', 'var', 'uns'] ) + except Exception: + return False + return False + + class GmxBinary(Binary): """ Base class for GROMACS binary files - xtc, trr, cpt From 6096671ef0215310d8c8eb6c7938b3be3cbc7f9e Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 25 Jan 2019 15:20:45 -0500 Subject: [PATCH 0031/1016] Make h5ad sniffer actually work --- config/datatypes_conf.xml.sample | 1 + lib/galaxy/datatypes/binary.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 29cc4b1cdea..b6e90e3788d 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -736,6 +736,7 @@ + diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 59f81c196b5..70d8babf658 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -840,7 +840,7 @@ class Anndata(H5): def sniff(self, filename): if super(Anndata, self).sniff(filename): try: - with h5py.File(filename) as anndata_file: + with h5py.File(filename) as f: return all( attr in f for attr in ['X', 'obs', 'var', 'uns'] ) except Exception: return False From 02be43e84440aae7ac533682fe580c1b9cd1e421 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 25 Jan 2019 16:35:57 -0500 Subject: [PATCH 0032/1016] PEP8 --- lib/galaxy/datatypes/binary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 70d8babf658..d8e65a31344 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -841,7 +841,7 @@ class Anndata(H5): if super(Anndata, self).sniff(filename): try: with h5py.File(filename) as f: - return all( attr in f for attr in ['X', 'obs', 'var', 'uns'] ) + return all(attr in f for attr in ['X', 'obs', 'var', 'uns']) except Exception: return False return False From 83217ea32167c0056248ef4b55ffd5e2b1062a6f Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 25 Jan 2019 17:16:04 -0500 Subject: [PATCH 0033/1016] Fix eslint issue --- client/galaxy/scripts/galaxy.interactive_environments.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/galaxy/scripts/galaxy.interactive_environments.js b/client/galaxy/scripts/galaxy.interactive_environments.js index 01451e4f924..d0632ba3811 100644 --- a/client/galaxy/scripts/galaxy.interactive_environments.js +++ b/client/galaxy/scripts/galaxy.interactive_environments.js @@ -235,7 +235,7 @@ export function keepAlive(notebookAccessURL){ console.log("Request " + request_count); if(request_count > 30){ window.clearInterval(interval); - IES.clear_main_area(); + clear_main_area(); toastr.error( "Could not connect to IE, contact your administrator", "Error", From d1f10c362e5864b0a02fce829075a5ee40a923e4 Mon Sep 17 00:00:00 2001 From: vahid Date: Sat, 26 Jan 2019 18:11:15 -0800 Subject: [PATCH 0034/1016] Parse provider-specific configuration for cloud backend of objectstore. --- lib/galaxy/objectstore/cloud.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 3c8a3a15957..66d52b9ab05 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -100,7 +100,37 @@ class Cloud(ObjectStore, CloudConfigMixin): @classmethod def parse_xml(clazz, config_xml): - return parse_config_xml(config_xml) + # The following reads common cloud-based storage configuration + # as implemented for the S3 backend. Hence, it also attempts to + # parse S3-specific configuration (e.g., credentials); however, + # such provider-specific configuration is overwritten in the + # following. + config = parse_config_xml(config_xml) + + + provider = config_xml.attrib.get("provider").lower() + if provider is None: + raise Exception("Missing `provider` attribute from the Cloud backend of the ObjectStore.") + + # Read any provider-specific configuration. + auth_element = config_xml.findall("auth")[0] + if provider == "aws": + config["auth"] = { + "access_key": auth_element.get("access_key"), + "secret_key": auth_element.get("secret_key")} + elif provider == "azure": + config["auth"] = { + "subscription_id": auth_element.get("subscription_id"), + "client_id": auth_element.get("client_id"), + "secret": auth_element.get("secret"), + "tenant": auth_element.get("tenant")} + elif provider == "google": + config["auth"] = { + "credentials_file": auth_element.get("credentials_file")} + else: + raise Exception("Unsupported provider `{}`.".format(provider)) + + return config def to_dict(self): as_dict = super(Cloud, self).to_dict() From 0510e55da310a16ba74b28cd2cc733bbfb28263b Mon Sep 17 00:00:00 2001 From: vahid Date: Sat, 26 Jan 2019 20:18:54 -0800 Subject: [PATCH 0035/1016] Update cloud backend configuration. --- lib/galaxy/objectstore/cloud.py | 42 +++++++++++++++++---------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 66d52b9ab05..fff9d96beb4 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -47,14 +47,10 @@ class Cloud(ObjectStore, CloudConfigMixin): super(Cloud, self).__init__(config, config_dict) self.transfer_progress = 0 - auth_dict = config_dict['auth'] bucket_dict = config_dict['bucket'] connection_dict = config_dict.get('connection', {}) cache_dict = config_dict['cache'] - self.access_key = auth_dict.get('access_key') - self.secret_key = auth_dict.get('secret_key') - self.bucket = bucket_dict.get('name') self.use_rr = bucket_dict.get('use_reduced_redundancy', False) self.max_chunk_size = bucket_dict.get('max_chunk_size', 250) @@ -68,13 +64,13 @@ class Cloud(ObjectStore, CloudConfigMixin): self.cache_size = cache_dict.get('size', -1) self.staging_path = cache_dict.get('path') or self.config.object_store_cache_path - self._initialize() + self._initialize(config_dict["provider"], config_dict["auth"]) - def _initialize(self): + def _initialize(self, provider, credentials): if CloudProviderFactory is None: raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE) - self._configure_connection() + self.conn = self._get_connection(provider, credentials) self.bucket = self._get_bucket(self.bucket) # Clean cache only if value is set in galaxy.ini if self.cache_size != -1: @@ -92,11 +88,17 @@ class Cloud(ObjectStore, CloudConfigMixin): except OSError: self.use_axel = False - def _configure_connection(self): - log.debug("Configuring AWS-S3 Connection") - aws_config = {'aws_access_key': self.access_key, - 'aws_secret_key': self.secret_key} - self.conn = CloudProviderFactory().create_provider(ProviderList.AWS, aws_config) + @staticmethod + def _get_connection(provider, credentials): + log.debug("Configuring `{}` Connection".format(provider)) + if provider == "aws": + return CloudProviderFactory().create_provider(ProviderList.AWS, credentials) + elif provider == "azure": + return CloudProviderFactory().create_provider(ProviderList.AZURE, credentials) + elif provider == "google": + return CloudProviderFactory().create_provider(ProviderList.GCE, credentials) + else: + raise Exception("Unsupported provider `{}`.".format(provider)) @classmethod def parse_xml(clazz, config_xml): @@ -107,26 +109,26 @@ class Cloud(ObjectStore, CloudConfigMixin): # following. config = parse_config_xml(config_xml) - provider = config_xml.attrib.get("provider").lower() if provider is None: raise Exception("Missing `provider` attribute from the Cloud backend of the ObjectStore.") + config["provider"] = provider # Read any provider-specific configuration. auth_element = config_xml.findall("auth")[0] if provider == "aws": config["auth"] = { - "access_key": auth_element.get("access_key"), - "secret_key": auth_element.get("secret_key")} + "aws_access_key": auth_element.get("access_key"), + "aws_secret_key": auth_element.get("secret_key")} elif provider == "azure": config["auth"] = { - "subscription_id": auth_element.get("subscription_id"), - "client_id": auth_element.get("client_id"), - "secret": auth_element.get("secret"), - "tenant": auth_element.get("tenant")} + "AZURE_SUBSCRIPTION_ID": auth_element.get("subscription_id"), + "AZURE_CLIENT_ID": auth_element.get("client_id"), + "AZURE_SECRET": auth_element.get("secret"), + "AZURE_TENANT": auth_element.get("tenant")} elif provider == "google": config["auth"] = { - "credentials_file": auth_element.get("credentials_file")} + "GCE_SERVICE_CREDS_FILE": auth_element.get("credentials_file")} else: raise Exception("Unsupported provider `{}`.".format(provider)) From 68dd5503b4e8b22313e2cdab139b7b65fdb860c9 Mon Sep 17 00:00:00 2001 From: vahid Date: Sat, 26 Jan 2019 21:41:45 -0800 Subject: [PATCH 0036/1016] Check of all cloud backend config, and raise exception if any missing. --- lib/galaxy/objectstore/cloud.py | 44 +++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index fff9d96beb4..eac667b59ce 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -116,23 +116,50 @@ class Cloud(ObjectStore, CloudConfigMixin): # Read any provider-specific configuration. auth_element = config_xml.findall("auth")[0] + missing_config = [] if provider == "aws": + akey = auth_element.get("access_key") + if akey is None: + missing_config.append("access_key") + skey = auth_element.get("secret_key") + if skey is None: + missing_config.append("secret_key") + config["auth"] = { - "aws_access_key": auth_element.get("access_key"), - "aws_secret_key": auth_element.get("secret_key")} + "aws_access_key": akey, + "aws_secret_key": skey} elif provider == "azure": + sid = auth_element.get("subscription_id") + if sid is None: + missing_config.append("subscription_id") + cid = auth_element.get("client_id") + if cid is None: + missing_config.append("client_id") + sec = auth_element.get("secret") + if sec is None: + missing_config.append("secret") + ten = auth_element.get("tenant") + if ten is None: + missing_config.append("tenant") config["auth"] = { - "AZURE_SUBSCRIPTION_ID": auth_element.get("subscription_id"), - "AZURE_CLIENT_ID": auth_element.get("client_id"), - "AZURE_SECRET": auth_element.get("secret"), - "AZURE_TENANT": auth_element.get("tenant")} + "azure_subscription_id": sid, + "azure_client_id": cid, + "azure_secret": sec, + "azure_tenant": ten} elif provider == "google": + cre = auth_element.get("credentials_file") + if cre is None: + missing_config.append("credentials_file") config["auth"] = { - "GCE_SERVICE_CREDS_FILE": auth_element.get("credentials_file")} + "gce_service_creds_file": cre} else: raise Exception("Unsupported provider `{}`.".format(provider)) - return config + if len(missing_config) > 0: + raise Exception("The following configuration required for {} cloud backend " + "are missing: {}".format(provider, missing_config)) + else: + return config def to_dict(self): as_dict = super(Cloud, self).to_dict() @@ -205,6 +232,7 @@ class Cloud(ObjectStore, CloudConfigMixin): def _get_bucket(self, bucket_name): try: + print '\n\n\n------------------bucket_name:\t', bucket_name bucket = self.conn.storage.buckets.get(bucket_name) if bucket is None: log.debug("Bucket not found, creating a bucket with handle '%s'", bucket_name) From 0fee45fd42028ad920cefdf58fff94861d0ef957 Mon Sep 17 00:00:00 2001 From: vahid Date: Sat, 26 Jan 2019 21:48:17 -0800 Subject: [PATCH 0037/1016] Remove a print statement. --- lib/galaxy/objectstore/cloud.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index eac667b59ce..50e74b81640 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -232,7 +232,6 @@ class Cloud(ObjectStore, CloudConfigMixin): def _get_bucket(self, bucket_name): try: - print '\n\n\n------------------bucket_name:\t', bucket_name bucket = self.conn.storage.buckets.get(bucket_name) if bucket is None: log.debug("Bucket not found, creating a bucket with handle '%s'", bucket_name) From ecb457820fa96880ad4d0f7cf4a7c0dd2f37de16 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Tue, 29 Jan 2019 14:16:49 -0500 Subject: [PATCH 0038/1016] Add sniffer for Anndata --- lib/galaxy/datatypes/binary.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index d8e65a31344..84e0e1d59a3 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -833,7 +833,12 @@ class Loom(H5): class Anndata(H5): """ - Class describing an anndata file: http://anndata.rtfd.io + Class describing an HDF5 anndata files: http://anndata.rtfd.io + >>> from galaxy.datatypes.sniff import get_test_fname + >>> Anndata().sniff(get_test_fname('pbmc3k_tiny.h5ad')) + True + >>> Anndata().sniff(get_test_fname('test.mz5')) + False """ file_ext = 'h5ad' @@ -841,7 +846,7 @@ class Anndata(H5): if super(Anndata, self).sniff(filename): try: with h5py.File(filename) as f: - return all(attr in f for attr in ['X', 'obs', 'var', 'uns']) + return all(attr in f for attr in ['X', 'obs', 'var']) except Exception: return False return False From 591b8fc18294e33e658971d2e0b0862121a9d6c8 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Tue, 29 Jan 2019 15:28:28 -0500 Subject: [PATCH 0039/1016] Modify cellxgene Dockerfile for smaller image. --- .../cellxgene/docker/Dockerfile | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/config/plugins/interactive_environments/cellxgene/docker/Dockerfile b/config/plugins/interactive_environments/cellxgene/docker/Dockerfile index f72207e417d..b172e4a0ec5 100644 --- a/config/plugins/interactive_environments/cellxgene/docker/Dockerfile +++ b/config/plugins/interactive_environments/cellxgene/docker/Dockerfile @@ -1,12 +1,6 @@ -FROM ubuntu:bionic - -ENV LC_ALL=C.UTF-8 -ENV LANG=C.UTF-8 - -RUN apt-get update && \ - apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev - -RUN pip3 install cellxgene +FROM python:3.6.8-slim + +RUN pip3 install cellxgene && rm -rf ~/.cache ADD ./run_cellxgene.sh / From 7a1076e04b24e818959ff274847c011d0cfaf6e2 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Tue, 29 Jan 2019 15:29:03 -0500 Subject: [PATCH 0040/1016] Test dataset for Anndata datatype. --- lib/galaxy/datatypes/test/pbmc3k_tiny.h5ad | Bin 0 -> 4656 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 lib/galaxy/datatypes/test/pbmc3k_tiny.h5ad diff --git a/lib/galaxy/datatypes/test/pbmc3k_tiny.h5ad b/lib/galaxy/datatypes/test/pbmc3k_tiny.h5ad new file mode 100644 index 0000000000000000000000000000000000000000..44d8d853a878fd81b2c45a79b2624c8a1e012e36 GIT binary patch literal 4656 zcmeD5aB<`1lHy_j0S*oZ76t(@6Gr@p0s|q42#gPtPk=HQp>zk7Ucm%mFfxE31A_!q zTo7tLy1I}cS62q0N|^aD8mf)KfCa+RfC-G!BPs+uTpa^I9*%(e+5k$QfTlAj7cQTl zR16j>ODqD@Kz=TW0px>eSb8Oto){U}!08u4fKn(#2O|^21eiIvXl7u_2gwO4$OH2h z2q-X!b1;b4gV_uWtPC7rc_t=^fgI4B!psPi00U^khpHEX3WLntljtAAz>oozp$`@8 z=kLM@b~VI41!nyA!b+10R{ZJ|VA3!RFg_X$D@PWvp-G@|VezUUh$b;exS%iu#i0a< z00Ks6yfZRF1farTH6VFrh7Dl07%;qLfc(t7l++541OXrnGzb)=dGYC~d8x%{r8}l( z0br_wBs>S8p5*+}yb?%80xE@tJ*F;*2(~niJAJc2tO5H5VIq3^#>{^ZSKtn(0!TOk z%LP85v803(C`keRoSKtU49xZT0dhEjRO0rh0@R-%*B}78KN%Pl5R!CcImFuRaddQa z4heAwQb2~gu3-$U$XQV0>JsD_;pxW?bA*<`<7Y_RQ>7csjxI`bU3{6%fwg)<@cuTx&` zD-C$KFZ5#fzVfL;2V$38v)`#Z+pbj2Yyaafv-a!iecrPn$izX2NzXq1le~Rj9`k`P zK1&C#7q9JDw!82D$(6ltS^8=F`vKeR8ra(HBbv;0i*7hMlzi1JSba>=q4B3gv*DZP z`x%z*+vUKa;Lxzjz-rr5`&~*Wxpo#T{@d+N8j@0O|c+bwco&h8y6?DwoY%CTEx zk-+Xr^AA{?FyGy^%SPX(K0;%U+0VR?uy`|7 zclV{)_jdX|+HalTHhm9EkB8;_zXH3vel%O(UDmhfnZEIEmnf0l8s%~}uhL)d{-eEd zx1p4TRYEq?9-TWa)^torqgIWEz-S1JhQMeD45<)+wL=A<4Nk)C&=uhJl^U=e`U2?F zi1>o!M5vFofb7QF4Kdy%Rs(B0Q7bl>FrMkNcaFf1PY4)XL$1&>~3Ph2jm*u X?MX3Ch-*P6App9&fi@!~smKBVevL(H literal 0 HcmV?d00001 From 433fdda5ce06c0cff8feee9e450f39a04f3ceb93 Mon Sep 17 00:00:00 2001 From: vahid Date: Tue, 29 Jan 2019 20:01:55 -0800 Subject: [PATCH 0041/1016] Add Azure and GCE to object store config file. --- config/object_store_conf.xml.sample | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/config/object_store_conf.xml.sample b/config/object_store_conf.xml.sample index 59170e046ca..507121f0557 100644 --- a/config/object_store_conf.xml.sample +++ b/config/object_store_conf.xml.sample @@ -52,8 +52,9 @@ --> - + + + + + + From a676aff6c340a7d32aa0c7de9f9248a1438f292e Mon Sep 17 00:00:00 2001 From: vahid Date: Tue, 29 Jan 2019 20:32:13 -0800 Subject: [PATCH 0042/1016] Add a comment to object store on connection authorization assertion. --- lib/galaxy/objectstore/cloud.py | 34 ++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 50e74b81640..340357b5bd3 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -92,14 +92,42 @@ class Cloud(ObjectStore, CloudConfigMixin): def _get_connection(provider, credentials): log.debug("Configuring `{}` Connection".format(provider)) if provider == "aws": - return CloudProviderFactory().create_provider(ProviderList.AWS, credentials) + connection = CloudProviderFactory().create_provider(ProviderList.AWS, credentials) elif provider == "azure": - return CloudProviderFactory().create_provider(ProviderList.AZURE, credentials) + connection = CloudProviderFactory().create_provider(ProviderList.AZURE, credentials) elif provider == "google": - return CloudProviderFactory().create_provider(ProviderList.GCE, credentials) + connection = CloudProviderFactory().create_provider(ProviderList.GCE, credentials) else: raise Exception("Unsupported provider `{}`.".format(provider)) + # Ideally it would be better to assert if the connection is + # authorized to perform operations required by ObjectStore + # before returning it (and initializing ObjectStore); hence + # any related issues can be handled properly here, and ObjectStore + # can "trust" the connection is established. + # + # However, the mechanism implemented in Cloudbridge to assert if + # a user/service is authorized to perform an operation, assumes + # the user/service is granted with an elevated privileges, such + # as admin/owner-level access to all resources. For a detailed + # discussion see: + # + # https://github.com/CloudVE/cloudbridge/issues/135 + # + # Hence, if a resource owner wants to only authorize Galaxy to r/w + # a bucket/container on the provider, but does not allow it to access + # other resources, Cloudbridge may fail asserting credentials. + # For instance, to r/w an Amazon S3 bucket, the resource owner + # also needs to authorize full access to Amazon EC2, because Cloudbridge + # leverages EC2-specific functions to assert the credentials. + # + # Therefore, to adhere with principle of least privilege, we do not + # assert credentials; instead, we handle exceptions raised as a + # result of signing API calls to cloud provider (e.g., GCE) using + # incorrect, invalid, or unauthorized credentials. + + return connection + @classmethod def parse_xml(clazz, config_xml): # The following reads common cloud-based storage configuration From 875c1faf6f4ac416cb3ef1923f1acbcb46bdc745 Mon Sep 17 00:00:00 2001 From: vahid Date: Tue, 29 Jan 2019 20:46:34 -0800 Subject: [PATCH 0043/1016] Lower provider name after checking if it exists. --- lib/galaxy/objectstore/cloud.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 340357b5bd3..6dc70328005 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -137,9 +137,10 @@ class Cloud(ObjectStore, CloudConfigMixin): # following. config = parse_config_xml(config_xml) - provider = config_xml.attrib.get("provider").lower() + provider = config_xml.attrib.get("provider") if provider is None: raise Exception("Missing `provider` attribute from the Cloud backend of the ObjectStore.") + provider = provider.lower() config["provider"] = provider # Read any provider-specific configuration. From 70820e820f7e042f603846f9c1f8343821adb911 Mon Sep 17 00:00:00 2001 From: vahid Date: Tue, 29 Jan 2019 21:08:40 -0800 Subject: [PATCH 0044/1016] Assert if specified file containing GCE credentials is accessible. --- lib/galaxy/objectstore/cloud.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 6dc70328005..45d6caa6f4a 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -5,6 +5,7 @@ Object Store plugin for Cloud storage. import logging import multiprocessing import os +import os.path import shutil import subprocess import threading @@ -177,6 +178,8 @@ class Cloud(ObjectStore, CloudConfigMixin): "azure_tenant": ten} elif provider == "google": cre = auth_element.get("credentials_file") + if not os.path.isfile(cre): + raise IOError("The following file specified for GCE credentials not found: {}".format(cre)) if cre is None: missing_config.append("credentials_file") config["auth"] = { From 7c1e133013d7c3f246b7dbcc430b72dff84c720f Mon Sep 17 00:00:00 2001 From: vahid Date: Tue, 29 Jan 2019 23:02:08 -0800 Subject: [PATCH 0045/1016] Update exception handling in cloud objectstore. --- lib/galaxy/objectstore/cloud.py | 118 ++++++++++++++++++-------------- 1 file changed, 65 insertions(+), 53 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 45d6caa6f4a..e28499a00a5 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -65,9 +65,9 @@ class Cloud(ObjectStore, CloudConfigMixin): self.cache_size = cache_dict.get('size', -1) self.staging_path = cache_dict.get('path') or self.config.object_store_cache_path - self._initialize(config_dict["provider"], config_dict["auth"]) + self._setup(config_dict["provider"], config_dict["auth"]) - def _initialize(self, provider, credentials): + def _setup(self, provider, credentials): if CloudProviderFactory is None: raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE) @@ -138,60 +138,72 @@ class Cloud(ObjectStore, CloudConfigMixin): # following. config = parse_config_xml(config_xml) - provider = config_xml.attrib.get("provider") - if provider is None: - raise Exception("Missing `provider` attribute from the Cloud backend of the ObjectStore.") - provider = provider.lower() - config["provider"] = provider + try: + provider = config_xml.attrib.get("provider") + if provider is None: + msg = "Missing `provider` attribute from the Cloud backend of the ObjectStore." + log.error(msg) + raise Exception(msg) + provider = provider.lower() + config["provider"] = provider - # Read any provider-specific configuration. - auth_element = config_xml.findall("auth")[0] - missing_config = [] - if provider == "aws": - akey = auth_element.get("access_key") - if akey is None: - missing_config.append("access_key") - skey = auth_element.get("secret_key") - if skey is None: - missing_config.append("secret_key") + # Read any provider-specific configuration. + auth_element = config_xml.findall("auth")[0] + missing_config = [] + if provider == "aws": + akey = auth_element.get("access_key") + if akey is None: + missing_config.append("access_key") + skey = auth_element.get("secret_key") + if skey is None: + missing_config.append("secret_key") - config["auth"] = { - "aws_access_key": akey, - "aws_secret_key": skey} - elif provider == "azure": - sid = auth_element.get("subscription_id") - if sid is None: - missing_config.append("subscription_id") - cid = auth_element.get("client_id") - if cid is None: - missing_config.append("client_id") - sec = auth_element.get("secret") - if sec is None: - missing_config.append("secret") - ten = auth_element.get("tenant") - if ten is None: - missing_config.append("tenant") - config["auth"] = { - "azure_subscription_id": sid, - "azure_client_id": cid, - "azure_secret": sec, - "azure_tenant": ten} - elif provider == "google": - cre = auth_element.get("credentials_file") - if not os.path.isfile(cre): - raise IOError("The following file specified for GCE credentials not found: {}".format(cre)) - if cre is None: - missing_config.append("credentials_file") - config["auth"] = { - "gce_service_creds_file": cre} - else: - raise Exception("Unsupported provider `{}`.".format(provider)) + config["auth"] = { + "aws_access_key": akey, + "aws_secret_key": skey} + elif provider == "azure": + sid = auth_element.get("subscription_id") + if sid is None: + missing_config.append("subscription_id") + cid = auth_element.get("client_id") + if cid is None: + missing_config.append("client_id") + sec = auth_element.get("secret") + if sec is None: + missing_config.append("secret") + ten = auth_element.get("tenant") + if ten is None: + missing_config.append("tenant") + config["auth"] = { + "azure_subscription_id": sid, + "azure_client_id": cid, + "azure_secret": sec, + "azure_tenant": ten} + elif provider == "google": + cre = auth_element.get("credentials_file") + if not os.path.isfile(cre): + msg = "The following file specified for GCE credentials not found: {}".format(cre) + log.error(msg) + raise IOError(msg) + if cre is None: + missing_config.append("credentials_file") + config["auth"] = { + "gce_service_creds_file": cre} + else: + msg = "Unsupported provider `{}`.".format(provider) + log.error(msg) + raise Exception(msg) - if len(missing_config) > 0: - raise Exception("The following configuration required for {} cloud backend " - "are missing: {}".format(provider, missing_config)) - else: - return config + if len(missing_config) > 0: + msg = "The following configuration required for {} cloud backend " \ + "are missing: {}".format(provider, missing_config) + log.error(msg) + raise Exception(msg) + else: + return config + except Exception: + log.exception("TMalformed ObjectStore Configuration XML -- unable to continue") + raise def to_dict(self): as_dict = super(Cloud, self).to_dict() From 299dc5c586e916bbbf47484ece3afbd7b9db471d Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 30 Jan 2019 00:10:47 -0800 Subject: [PATCH 0046/1016] Refactor a bucket to `bucket_name` to diff with a bucket object. --- lib/galaxy/objectstore/cloud.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index e28499a00a5..2e9949f2f86 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -52,7 +52,7 @@ class Cloud(ObjectStore, CloudConfigMixin): connection_dict = config_dict.get('connection', {}) cache_dict = config_dict['cache'] - self.bucket = bucket_dict.get('name') + self.bucket_name = bucket_dict.get('name') self.use_rr = bucket_dict.get('use_reduced_redundancy', False) self.max_chunk_size = bucket_dict.get('max_chunk_size', 250) @@ -72,7 +72,7 @@ class Cloud(ObjectStore, CloudConfigMixin): raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE) self.conn = self._get_connection(provider, credentials) - self.bucket = self._get_bucket(self.bucket) + self.bucket = self._get_bucket(self.bucket_name) # Clean cache only if value is set in galaxy.ini if self.cache_size != -1: # Convert GBs to bytes for comparison From 8da9213508e7e218ca60a8680a714151622a7929 Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 30 Jan 2019 00:21:46 -0800 Subject: [PATCH 0047/1016] Add a comment to ObjectStore configuration. --- config/object_store_conf.xml.sample | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/config/object_store_conf.xml.sample b/config/object_store_conf.xml.sample index 507121f0557..c40e66931da 100644 --- a/config/object_store_conf.xml.sample +++ b/config/object_store_conf.xml.sample @@ -25,7 +25,7 @@ - + @@ -36,7 +36,7 @@ - + @@ -46,7 +46,7 @@ - + @@ -57,7 +57,7 @@ - + @@ -68,7 +68,7 @@ - + @@ -79,7 +79,7 @@ - + From 643f5210837a736de3bd30d8457d9de45b42c8d2 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Wed, 30 Jan 2019 14:16:28 -0500 Subject: [PATCH 0048/1016] Reference latest docker image for cellxgene --- .../cellxgene/config/allowed_images.yml.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/plugins/interactive_environments/cellxgene/config/allowed_images.yml.sample b/config/plugins/interactive_environments/cellxgene/config/allowed_images.yml.sample index 743bf2b3aff..f3ec2cde2b8 100644 --- a/config/plugins/interactive_environments/cellxgene/config/allowed_images.yml.sample +++ b/config/plugins/interactive_environments/cellxgene/config/allowed_images.yml.sample @@ -1,5 +1,5 @@ --- - - image: quay.io/galaxy/cellxgene-galaxy-ie:2019.01.25 + image: quay.io/galaxy/cellxgene-galaxy-ie:2019.01.30 description: | An interactive data explorer for single-cell transcriptomics datasets From fb01e0c0b3cb49620f2c3f66bd5dc7bd95b91ebf Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 30 Jan 2019 20:18:10 -0800 Subject: [PATCH 0049/1016] Changes to cloud auth configs, and implement config-to-dict method. --- lib/galaxy/objectstore/cloud.py | 59 +++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 2e9949f2f86..14fb998923f 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -19,7 +19,7 @@ from galaxy.util import ( umask_fix_perms, ) from galaxy.util.sleeper import Sleeper -from .s3 import CloudConfigMixin, parse_config_xml +from .s3 import parse_config_xml from ..objectstore import convert_bytes, ObjectStore try: from cloudbridge.cloud.factory import CloudProviderFactory, ProviderList @@ -35,6 +35,28 @@ NO_CLOUDBRIDGE_ERROR_MESSAGE = ( "Please install CloudBridge or modify ObjectStore configuration." ) +class CloudConfigMixin(object): + + def _config_to_dict(self): + return { + "provider": self.provider, + "auth": self.credentials, + "bucket": { + "name": self.bucket_name, + "use_reduced_redundancy": self.use_rr, + }, + "connection": { + "host": self.host, + "port": self.port, + "multipart": self.multipart, + "is_secure": self.is_secure, + "conn_path": self.conn_path, + }, + "cache": { + "size": self.cache_size, + "path": self.staging_path, + } + } class Cloud(ObjectStore, CloudConfigMixin): """ @@ -52,6 +74,8 @@ class Cloud(ObjectStore, CloudConfigMixin): connection_dict = config_dict.get('connection', {}) cache_dict = config_dict['cache'] + self.provider = config_dict["provider"] + self.credentials = config_dict["auth"] self.bucket_name = bucket_dict.get('name') self.use_rr = bucket_dict.get('use_reduced_redundancy', False) self.max_chunk_size = bucket_dict.get('max_chunk_size', 250) @@ -65,13 +89,13 @@ class Cloud(ObjectStore, CloudConfigMixin): self.cache_size = cache_dict.get('size', -1) self.staging_path = cache_dict.get('path') or self.config.object_store_cache_path - self._setup(config_dict["provider"], config_dict["auth"]) + self._initialize() - def _setup(self, provider, credentials): + def _initialize(self): if CloudProviderFactory is None: raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE) - self.conn = self._get_connection(provider, credentials) + self.conn = self._get_connection(self.provider, self.credentials) self.bucket = self._get_bucket(self.bucket_name) # Clean cache only if value is set in galaxy.ini if self.cache_size != -1: @@ -93,11 +117,18 @@ class Cloud(ObjectStore, CloudConfigMixin): def _get_connection(provider, credentials): log.debug("Configuring `{}` Connection".format(provider)) if provider == "aws": - connection = CloudProviderFactory().create_provider(ProviderList.AWS, credentials) + config = {"aws_access_key": credentials["access_key"], + "aws_secret_key": credentials["secret_key"]} + connection = CloudProviderFactory().create_provider(ProviderList.AWS, config) elif provider == "azure": - connection = CloudProviderFactory().create_provider(ProviderList.AZURE, credentials) + config = {"azure_subscription_id": credentials["subscription_id"], + "azure_client_id": credentials["client_id"], + "azure_secret": credentials["secret"], + "azure_tenant": credentials["tenant"]} + connection = CloudProviderFactory().create_provider(ProviderList.AZURE, config) elif provider == "google": - connection = CloudProviderFactory().create_provider(ProviderList.GCE, credentials) + config = {"gce_service_creds_file": credentials["service_creds_file"]} + connection = CloudProviderFactory().create_provider(ProviderList.GCE, config) else: raise Exception("Unsupported provider `{}`.".format(provider)) @@ -159,8 +190,8 @@ class Cloud(ObjectStore, CloudConfigMixin): missing_config.append("secret_key") config["auth"] = { - "aws_access_key": akey, - "aws_secret_key": skey} + "access_key": akey, + "secret_key": skey} elif provider == "azure": sid = auth_element.get("subscription_id") if sid is None: @@ -175,10 +206,10 @@ class Cloud(ObjectStore, CloudConfigMixin): if ten is None: missing_config.append("tenant") config["auth"] = { - "azure_subscription_id": sid, - "azure_client_id": cid, - "azure_secret": sec, - "azure_tenant": ten} + "subscription_id": sid, + "client_id": cid, + "secret": sec, + "tenant": ten} elif provider == "google": cre = auth_element.get("credentials_file") if not os.path.isfile(cre): @@ -188,7 +219,7 @@ class Cloud(ObjectStore, CloudConfigMixin): if cre is None: missing_config.append("credentials_file") config["auth"] = { - "gce_service_creds_file": cre} + "service_creds_file": cre} else: msg = "Unsupported provider `{}`.".format(provider) log.error(msg) From 2e8342d6753331c3bf316aaf63d874a84edb862c Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 30 Jan 2019 20:19:12 -0800 Subject: [PATCH 0050/1016] Fix a typo. --- lib/galaxy/objectstore/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 14fb998923f..08d1244e90f 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -233,7 +233,7 @@ class Cloud(ObjectStore, CloudConfigMixin): else: return config except Exception: - log.exception("TMalformed ObjectStore Configuration XML -- unable to continue") + log.exception("Malformed ObjectStore Configuration XML -- unable to continue") raise def to_dict(self): From 0a8feb2bbb4d4200e31523d80f6b42415e92eb57 Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 30 Jan 2019 20:19:42 -0800 Subject: [PATCH 0051/1016] Update Cloud unit tests. --- test/unit/test_objectstore.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/test/unit/test_objectstore.py b/test/unit/test_objectstore.py index 34e06245197..36b629667c6 100644 --- a/test/unit/test_objectstore.py +++ b/test/unit/test_objectstore.py @@ -508,7 +508,7 @@ def test_config_parse_s3(): assert len(extra_dirs) == 2 -CLOUD_TEST_CONFIG = """ +CLOUD_TEST_CONFIG = """ @@ -519,7 +519,8 @@ CLOUD_TEST_CONFIG = """ CLOUD_TEST_CONFIG_YAML = """ -type: s3 +type: cloud +provider: aws auth: access_key: access_moo secret_key: secret_cow @@ -543,10 +544,8 @@ extra_dirs: def test_config_parse_cloud(): for config_str in [CLOUD_TEST_CONFIG, CLOUD_TEST_CONFIG_YAML]: with TestConfig(config_str, clazz=UnitializedCloudObjectStore) as (directory, object_store): - assert object_store.access_key == "access_moo" - assert object_store.secret_key == "secret_cow" - assert object_store.bucket == "unique_bucket_name_all_lowercase" + assert object_store.bucket_name == "unique_bucket_name_all_lowercase" assert object_store.use_rr is False assert object_store.host is None @@ -555,13 +554,13 @@ def test_config_parse_cloud(): assert object_store.is_secure is True assert object_store.conn_path == "/" - assert object_store.cache_size == 1000 + assert object_store.cache_size == 1000.0 assert object_store.staging_path == "database/object_store_cache" assert object_store.extra_dirs["job_work"] == "database/job_working_directory_cloud" assert object_store.extra_dirs["temp"] == "database/tmp_cloud" as_dict = object_store.to_dict() - _assert_has_keys(as_dict, ["auth", "bucket", "connection", "cache", "extra_dirs", "type"]) + _assert_has_keys(as_dict, ["provider", "auth", "bucket", "connection", "cache", "extra_dirs", "type"]) _assert_key_has_value(as_dict, "type", "cloud") @@ -581,7 +580,7 @@ def test_config_parse_cloud(): _assert_key_has_value(connection_dict, "multipart", True) _assert_key_has_value(connection_dict, "is_secure", True) - _assert_key_has_value(cache_dict, "size", 1000) + _assert_key_has_value(cache_dict, "size", 1000.0) _assert_key_has_value(cache_dict, "path", "database/object_store_cache") extra_dirs = as_dict["extra_dirs"] From f8e13c153845e76274dfc3fbe46d79feebf55de0 Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 30 Jan 2019 20:30:53 -0800 Subject: [PATCH 0052/1016] Refactor a cloud config for S3. --- test/unit/test_objectstore.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/unit/test_objectstore.py b/test/unit/test_objectstore.py index 36b629667c6..84922add1ad 100644 --- a/test/unit/test_objectstore.py +++ b/test/unit/test_objectstore.py @@ -508,7 +508,7 @@ def test_config_parse_s3(): assert len(extra_dirs) == 2 -CLOUD_TEST_CONFIG = """ +CLOUD_AWS_TEST_CONFIG = """ @@ -518,7 +518,7 @@ CLOUD_TEST_CONFIG = """ """ -CLOUD_TEST_CONFIG_YAML = """ +CLOUD_AWS_TEST_CONFIG_YAML = """ type: cloud provider: aws auth: @@ -542,7 +542,7 @@ extra_dirs: def test_config_parse_cloud(): - for config_str in [CLOUD_TEST_CONFIG, CLOUD_TEST_CONFIG_YAML]: + for config_str in [CLOUD_AWS_TEST_CONFIG, CLOUD_AWS_TEST_CONFIG_YAML]: with TestConfig(config_str, clazz=UnitializedCloudObjectStore) as (directory, object_store): assert object_store.bucket_name == "unique_bucket_name_all_lowercase" From 62488a2097bf329d150ba81244fe4438257c0d87 Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 30 Jan 2019 21:01:29 -0800 Subject: [PATCH 0053/1016] Add Azure theory to cloud unit test. --- test/unit/test_objectstore.py | 47 ++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/test/unit/test_objectstore.py b/test/unit/test_objectstore.py index 84922add1ad..aca1cb1f56a 100644 --- a/test/unit/test_objectstore.py +++ b/test/unit/test_objectstore.py @@ -541,8 +541,42 @@ extra_dirs: """ +CLOUD_AZURE_TEST_CONFIG = """ + + + + + + +""" + +CLOUD_AZURE_TEST_CONFIG_YAML = """ +type: cloud +provider: azure +auth: + subscription_id: a_sub_id + client_id: and_a_client_id + secret: and_a_secret_key + tenant: and_some_tenant_info + +bucket: + name: unique_bucket_name_all_lowercase + use_reduced_redundancy: false + +cache: + path: database/object_store_cache + size: 1000 + +extra_dirs: +- type: job_work + path: database/job_working_directory_cloud +- type: temp + path: database/tmp_cloud +""" + + def test_config_parse_cloud(): - for config_str in [CLOUD_AWS_TEST_CONFIG, CLOUD_AWS_TEST_CONFIG_YAML]: + for config_str in [CLOUD_AWS_TEST_CONFIG, CLOUD_AWS_TEST_CONFIG_YAML, CLOUD_AZURE_TEST_CONFIG, CLOUD_AZURE_TEST_CONFIG_YAML]: with TestConfig(config_str, clazz=UnitializedCloudObjectStore) as (directory, object_store): assert object_store.bucket_name == "unique_bucket_name_all_lowercase" @@ -569,8 +603,15 @@ def test_config_parse_cloud(): connection_dict = as_dict["connection"] cache_dict = as_dict["cache"] - _assert_key_has_value(auth_dict, "access_key", "access_moo") - _assert_key_has_value(auth_dict, "secret_key", "secret_cow") + provider = as_dict["provider"] + if provider == "aws": + _assert_key_has_value(auth_dict, "access_key", "access_moo") + _assert_key_has_value(auth_dict, "secret_key", "secret_cow") + elif provider == "azure": + _assert_key_has_value(auth_dict, "subscription_id", "a_sub_id") + _assert_key_has_value(auth_dict, "client_id", "and_a_client_id") + _assert_key_has_value(auth_dict, "secret", "and_a_secret_key") + _assert_key_has_value(auth_dict, "tenant", "and_some_tenant_info") _assert_key_has_value(bucket_dict, "name", "unique_bucket_name_all_lowercase") _assert_key_has_value(bucket_dict, "use_reduced_redundancy", False) From 64fc91ae60cbb703868828eb6ab494744b23c757 Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 30 Jan 2019 22:29:51 -0800 Subject: [PATCH 0054/1016] Fix a bug naming GCE credentials file. --- lib/galaxy/objectstore/cloud.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 08d1244e90f..0438c67ec72 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -127,7 +127,7 @@ class Cloud(ObjectStore, CloudConfigMixin): "azure_tenant": credentials["tenant"]} connection = CloudProviderFactory().create_provider(ProviderList.AZURE, config) elif provider == "google": - config = {"gce_service_creds_file": credentials["service_creds_file"]} + config = {"gce_service_creds_file": credentials["credentials_file"]} connection = CloudProviderFactory().create_provider(ProviderList.GCE, config) else: raise Exception("Unsupported provider `{}`.".format(provider)) @@ -219,7 +219,7 @@ class Cloud(ObjectStore, CloudConfigMixin): if cre is None: missing_config.append("credentials_file") config["auth"] = { - "service_creds_file": cre} + "credentials_file": cre} else: msg = "Unsupported provider `{}`.".format(provider) log.error(msg) From 885b51d25ed9bde886759c643e3a2085b98affa1 Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 30 Jan 2019 22:44:20 -0800 Subject: [PATCH 0055/1016] Add GCE unit tests for the cloud backend of objectstore. --- test/unit/test_objectstore.py | 42 ++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/test/unit/test_objectstore.py b/test/unit/test_objectstore.py index aca1cb1f56a..8fa72c6f446 100644 --- a/test/unit/test_objectstore.py +++ b/test/unit/test_objectstore.py @@ -575,8 +575,46 @@ extra_dirs: """ +CLOUD_GOOGLE_TEST_CONFIG = """ + + + + + + +""" + +CLOUD_GOOGLE_TEST_CONFIG_YAML = """ +type: cloud +provider: google +auth: + credentials_file: gce.config + +bucket: + name: unique_bucket_name_all_lowercase + use_reduced_redundancy: false + +cache: + path: database/object_store_cache + size: 1000 + +extra_dirs: +- type: job_work + path: database/job_working_directory_cloud +- type: temp + path: database/tmp_cloud +""" + + def test_config_parse_cloud(): - for config_str in [CLOUD_AWS_TEST_CONFIG, CLOUD_AWS_TEST_CONFIG_YAML, CLOUD_AZURE_TEST_CONFIG, CLOUD_AZURE_TEST_CONFIG_YAML]: + for config_str in [CLOUD_AWS_TEST_CONFIG, CLOUD_AWS_TEST_CONFIG_YAML, CLOUD_AZURE_TEST_CONFIG, CLOUD_AZURE_TEST_CONFIG_YAML, CLOUD_GOOGLE_TEST_CONFIG, CLOUD_GOOGLE_TEST_CONFIG_YAML]: + if "google" in config_str: + tmpdir = mkdtemp() + if not os.path.exists(tmpdir): + os.makedirs(tmpdir) + path = os.path.join(tmpdir, "gce.config") + open(path, "w").write("some_gce_config") + config_str = config_str.replace("gce.config", path) with TestConfig(config_str, clazz=UnitializedCloudObjectStore) as (directory, object_store): assert object_store.bucket_name == "unique_bucket_name_all_lowercase" @@ -612,6 +650,8 @@ def test_config_parse_cloud(): _assert_key_has_value(auth_dict, "client_id", "and_a_client_id") _assert_key_has_value(auth_dict, "secret", "and_a_secret_key") _assert_key_has_value(auth_dict, "tenant", "and_some_tenant_info") + elif provider == "google": + _assert_key_has_value(auth_dict, "credentials_file", path) _assert_key_has_value(bucket_dict, "name", "unique_bucket_name_all_lowercase") _assert_key_has_value(bucket_dict, "use_reduced_redundancy", False) From c1920dfebd073ab585a4eee9c401feae9d9d30f2 Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 30 Jan 2019 22:45:23 -0800 Subject: [PATCH 0056/1016] Add some line breaks in unit tests. --- test/unit/test_objectstore.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/unit/test_objectstore.py b/test/unit/test_objectstore.py index 8fa72c6f446..80d2f226506 100644 --- a/test/unit/test_objectstore.py +++ b/test/unit/test_objectstore.py @@ -542,7 +542,8 @@ extra_dirs: CLOUD_AZURE_TEST_CONFIG = """ - + @@ -607,7 +608,9 @@ extra_dirs: def test_config_parse_cloud(): - for config_str in [CLOUD_AWS_TEST_CONFIG, CLOUD_AWS_TEST_CONFIG_YAML, CLOUD_AZURE_TEST_CONFIG, CLOUD_AZURE_TEST_CONFIG_YAML, CLOUD_GOOGLE_TEST_CONFIG, CLOUD_GOOGLE_TEST_CONFIG_YAML]: + for config_str in [CLOUD_AWS_TEST_CONFIG, CLOUD_AWS_TEST_CONFIG_YAML, + CLOUD_AZURE_TEST_CONFIG, CLOUD_AZURE_TEST_CONFIG_YAML, + CLOUD_GOOGLE_TEST_CONFIG, CLOUD_GOOGLE_TEST_CONFIG_YAML]: if "google" in config_str: tmpdir = mkdtemp() if not os.path.exists(tmpdir): From e28842b7a731a4a4cd99896e85d0ae2be42f5c7e Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 30 Jan 2019 22:54:53 -0800 Subject: [PATCH 0057/1016] Fix lint issues. --- lib/galaxy/objectstore/cloud.py | 2 ++ test/unit/test_objectstore.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 0438c67ec72..004b7137348 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -35,6 +35,7 @@ NO_CLOUDBRIDGE_ERROR_MESSAGE = ( "Please install CloudBridge or modify ObjectStore configuration." ) + class CloudConfigMixin(object): def _config_to_dict(self): @@ -58,6 +59,7 @@ class CloudConfigMixin(object): } } + class Cloud(ObjectStore, CloudConfigMixin): """ Object store that stores objects as items in an cloud storage. A local diff --git a/test/unit/test_objectstore.py b/test/unit/test_objectstore.py index 80d2f226506..1a905b404e7 100644 --- a/test/unit/test_objectstore.py +++ b/test/unit/test_objectstore.py @@ -542,7 +542,7 @@ extra_dirs: CLOUD_AZURE_TEST_CONFIG = """ - From 19816f2c25777805bbb3bb7406f36a1fe4fbdb52 Mon Sep 17 00:00:00 2001 From: Fabien Mareuil Date: Thu, 31 Jan 2019 19:15:05 +0100 Subject: [PATCH 0058/1016] Add a validator that ensures one metadata is between min and max values --- lib/galaxy/tools/parameters/validation.py | 59 ++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tools/parameters/validation.py b/lib/galaxy/tools/parameters/validation.py index 95d6670bbe0..5d4289e722f 100644 --- a/lib/galaxy/tools/parameters/validation.py +++ b/lib/galaxy/tools/parameters/validation.py @@ -448,6 +448,62 @@ class MetadataInDataTableColumnValidator(Validator): raise ValueError(self.message) +class MetadataInRangeValidator(Validator): + """ + Validator that ensures a metadata is in a specific range + """ + requires_dataset_metadata = True + + @classmethod + def from_element(cls, param, elem): + metadata_name = elem.get('metadata_name', None) + if metadata_name: + metadata_name = metadata_name.strip() + return cls(elem.get('message', None), elem.get('min'), + elem.get('max'), metadata_name, elem.get('exclude_min', 'false'), + elem.get('exclude_max', 'false')) + + def __init__(self, message, range_min, range_max, metadata_name, exclude_min=False, exclude_max=False): + self.metadata_name = metadata_name + self.min = float(range_min if range_min is not None else '-inf') + self.exclude_min = util.asbool(exclude_min) + self.max = float(range_max if range_max is not None else 'inf') + self.exclude_max = util.asbool(exclude_max) + assert self.min <= self.max, 'min must be less than or equal to max' + # Remove unneeded 0s and decimal from floats to make message pretty. + self_min_str = str(self.min).rstrip('0').rstrip('.') + self_max_str = str(self.max).rstrip('0').rstrip('.') + op1 = '>=' + op2 = '<=' + if self.exclude_min: + op1 = '>' + if self.exclude_max: + op2 = '<' + self.message = message or "{} must be {} {} and {} {}".format(metadata_name, op1, self_min_str, op2, self_max_str) + + def validate(self, value, trans=None): + if value: + if not isinstance(value, model.DatasetInstance): + raise ValueError('A non-dataset value was provided.') + value_to_check = int(value.metadata.spec[self.metadata_name].param.to_string(value.metadata.get(self.metadata_name))) + try: + float(value_to_check) + except ValueError: + raise ValueError('{} must be a float or an integer'.format(self.metadata_name)) + if self.exclude_min: + if not self.min < float(value_to_check): + raise ValueError(self.message) + else: + if not self.min <= float(value_to_check): + raise ValueError(self.message) + if self.exclude_max: + if not float(value_to_check) < self.max: + raise ValueError(self.message) + else: + if not float(value_to_check) <= self.max: + raise ValueError(self.message) + + validator_types = dict(expression=ExpressionValidator, regex=RegexValidator, in_range=InRangeValidator, @@ -460,7 +516,8 @@ validator_types = dict(expression=ExpressionValidator, empty_extra_files_path=DatasetExtraFilesPathEmptyValidator, dataset_metadata_in_file=MetadataInFileColumnValidator, dataset_metadata_in_data_table=MetadataInDataTableColumnValidator, - dataset_ok_validator=DatasetOkValidator,) + dataset_ok_validator=DatasetOkValidator, + dataset_metadata_in_range=MetadataInRangeValidator,) def get_suite(): From f729dab31312d1be5ef2226d533d4ad26fb25130 Mon Sep 17 00:00:00 2001 From: Mason Date: Wed, 23 Jan 2019 18:36:41 -0800 Subject: [PATCH 0059/1016] Tagging component --- client/.babelrc | 12 - client/galaxy/scripts/bundleEntries.js | 2 +- .../components/Tags/GalaxyTags.test.js | 49 +++ .../scripts/components/Tags/GalaxyTags.vue | 162 ++++++++ .../components/Tags/StandardTags.test.js | 112 ++++++ .../scripts/components/Tags/StandardTags.vue | 127 ++++++ .../Tags}/autocomplete_tagging.scss | 0 .../galaxy/scripts/components/Tags/index.js | 3 + .../galaxy/scripts/components/Tags/inits.js | 12 + .../galaxy/scripts/components/Tags/model.js | 46 +++ .../scripts/components/Tags/model.test.js | 60 +++ .../scripts/components/Tags/tagService.js | 111 ++++++ .../components/Tags/tagService.test.js | 152 ++++++++ .../scripts/components/Tags/tagStyles.scss | 74 ++++ .../Tags/testData/autocompleteResponse.txt | 3 + .../scripts/entry/analysis/AnalysisRouter.js | 3 +- client/galaxy/scripts/mvc/grid/grid-view.js | 66 +++- client/galaxy/scripts/sample.test.js | 12 - .../galaxy/scripts/store/gridSearchStore.js | 33 ++ client/galaxy/scripts/store/index.js | 17 + client/galaxy/scripts/ui/autocom_tagging.js | 369 ------------------ client/galaxy/scripts/unitTestBundle.js | 13 - client/galaxy/scripts/utils/mock.js | 2 +- .../galaxy/scripts/utils/mountVueComponent.js | 11 + client/galaxy/scripts/utils/redirect.js | 5 + client/galaxy/scripts/viz/trackster.js | 1 - client/galaxy/style/scss/base.scss | 1 - client/galaxy/style/scss/mixins.scss | 8 + client/karma/karma.config.mocha.js | 36 +- client/karma/karma.config.qunit.js | 3 +- client/karma/webpack.config.unittest.js | 55 ++- client/package.json | 20 +- client/webpack.config.js | 27 +- client/yarn.lock | 159 +++++++- lib/galaxy/web/framework/helpers/grids.py | 4 +- lib/galaxy/webapps/galaxy/controllers/tag.py | 2 - lib/galaxy/webapps/reports/framework/grids.py | 4 +- templates/tagging_common.mako | 305 ++++----------- templates/webapps/galaxy/workflow/editor.mako | 2 +- .../test_published_histories_grid.py | 2 +- test/selenium_tests/test_saved_histories.py | 10 +- 41 files changed, 1366 insertions(+), 729 deletions(-) delete mode 100644 client/.babelrc create mode 100644 client/galaxy/scripts/components/Tags/GalaxyTags.test.js create mode 100644 client/galaxy/scripts/components/Tags/GalaxyTags.vue create mode 100644 client/galaxy/scripts/components/Tags/StandardTags.test.js create mode 100644 client/galaxy/scripts/components/Tags/StandardTags.vue rename client/galaxy/{style/scss => scripts/components/Tags}/autocomplete_tagging.scss (100%) create mode 100644 client/galaxy/scripts/components/Tags/index.js create mode 100644 client/galaxy/scripts/components/Tags/inits.js create mode 100644 client/galaxy/scripts/components/Tags/model.js create mode 100644 client/galaxy/scripts/components/Tags/model.test.js create mode 100644 client/galaxy/scripts/components/Tags/tagService.js create mode 100644 client/galaxy/scripts/components/Tags/tagService.test.js create mode 100644 client/galaxy/scripts/components/Tags/tagStyles.scss create mode 100644 client/galaxy/scripts/components/Tags/testData/autocompleteResponse.txt delete mode 100644 client/galaxy/scripts/sample.test.js create mode 100644 client/galaxy/scripts/store/gridSearchStore.js create mode 100644 client/galaxy/scripts/store/index.js delete mode 100644 client/galaxy/scripts/ui/autocom_tagging.js delete mode 100644 client/galaxy/scripts/unitTestBundle.js create mode 100644 client/galaxy/scripts/utils/mountVueComponent.js create mode 100644 client/galaxy/scripts/utils/redirect.js create mode 100644 client/galaxy/style/scss/mixins.scss diff --git a/client/.babelrc b/client/.babelrc deleted file mode 100644 index f9d881a541d..00000000000 --- a/client/.babelrc +++ /dev/null @@ -1,12 +0,0 @@ -{ - "presets": [ - [ - "@babel/preset-env", - { - "modules": false - } - ] - ], - "plugins": ["transform-vue-template", "@babel/plugin-syntax-dynamic-import"], - "ignore": ["i18n.js", "utils/localization.js", "nls/*"] -} diff --git a/client/galaxy/scripts/bundleEntries.js b/client/galaxy/scripts/bundleEntries.js index c630dbb4dfc..eefb1ccf33c 100644 --- a/client/galaxy/scripts/bundleEntries.js +++ b/client/galaxy/scripts/bundleEntries.js @@ -96,7 +96,7 @@ export let chartUtilities = { export { initMasthead } from "components/Masthead/initMasthead"; export { panelManagement } from "onload/globalInits/panelManagement"; -export { init_tag_click_function } from "ui/autocom_tagging"; +export { mountTaggingComponent } from "components/Tags"; // Used in common.mako export { default as store } from "store"; diff --git a/client/galaxy/scripts/components/Tags/GalaxyTags.test.js b/client/galaxy/scripts/components/Tags/GalaxyTags.test.js new file mode 100644 index 00000000000..624dae8ce84 --- /dev/null +++ b/client/galaxy/scripts/components/Tags/GalaxyTags.test.js @@ -0,0 +1,49 @@ +import { mount } from "@vue/test-utils"; +import GalaxyTags from "./GalaxyTags"; + +describe("Tags/GalaxyTags.vue", () => { + + const testTags = ["abc", "def", "ghi"]; + + let wrapper, emitted; + + beforeEach(function () { + wrapper = mount(GalaxyTags); + wrapper.setProps({ + value: testTags + }); + emitted = wrapper.emitted(); + }) + + it("should render a div for each tag", () => { + let tags = wrapper.findAll(".ti-tag-center"); + assert(tags.length == testTags.length, "Wrong number of tags"); + for(let i=0; i < testTags.length; i++) { + assert(tags.at(i).is('div'), "button not a div"); + assert(tags.at(i).text() == testTags[i], "rendered tag label doesn't match test data"); + } + }) + + it("should emit a click event when the tag is clicked", () => { + let tags = wrapper.findAll(".ti-tag-center > div"); + tags.at(0).trigger("click"); + assert(emitted["tag-click"], "click event not detected"); + assert(emitted["tag-click"].length == 1, "wrong event count"); + }) + + it("should emit a tag model payload when tag is clicked", () => { + let tags = wrapper.findAll(".ti-tag-center > div"); + tags.at(0).trigger("click"); + let firstEvent = emitted["tag-click"][0]; + let firstArg = firstEvent[0]; + assert(firstArg.text = testTags[0], "returned tag model doesn't match test data"); + }) + + it("should change intermal model representation when new tag list assigned", async () => { + assert(wrapper.vm.tagModels.length == 3); + let newTags = ["floob", "clown", "hoohah", "doodoo"]; + wrapper.setProps({ value: newTags }); + assert(wrapper.vm.tagModels.length == newTags.length); + }) + +}) diff --git a/client/galaxy/scripts/components/Tags/GalaxyTags.vue b/client/galaxy/scripts/components/Tags/GalaxyTags.vue new file mode 100644 index 00000000000..fd714eb3412 --- /dev/null +++ b/client/galaxy/scripts/components/Tags/GalaxyTags.vue @@ -0,0 +1,162 @@ + + + + + + + diff --git a/client/galaxy/scripts/components/Tags/StandardTags.test.js b/client/galaxy/scripts/components/Tags/StandardTags.test.js new file mode 100644 index 00000000000..9f8edfc1d1f --- /dev/null +++ b/client/galaxy/scripts/components/Tags/StandardTags.test.js @@ -0,0 +1,112 @@ +import sinon from "sinon"; +import { mount } from "@vue/test-utils"; +import StandardTags from "./StandardTags"; +import store from "../../store"; + +let mockRedirect = sinon.stub(); +StandardTags.__Rewire__("redirectToUrl", mockRedirect); + +describe("Tags/StandardTags.vue", () => { + + let wrapper; + let testTags = ["abc", "def", "ghi"]; + + function clickFirstTag() { + let firstTag = wrapper.find(".ti-tag-center > div"); + firstTag.trigger('click'); + } + + // TODO: put this in a store-specific test file + it("the searchTags in the store should be a Set object", () => { + let searchTags = store.state.gridSearch.searchTags; // this is a Set() + assert(searchTags instanceof Set, "searchTags wrong variable type, should be Set()"); + }) + + + describe("grid tags (tagClickFn: add_tag_to_grid_filter)", () => { + + let propsData = { + tags: Array.from(testTags), + tagClickFn: "add_tag_to_grid_filter", + id: "fakeID", + itemClass: "fakeItemClass" + }; + + beforeEach(function () { + wrapper = mount(StandardTags, { + store, + propsData + }); + }) + + afterEach(function() { + mockRedirect.reset(); + store.dispatch("clearSearchTags"); + }) + + it("clicking on a search tag should put that tag in the global store", () => { + clickFirstTag(); + let searchTags = store.state.gridSearch.searchTags; // this is a Set() + assert(searchTags.has(testTags[0]), "clicked tag not in store"); + assert(searchTags.size == 1, `wrong number of searchTags in store: ${searchTags.size}`); + }) + + it("clicking the same tag twice should add and remove it from the global store", () => { + clickFirstTag(); + clickFirstTag(); + let searchTags = store.state.gridSearch.searchTags; + assert(!searchTags.has(testTags[0]), "clicked tag shouldn't be in store"); + }) + + it("clicking any odd number of times should put the tag in the store", () => { + let oddNumber = 2 * Math.floor(Math.random() * 10) + 1; + for(let i=0; i < oddNumber; i++) { + clickFirstTag(); + } + let searchTags = store.state.gridSearch.searchTags; + assert(searchTags.has(testTags[0]), "clicked tag not in store"); + }) + + it("clicking any even number of times should remove the tag from the store", () => { + let evenNumber = 2 * Math.floor(Math.random() * 10); + for(let i=0; i < evenNumber; i++) { + clickFirstTag(); + } + let searchTags = store.state.gridSearch.searchTags; + assert(!searchTags.has(testTags[0]), "clicked tag shouldn't be in store"); + }) + + }) + + describe("community tag (tagClickFn: community_tag_click)", () => { + + let propsData = { + tags: ["abc", "def"], + tagClickFn: "community_tag_click", + clickUrl: "foo/bar", + id: "fakeID", + itemClass: "fakeItemClass" + }; + + beforeEach(function () { + wrapper = mount(StandardTags, { + store, + propsData + }); + }) + + afterEach(function() { + mockRedirect.reset(); + store.dispatch("clearSearchTags"); + }) + + // tagClickFn=community_tag_click, clickUrl=something + it("should try to redirect when you click a 'community tag'", () => { + clickFirstTag(); + let expectedUrl = `${propsData.clickUrl}?f-tags=${testTags[0]}`; + assert(mockRedirect.calledWith(expectedUrl), "requested wrong url"); + }) + + }) + +}) diff --git a/client/galaxy/scripts/components/Tags/StandardTags.vue b/client/galaxy/scripts/components/Tags/StandardTags.vue new file mode 100644 index 00000000000..d86116ac2c0 --- /dev/null +++ b/client/galaxy/scripts/components/Tags/StandardTags.vue @@ -0,0 +1,127 @@ + + + + + diff --git a/client/galaxy/style/scss/autocomplete_tagging.scss b/client/galaxy/scripts/components/Tags/autocomplete_tagging.scss similarity index 100% rename from client/galaxy/style/scss/autocomplete_tagging.scss rename to client/galaxy/scripts/components/Tags/autocomplete_tagging.scss diff --git a/client/galaxy/scripts/components/Tags/index.js b/client/galaxy/scripts/components/Tags/index.js new file mode 100644 index 00000000000..effa89eae7b --- /dev/null +++ b/client/galaxy/scripts/components/Tags/index.js @@ -0,0 +1,3 @@ +export { default as GalaxyTags } from "./GalaxyTags"; +export { default as StandardTags } from "./StandardTags"; +export { mountTaggingComponent } from "./inits"; diff --git a/client/galaxy/scripts/components/Tags/inits.js b/client/galaxy/scripts/components/Tags/inits.js new file mode 100644 index 00000000000..4b2573b458b --- /dev/null +++ b/client/galaxy/scripts/components/Tags/inits.js @@ -0,0 +1,12 @@ +/** + * Initialization functions for tagging component. This is a bridge between the + * python-rendered page and an eventual component-based architecture. These + * functions pass in a set of python-rendered configuration variables and + * instantiate a component. In the near future, we'll just pass props to the + * component from the parent components and do away with this hybrid approach. + */ + +import StandardTags from "./StandardTags"; +import { mountVueComponent } from "utils/mountVueComponent"; + +export const mountTaggingComponent = mountVueComponent(StandardTags); diff --git a/client/galaxy/scripts/components/Tags/model.js b/client/galaxy/scripts/components/Tags/model.js new file mode 100644 index 00000000000..59f1054c2ec --- /dev/null +++ b/client/galaxy/scripts/components/Tags/model.js @@ -0,0 +1,46 @@ +/** + * The tag model is pretty simple, so this extra file might be overkill, but + * it's good practice to separate data modeling from data retrieval + */ + + +// model prototype + +function TagModel(props = {}) { + this.text = ""; + Object.assign(this, props); +} + +TagModel.prototype.equals = function(otherTag) { + return (this.text == otherTag.text); +} + +TagModel.prototype.toString = function() { + return this.text; +} + + +// Public factory + +export function createTag(data) { + let props = {}; + switch (typeof(data)) { + case "string": + props = { text: data }; + break; + case "object": + props = Object.assign({}, data); + break; + } + return new TagModel(props); +} + + + +// Returns tags in "newTags" that aren't present in "existingTags" + +export const diffTags = (newTags, existingTags) => { + let newModels = newTags.map(createTag); + let existingModels = existingTags.map(createTag); + return newModels.filter(tag => !existingModels.some(st => st.equals(tag))); +} diff --git a/client/galaxy/scripts/components/Tags/model.test.js b/client/galaxy/scripts/components/Tags/model.test.js new file mode 100644 index 00000000000..ce43b0d25c5 --- /dev/null +++ b/client/galaxy/scripts/components/Tags/model.test.js @@ -0,0 +1,60 @@ +import { createTag, diffTags } from "./model"; + + +describe("Tags/model.js", () => { + + + // Basic props + + describe("tag model", () => { + it("should have a string representation equal to text prop", () => { + let testLabel = "abc"; + let model = createTag(testLabel); + assert.equal(model, testLabel); + assert.equal(model.text, testLabel); + assert.equal(model.toString(), testLabel); + }) + }) + + + // Factory Function + + describe("createTag", () => { + + it("should build a model from a string", () => { + let label = "floob"; + let model = createTag(label); + expect(model.text).to.equal(label); + }) + + it("should build a model from an object", () => { + let data = { text: "floob" }; + let model = createTag(data); + expect(model.text).to.equal(data.text); + }) + }) + + + // Filtered select function, currently used in component to remove + // selected items from a list of returned autocomplete options + + describe("diffTags", () => { + + let source, selected; + + beforeEach(() => { + source = ["a", "b", "c", "d"].map(createTag); + selected = ["a", "d", "f"].map(createTag); + }) + + it("should remove duplicates from a passed array", () => { + let result = diffTags(source, selected); + expect(result.length).to.equal(2); + assert(result[0].equals(source[1]), true); + assert(result[0].equals(createTag("b")), true); + assert(result[1].equals(source[2]), true); + assert(result[1].equals(createTag("c")), true); + }) + }) + +}) diff --git a/client/galaxy/scripts/components/Tags/tagService.js b/client/galaxy/scripts/components/Tags/tagService.js new file mode 100644 index 00000000000..0b74a28ea0d --- /dev/null +++ b/client/galaxy/scripts/components/Tags/tagService.js @@ -0,0 +1,111 @@ +/** + * Generates a service object used by the tagging component to save, delete and + * lookup potential tag options for the autocomplete feature. Standard typeahead + * debouncing and ajax cancelling functionality is also provided here, though an + * argument can be made that it more properly belongs in the component that is + * handling the inputs. + * + * TODO: convert this python endpoint to a legit json REST service + */ + +import axios from "axios"; +import { createTag } from "./model"; +import { Subject } from "rxjs"; +import { filter, debounceTime, switchMap, distinctUntilChanged } from "rxjs/operators"; + + +export function buildTagService({ id, itemClass, context, debounceInterval = 150 }) { + + + /** + * Save tag, input can be text string or tag model + * @param {string|Tag} tag + * @returns Promise yielding new tag model + */ + async function saveTag(rawTag) { + let tag = createTag(rawTag); + let url = `/tag/add_tag_async?item_id=${id}&item_class=${itemClass}&context=${context}&new_tag=${tag.text}`; + let response = await axios.get(url); + if (response.status !== 200) { + throw new Error(`Unable to save tag: ${tag}`); + } + return createTag(tag); + } + + /** + * Delete tag, input can be text string or tag model + * @param {string|Tag} tag + * @returns Promise yielding deleted tag model + */ + async function deleteTag(rawTag) { + let tag = createTag(rawTag); + let url = `/tag/remove_tag_async?item_id=${id}&item_class=${itemClass}&context=${context}&tag_name=${tag.text}`; + let response = await axios.get(url); + if (response.status !== 200) { + throw new Error(`Unable to delete tag: ${tag}`); + } + return tag; + } + + /** + * Looks up autocomplete options based on search text + * @param {string} searchText + * @returns Promise yielding an array of tag models + */ + async function autocomplete(searchText) { + let url = `/tag/tag_autocomplete_data?item_id=${id}&item_class=${itemClass}&q=${searchText}`; + let response = await axios.get(url); + if (response.status !== 200) { + throw new Error(`Unable to retrieve autocomplete tags for search string: ${searchText}`); + } + return parseAutocompleteResults(response.data).map(createTag); + } + + /** + * Incoming autocomplete search text buffer + */ + const _searchText = new Subject(); + + return { + + // saves a single tag + save: saveTag, + + // deletes a single tag + delete: deleteTag, + + // returns options matching a search string for autocomplete this is + // exposed for testing purposes only, in practice a consuming component + // will set the autocompleteSearchText property and observe results by + // subscribing to the autocompleteOptions observable property + autocomplete, + + // input point for autocomplete text search + set autocompleteSearchText(txt) { + _searchText.next(txt); + }, + + // output of autocomplete search results + // subscribe to this observable to get results + autocompleteOptions: _searchText.pipe( + filter(txt => txt.length), + debounceTime(debounceInterval), + distinctUntilChanged(), + switchMap(autocomplete) + ) + } +} + + +/** + * Parser for the archaic result format in the current API. + * See testData/autocompleteResponse.txt for a sample + * @param {string} rawResponse + */ +export function parseAutocompleteResults(rawResponse) { + return rawResponse.split("\n") + .filter(line => line.includes("|")) + .map(line => line.split("|")[0]) + .filter(label => label.length) + .filter(label => label !== "#Header"); +} diff --git a/client/galaxy/scripts/components/Tags/tagService.test.js b/client/galaxy/scripts/components/Tags/tagService.test.js new file mode 100644 index 00000000000..3726cca0993 --- /dev/null +++ b/client/galaxy/scripts/components/Tags/tagService.test.js @@ -0,0 +1,152 @@ +import sinon from "sinon"; +import { buildTagService, __RewireAPI__ as rewire } from "./tagService"; +import { createTag } from "./model"; +import { interval } from "rxjs"; +import { take, takeUntil } from "rxjs/operators"; + +// test response +import autocompleteResponse from "./testData/autocompleteResponse.txt"; + + +describe("Tags/tagService.js", () => { + + let svcParams = { + id: 123, + itemClass: "fooClass", + context: "", + debounceInterval: 50 // shorter value than default for unit tests + }; + + let svc = buildTagService(svcParams); + + let mockAxios = { + get: () => null + }; + + let stub; + + beforeEach(() => { + rewire.__Rewire__("axios", mockAxios); + }) + + afterEach(() => { + if (stub) stub.restore(); + }) + + describe("save", () => { + + let testLabel = "fo0bar123"; + let testTag = createTag(testLabel); + + let { id, itemClass, context } = svcParams; + let expectedSaveUrl = `/tag/add_tag_async?item_id=${id}&item_class=${itemClass}&context=${context}&new_tag=${testLabel}` + + it("should save a string tag", async () => { + stub = sinon.stub(mockAxios, "get").resolves({ status: 200 }); + let savedTag = await svc.save(testLabel); + expect(savedTag.text).to.equal(testLabel); + assert(stub.calledWith(expectedSaveUrl)); + }) + + it("should save an object tag", async () => { + stub = sinon.stub(mockAxios, "get").resolves({ status: 200 }); + let savedTag = await svc.save(testTag); + expect(savedTag.text).to.equal(testLabel); + assert(stub.calledWith(expectedSaveUrl)); + }) + + // TODO: test error conditions + + }) + + describe("delete", () => { + + let testLabel = "fo0bar123"; + let testTag = createTag(testLabel); + + let { id, itemClass, context } = svcParams; + let expectedDeleteUrl = `/tag/remove_tag_async?item_id=${id}&item_class=${itemClass}&context=${context}&tag_name=${testLabel}`; + + it("should delete a text tag", async () => { + stub = sinon.stub(mockAxios, "get").resolves({ status: 200 }); + let result = await svc.delete(testTag); + assert(result); + assert(stub.calledWith(expectedDeleteUrl)); + }) + + it("should delete an object tag", async () => { + stub = sinon.stub(mockAxios, "get") + .resolves({ status: 200 }); + let result = await svc.delete(testTag); + assert(result); + assert(stub.calledWith(expectedDeleteUrl)); + }) + + // TODO: test error conditions + + }) + + describe("autocomplete", () => { + + let searchString = "foo"; + let { id, itemClass } = svcParams; + let expectedSearchUrl = `/tag/tag_autocomplete_data?item_id=${id}&item_class=${itemClass}&q=${searchString}`; + + let successResponse = { + status: 200, + data: autocompleteResponse + } + + let checkAutocompleteResult = (result) => { + assert(result); + assert(result instanceof Array); + assert(result.length == 2); + assert(result[0] instanceof Object); + assert(result[0].text = "abc"); + } + + // straight ajax request, unused in practice, but it's easier to + // test this call if we just expose it + it("ajax call should return tag objects", async () => { + stub = sinon.stub(mockAxios, "get").resolves(successResponse); + let result = await svc.autocomplete(searchString); + assert(stub.calledWith(expectedSearchUrl), "Called with wrong search url"); + checkAutocompleteResult(result); + }) + + // hit the search input with multiple entries, only one ajax call + // should result because of debouncing + it("should debounce autocomplete search inputs", (done) => { + + let searchResult; + + // ends subscription to observable so test doesn't go on forever + let spamCount = Math.floor(Math.random() * 10); + let timer = interval(1500).pipe(take(1)); + + // stub ajax request to return the success response if the + // searchString is the expected input + stub = sinon.stub(mockAxios, "get").resolves(successResponse); + + svc.autocompleteOptions + .pipe(takeUntil(timer)) + .subscribe( + result => searchResult = result, + err => console.log("error", err), + () => { + assert(stub.called, "Ajax call not made"); + assert(stub.callCount == 1, `Wrong number of ajax calls: ${stub.callCount}`); + checkAutocompleteResult(searchResult); + done(); + } + ); + + // spam a bunch of key inputs + for(var i=0; i < spamCount; i++) + svc.autocompleteSearchText = new String(Math.random()); + svc.autocompleteSearchText = searchString; + }) + + }) + +}) diff --git a/client/galaxy/scripts/components/Tags/tagStyles.scss b/client/galaxy/scripts/components/Tags/tagStyles.scss new file mode 100644 index 00000000000..2d7258140f5 --- /dev/null +++ b/client/galaxy/scripts/components/Tags/tagStyles.scss @@ -0,0 +1,74 @@ +// TODO: use general color definition file +@import "theme/blue.scss"; +@import "scss/mixins.scss"; + +// Puts a little graphic in place of the text-input +// when the input is not in focus +@mixin newTagHoverButton() { + .vue-tags-input .ti-tags .ti-new-tag-input-wrapper { + input { + background-color: transparent; + } + input:not(:focus) { + background: url("/static/images/fugue/tag--plus.png"); + background-repeat: no-repeat; + color: transparent; + &::placeholder { + color: transparent; + } + } + } +} + +// hides tag container +@mixin hideTagContainer() { + .vue-tags-input { + background-color: transparent; + .ti-input { + border: none; + } + } +} + +// general style butchering +@mixin matchBootstrapStyling() { + .vue-tags-input { + @include fill(); + .ti-tag { + border-radius: 4px; + font-size: 0.8rem; + font-weight: 400; + } + } +} + +// Version of the tags that only allow clicking +// existing tags instead of the full editing UI +@mixin forDisplayOnly() { + .vue-tags-input { + .ti-actions, + .ti-new-tag-input-wrapper { + display: none; + } + } +} + +.galaxy-tags { + + // adds in a graphic in place of the text input + @include newTagHoverButton(); + + // match bootstrap tag styles/colors + @include matchBootstrapStyling(); + + // display-only tags + &.disabled { + @include forDisplayOnly(); + } +} + +// Hide the border around the tag editing container +// when inside a grid row +.grid .galaxy-tags { + @include hideTagContainer(); +} diff --git a/client/galaxy/scripts/components/Tags/testData/autocompleteResponse.txt b/client/galaxy/scripts/components/Tags/testData/autocompleteResponse.txt new file mode 100644 index 00000000000..7d8b85f44af --- /dev/null +++ b/client/galaxy/scripts/components/Tags/testData/autocompleteResponse.txt @@ -0,0 +1,3 @@ +#Header|Your Tags +abc|abc +def|def \ No newline at end of file diff --git a/client/galaxy/scripts/entry/analysis/AnalysisRouter.js b/client/galaxy/scripts/entry/analysis/AnalysisRouter.js index 2d8b1eb123e..3fc93a5cb39 100644 --- a/client/galaxy/scripts/entry/analysis/AnalysisRouter.js +++ b/client/galaxy/scripts/entry/analysis/AnalysisRouter.js @@ -172,7 +172,8 @@ export const getAnalysisRouter = Galaxy => }, show_histories: function(action_id) { - this.page.display(new HistoryList.View({ action_id: action_id })); + let view = new HistoryList.View({ action_id: action_id }); + this.page.display(view); }, show_history_citations: function() { diff --git a/client/galaxy/scripts/mvc/grid/grid-view.js b/client/galaxy/scripts/mvc/grid/grid-view.js index 63f5cf0922b..88c32522068 100644 --- a/client/galaxy/scripts/mvc/grid/grid-view.js +++ b/client/galaxy/scripts/mvc/grid/grid-view.js @@ -8,6 +8,8 @@ import Templates from "mvc/grid/grid-template"; import PopupMenu from "mvc/ui/popup-menu"; import LoadingIndicator from "ui/loading-indicator"; import { init_refresh_on_change } from "onload/globalInits/init_refresh_on_change"; +import store from "../../store"; +import slug from "slug"; // This is necessary so that, when nested arrays are used in ajax/post/get methods, square brackets ('[]') are // not appended to the identifier of a nested array. @@ -20,22 +22,27 @@ export default Backbone.View.extend({ // Initialize initialize: function(grid_config) { + console.log("Grid initialize", grid_config, this); this.grid = new GridModel(); this.title = grid_config.title; this.active_tab = grid_config.active_tab; var self = this; - // Why is this a global? - window.add_tag_to_grid_filter = (tag_name, tag_value) => { - // Put tag name and value together. - var tag = tag_name + (tag_value !== undefined && tag_value !== "" ? `:${tag_value}` : ""); - var advanced_search = $("#advanced-search").is(":visible"); - if (!advanced_search) { - $("#standard-search").slideToggle("fast"); - $("#advanced-search").slideToggle("fast"); + // Subscribe to changes in the store, currently just storing + // tag changes from the tagging components, but that will change + // when we rework the grid. This subscription ties this older grid + // code to the new vue components + store.watch( + (state) => state.gridSearch.searchTags, + (newTags) => { + let tagArray = Array.from(newTags); + self.grid.add_filter("tags", tagArray, false); + self.openAdvancedSearch(); + self.render_filter_button("tags", tagArray); + self.go_page_one(); + self.execute(); } - self.add_filter_condition("tags", tag); - }; + ); if (grid_config.url_base && !grid_config.items) { LoadingIndicator.markViewAsLoading(this); @@ -66,6 +73,14 @@ export default Backbone.View.extend({ } }, + openAdvancedSearch: function() { + var isOpen = $("#advanced-search").is(":visible"); + if (!isOpen) { + $("#standard-search").slideToggle("fast"); + $("#advanced-search").slideToggle("fast"); + } + }, + // refresh frames handle_refresh: function(refresh_frames) { if (refresh_frames) { @@ -98,6 +113,9 @@ export default Backbone.View.extend({ // append main template this.$el.html(Templates.grid(options)); + + // add a class identifier for styling purposes + this.$el.addClass(this.getRootClassName(grid_config)); // update div contents this.$el.find("#grid-table-header").html(Templates.header(options)); @@ -310,6 +328,15 @@ export default Backbone.View.extend({ // Add condition to grid. this.grid.add_filter(name, value, true); + this.render_filter_button(name, value); + + // execute + this.go_page_one(); + this.execute(); + }, + + render_filter_button: function(name, value) { + // Add button that displays filter and provides a button to delete it. var t = $(Templates.filter_element(name, value)); var self = this; @@ -324,16 +351,17 @@ export default Backbone.View.extend({ // append to container var container = this.$el.find(`#${name}-filtering-criteria`); container.append(t); - - // execute - this.go_page_one(); - this.execute(); }, // Remove a condition to the grid filter; this adds the condition and refreshes the grid. - remove_filter_condition: function(name, value) { + remove_filter_condition: function(name, value) { // Remove filter condition. this.grid.remove_filter(name, value); + + // update vuex if the one criteria we're currently tracking changes + if (name == "tags") { + store.dispatch("removeSearchTag", { text: value }); + } // Execute this.go_page_one(); @@ -665,5 +693,13 @@ export default Backbone.View.extend({ }); } }); + }, + + // Generates a class name at the root of the view that we can + // use for conditional styling in the various kinds of grids + // instead of acres of if/then statements in javascript + getRootClassName({ title = "grid" }) { + return slug(title).toLowerCase(); } + }); diff --git a/client/galaxy/scripts/sample.test.js b/client/galaxy/scripts/sample.test.js deleted file mode 100644 index de7697b1a7b..00000000000 --- a/client/galaxy/scripts/sample.test.js +++ /dev/null @@ -1,12 +0,0 @@ -describe("Sample Test", () => { - it("hey look, tests run", () => { - assert.equal(1, 1); - }); - - it("prove I can use a Proxy", () => { - let target = {}; - let thing = new Proxy(target, {}); - thing.foo = 232; - assert.equal(target.foo, thing.foo); - }); -}); diff --git a/client/galaxy/scripts/store/gridSearchStore.js b/client/galaxy/scripts/store/gridSearchStore.js new file mode 100644 index 00000000000..44ea51fe266 --- /dev/null +++ b/client/galaxy/scripts/store/gridSearchStore.js @@ -0,0 +1,33 @@ +/** + * Vuex store module used for managing search parameters in the grid. Currently + * only manages the tags that access this store via the new tagging components, + * but presumably the entire grid search filter criteria will live here one day. + */ + +export const gridSearchStore = { + state: { + searchTags: new Set() + }, + mutations: { + // TODO: we could write an equivalence comparator here for searchTag + // Sets and not register a change if the new set is equivalent + setSearchTags(state, tags) { + state.searchTags = new Set(tags); + } + }, + actions: { + toggleSearchTag({ state, commit }, { text }) { + let tags = new Set(state.searchTags); + tags.has(text) ? tags.delete(text) : tags.add(text); + commit("setSearchTags", tags); + }, + removeSearchTag({ state, commit }, { text }) { + let tags = new Set(state.searchTags); + tags.delete(text); + commit("setSearchTags", tags); + }, + clearSearchTags({ state, commit }) { + commit("setSearchTags", new Set()); + } + } +} diff --git a/client/galaxy/scripts/store/index.js b/client/galaxy/scripts/store/index.js new file mode 100644 index 00000000000..14a1e97254a --- /dev/null +++ b/client/galaxy/scripts/store/index.js @@ -0,0 +1,17 @@ +/** + * Central Vuex store + */ + +import Vue from "vue"; +import Vuex from "vuex"; + +// initial use of central store for search parameter housing. Test was to see if +// we could set store values from a component (CommunityTags.vue) and observe +// changes in legacy code (see grid-view.js) +import { gridSearchStore as gridSearch } from "./gridSearchStore"; + +Vue.use(Vuex); + +export default new Vuex.Store({ + modules: { gridSearch } +}); diff --git a/client/galaxy/scripts/ui/autocom_tagging.js b/client/galaxy/scripts/ui/autocom_tagging.js deleted file mode 100644 index 6420484002b..00000000000 --- a/client/galaxy/scripts/ui/autocom_tagging.js +++ /dev/null @@ -1,369 +0,0 @@ -import $ from "jquery"; -import _ from "underscore"; - -// ============================================================================ -/** - * JQuery extension for tagging with autocomplete. - * @author: Jeremy Goecks - * @require: jquery.autocomplete plugin - */ -// -// Initialize "tag click functions" for tags. -// -export function init_tag_click_function(tag_elt, click_func) { - $(tag_elt) - .find(".tag-name") - .each(function() { - $(this).click(function() { - var tag_str = $(this).text(); - var tag_name_and_value = tag_str.split(":"); - click_func(tag_name_and_value[0], tag_name_and_value[1]); - return true; - }); - }); -} - -$.fn.autocomplete_tagging = function(options) { - var defaults = { - get_toggle_link_text_fn: function(tags) { - var text = ""; - var num_tags = _.size(tags); - if (num_tags > 0) { - text = num_tags + (num_tags > 1 ? " Tags" : " Tag"); - } else { - text = "Add tags"; - } - return text; - }, - tag_click_fn: function(name, value) {}, - editable: true, - input_size: 20, - in_form: false, - tags: {}, - use_toggle_link: true, - item_id: "", - add_tag_img: "", - add_tag_img_rollover: "", - delete_tag_img: "", - ajax_autocomplete_tag_url: "", - ajax_retag_url: "", - ajax_delete_tag_url: "", - ajax_add_tag_url: "" - }; - - var settings = $.extend(defaults, options); - - // - // Initalize object's elements. - // - - // Get elements for this object. For this_obj, assume the last element with the id is the "this"; this is somewhat of a hack to address the problem - // that there may be two tagging elements for a single item if there are both community and individual tags for an element. - var this_obj = $(this); - var tag_area = this_obj.find(".tag-area"); - var toggle_link = this_obj.find(".toggle-link"); - var tag_input_field = this_obj.find(".tag-input"); - var add_tag_button = this_obj.find(".add-tag-button"); - - // Initialize toggle link. - toggle_link.click(function() { - // Take special actions depending on whether toggle is showing or hiding link. - var after_toggle_fn; - if (tag_area.is(":hidden")) { - after_toggle_fn = function() { - // If there are no tags, go right to editing mode by generating a click on the area. - var num_tags = $(this).find(".tag-button").length; - if (num_tags === 0) { - tag_area.click(); - } - }; - } else { - after_toggle_fn = () => { - tag_area.blur(); - }; - } - tag_area.slideToggle("fast", after_toggle_fn); - return $(this); - }); - - // Initialize tag input field. - if (settings.editable) { - tag_input_field.hide(); - } - tag_input_field.keyup(function(e) { - if (e.keyCode === 27) { - // Escape key - $(this).trigger("blur"); - } else if ( - e.keyCode === 13 || // Return Key - e.keyCode === 188 || // Comma - e.keyCode === 32 // Space - ) { - // - // Check input. - // - - var new_value = this.value; - - // Suppress space after a ":" - if (new_value.indexOf(": ", new_value.length - 2) !== -1) { - this.value = new_value.substring(0, new_value.length - 1); - return false; - } - - // Remove trigger keys from input. - if (e.keyCode === 188 || e.keyCode === 32) { - new_value = new_value.substring(0, new_value.length - 1); - } - - // Trim whitespace. - new_value = $.trim(new_value); - - // Too short? - if (new_value.length < 2) { - return false; - } - - // - // New tag OK - apply it. - // - - this.value = ""; // Reset text field now that tag is being added - - // Add button for tag after all other tag buttons. - var new_tag_button = build_tag_button(new_value); - var tag_buttons = tag_area.children(".tag-button"); - if (tag_buttons.length !== 0) { - var last_tag_button = tag_buttons.slice(tag_buttons.length - 1); - last_tag_button.after(new_tag_button); - } else { - tag_area.prepend(new_tag_button); - } - - // Add tag to internal list. - var tag_name_and_value = new_value.split(":"); - settings.tags[tag_name_and_value[0]] = tag_name_and_value[1]; - - // Update toggle link text. - var new_text = settings.get_toggle_link_text_fn(settings.tags); - toggle_link.text(new_text); - - // Commit tag to server. - var zz = $(this); - $.ajax({ - url: settings.ajax_add_tag_url, - data: { new_tag: new_value }, - error: function() { - // Failed. Roll back changes and show alert. - new_tag_button.remove(); - delete settings.tags[tag_name_and_value[0]]; - var new_text = settings.get_toggle_link_text_fn(settings.tags); - toggle_link.text(new_text); - alert("Add tag failed"); - }, - success: function() { - // Flush autocomplete cache because it's not out of date. - // TODO: in the future, we could remove the particular item - // that was chosen from the cache rather than flush it. - zz.data("autocompleter").cacheFlush(); - } - }); - - return false; - } - }); - - // Add autocomplete to input. - var format_item_func = (key, row_position, num_rows, value, search_term) => { - var tag_name_and_value = value.split(":"); - return tag_name_and_value.length === 1 ? tag_name_and_value[0] : tag_name_and_value[1]; - }; - var autocomplete_options = { - selectFirst: false, - formatItem: format_item_func, - autoFill: false, - highlight: false - }; - tag_input_field.autocomplete_verheul(settings.ajax_autocomplete_tag_url, autocomplete_options); - - // Initialize delete tag images for current tags. - this_obj.find(".delete-tag-img").each(function() { - init_delete_tag_image($(this)); - }); - - // Initialize tag click function. - init_tag_click_function($(this), settings.tag_click_fn); - - // Initialize "add tag" button. - add_tag_button.click(function() { - $(this).hide(); - - // Clicking on button is the same as clicking on the tag area. - tag_area.click(); - return false; - }); - - // - // Set up tag area interactions; these are needed only if tags are editable. - // - if (settings.editable) { - // When the tag area blurs, go to "view tag" mode. - tag_area.bind("blur", e => { - if (_.size(settings.tags) > 0) { - add_tag_button.show(); - tag_input_field.hide(); - tag_area.removeClass("active-tag-area"); - // tag_area.addClass("tooltip"); - } else { - // No tags, so do nothing to ensure that input is still visible. - } - }); - - // On click, enable user to add tags. - tag_area.click(function(e) { - var is_active = $(this).hasClass("active-tag-area"); - - // If a "delete image" object was pressed and area is inactive, do nothing. - if ($(e.target).hasClass("delete-tag-img") && !is_active) { - return false; - } - - // If a "tag name" object was pressed and area is inactive, do nothing. - if ($(e.target).hasClass("tag-name") && !is_active) { - return false; - } - - // Remove tooltip. - // $(this).removeClass("tooltip"); - - // Hide add tag button, show tag_input field. Change background to show - // area is active. - $(this).addClass("active-tag-area"); - add_tag_button.hide(); - tag_input_field.show(); - tag_input_field.focus(); - - // Add handler to document that will call blur when the tag area is blurred; - // a tag area is blurred when a user clicks on an element outside the area. - var handle_document_click = e => { - var check_click = function(tag_area, target) { - // Blur the tag area if the element clicked on is not in the tag area. - if (target !== tag_area) { - tag_area.blur(); - $(window).unbind("click.tagging_blur"); - $(this).addClass("tooltip"); - } - }; - check_click(tag_area, $(e.target)); - }; - // TODO: we should attach the click handler to all frames in order to capture - // clicks outside the frame that this element is in. - //window.parent.document.onclick = handle_document_click; - //var temp = $(window.parent.document.body).contents().find("iframe").html(); - //alert(temp); - //$(document).parent().click(handle_document_click); - $(window).bind("click.tagging_blur", handle_document_click); - - return false; - }); - } - - // If using toggle link, hide the tag area. Otherwise, show the tag area. - if (settings.use_toggle_link) { - tag_area.hide(); - } - - // - // Helper functions. - // - - // Initialize a "delete tag image": when click, delete tag from UI and send delete request to server. - function init_delete_tag_image(delete_img) { - $(delete_img).mouseenter(function() { - $(this).attr("src", settings.delete_tag_img_rollover); - }); - $(delete_img).mouseleave(function() { - $(this).attr("src", settings.delete_tag_img); - }); - $(delete_img).click(function() { - // Tag button is image's parent. - var tag_button = $(this).parent(); - - // Get tag name, value. - var tag_name_elt = tag_button.find(".tag-name").eq(0); - var tag_str = tag_name_elt.text(); - var tag_name_and_value = tag_str.split(":"); - var tag_name = tag_name_and_value[0]; - var tag_value = tag_name_and_value[1]; - - var prev_button = tag_button.prev(); - tag_button.remove(); - - // Remove tag from local list for consistency. - delete settings.tags[tag_name]; - - // Update toggle link text. - var new_text = settings.get_toggle_link_text_fn(settings.tags); - toggle_link.text(new_text); - - // Delete tag. - $.ajax({ - url: settings.ajax_delete_tag_url, - data: { tag_name: tag_name }, - error: function() { - // Failed. Roll back changes and show alert. - settings.tags[tag_name] = tag_value; - if (prev_button.hasClass("tag-button")) { - prev_button.after(tag_button); - } else { - tag_area.prepend(tag_button); - } - alert("Remove tag failed"); - - toggle_link.text(settings.get_toggle_link_text_fn(settings.tags)); - - // TODO: no idea why it's necessary to set this up again. - delete_img.mouseenter(function() { - $(this).attr("src", settings.delete_tag_img_rollover); - }); - delete_img.mouseleave(function() { - $(this).attr("src", settings.delete_tag_img); - }); - }, - success: function() {} - }); - - return true; - }); - } - - // - // Function that builds a tag button. - // - function build_tag_button(tag_str) { - // Build "delete tag" image. - var delete_img = $("") - .attr("src", settings.delete_tag_img) - .addClass("delete-tag-img"); - init_delete_tag_image(delete_img); - - // Build tag button. - var tag_name_elt = $("") - .text(tag_str) - .addClass("tag-name"); - tag_name_elt.click(() => { - var tag_name_and_value = tag_str.split(":"); - settings.tag_click_fn(tag_name_and_value[0], tag_name_and_value[1]); - return true; - }); - - var tag_button = $("").addClass("tag-button"); - tag_button.append(tag_name_elt); - // Allow delete only if element is editable. - if (settings.editable) { - tag_button.append(delete_img); - } - - return tag_button; - } -}; diff --git a/client/galaxy/scripts/unitTestBundle.js b/client/galaxy/scripts/unitTestBundle.js deleted file mode 100644 index 7de155c4784..00000000000 --- a/client/galaxy/scripts/unitTestBundle.js +++ /dev/null @@ -1,13 +0,0 @@ -/** - * A combination of all the mocha test files so that we can run them as a unit, - * which is much faster during deployment. - * - * Note, this is non-intuitive, but the parameters of require.context must be - * literals, can't even be stored in local variables or webpack can't do static - * dependency analysis. - * - * https://webpack.js.org/guides/dependency-management/ - */ -// eslint-disable-next-line no-undef -let testContext = require.context("./", true, /\.test\.js$/); -testContext.keys().forEach(testContext); diff --git a/client/galaxy/scripts/utils/mock.js b/client/galaxy/scripts/utils/mock.js index 571b511c18c..6761196706c 100644 --- a/client/galaxy/scripts/utils/mock.js +++ b/client/galaxy/scripts/utils/mock.js @@ -1,4 +1,4 @@ -// console.log api but does nothing +// Applies noop to all methods for mock creation const doNothing = () => null; diff --git a/client/galaxy/scripts/utils/mountVueComponent.js b/client/galaxy/scripts/utils/mountVueComponent.js new file mode 100644 index 00000000000..020645a70bb --- /dev/null +++ b/client/galaxy/scripts/utils/mountVueComponent.js @@ -0,0 +1,11 @@ +// Generic Vue component mount for use in transitional +// mount functions + +import Vue from "vue"; +import store from "../store"; + +export const mountVueComponent = (ComponentDefinition) => (propsData, el) => { + // console.log("mount function", propsData); + let component = Vue.extend(ComponentDefinition); + return new component({ store, propsData, el }); +} diff --git a/client/galaxy/scripts/utils/redirect.js b/client/galaxy/scripts/utils/redirect.js new file mode 100644 index 00000000000..add91b9cc3a --- /dev/null +++ b/client/galaxy/scripts/utils/redirect.js @@ -0,0 +1,5 @@ +// This file exists purely to make unit testing easier + +export function redirectToUrl(url) { + window.location = url; +} \ No newline at end of file diff --git a/client/galaxy/scripts/viz/trackster.js b/client/galaxy/scripts/viz/trackster.js index 9ab6dc122a3..01cdc27c62f 100644 --- a/client/galaxy/scripts/viz/trackster.js +++ b/client/galaxy/scripts/viz/trackster.js @@ -24,7 +24,6 @@ import "libs/jquery/jquery.form"; import "libs/jquery/jquery.rating"; import "ui/editable-text"; -//import "style/scss/autocomplete_tagging.scss"; //import "static/style/jquery-ui/smoothness/jquery-ui.css"; //import "static/style/library.css"; //import "static/style/trackster.css"; diff --git a/client/galaxy/style/scss/base.scss b/client/galaxy/style/scss/base.scss index bdc70014e91..f56d1ab5621 100644 --- a/client/galaxy/style/scss/base.scss +++ b/client/galaxy/style/scss/base.scss @@ -30,7 +30,6 @@ $fa-font-path: "../../../node_modules/font-awesome/fonts/"; @import "ui.scss"; @import "library.scss"; @import "trackster.scss"; -@import "autocomplete_tagging.scss"; @import "toastr.scss"; @import "jstree.scss"; @import "tour.scss"; diff --git a/client/galaxy/style/scss/mixins.scss b/client/galaxy/style/scss/mixins.scss new file mode 100644 index 00000000000..4e5685d4fc5 --- /dev/null +++ b/client/galaxy/style/scss/mixins.scss @@ -0,0 +1,8 @@ +// Utility mixin expands to container edges +@mixin fill() { + position: relative; + top: 0; + left: 0; + width: 100%; + height: 100%; +} diff --git a/client/karma/karma.config.mocha.js b/client/karma/karma.config.mocha.js index 058e1587095..122537c32db 100644 --- a/client/karma/karma.config.mocha.js +++ b/client/karma/karma.config.mocha.js @@ -4,38 +4,20 @@ const baseKarmaConfig = require("./karma.config.base"); -const single_pack = (process.env.GALAXY_TEST_AS_SINGLE_PACK == "true"); - -const testBundles = [ - "**/unitTestBundle.js", - "**/mocha/test.js" -]; - -const separateTests = [ - "**/*.test.js", - "**/mocha/tests/*_tests.js" -]; - module.exports = function (config) { - console.log("single_pack?", single_pack); - - // pick all separate tests or the dynamic test-bundles - let files = single_pack - ? testBundles - : separateTests; - - let preprocessors = files.reduce((result, path) => { - result[path] = ["webpack"]; - return result; - }, {}); - let settings = Object.assign({}, baseKarmaConfig, { - files: files, + files: [ + "../../node_modules/@babel/polyfill/dist/polyfill.js", + { pattern: "**/*.test.js", watched: true }, + // { pattern: "**/mocha/tests/*_tests.js" } + ], + preprocessors: { + "**/*.js": ["webpack"] + }, exclude: ["**/qunit/*"], - preprocessors: preprocessors, reporters: ["mocha"], - frameworks: ["polyfill", "mocha", "chai"] + frameworks: ["mocha", "chai"] }); config.set(settings); diff --git a/client/karma/karma.config.qunit.js b/client/karma/karma.config.qunit.js index 743b0eb86c7..54e585a4dc2 100644 --- a/client/karma/karma.config.qunit.js +++ b/client/karma/karma.config.qunit.js @@ -29,7 +29,8 @@ module.exports = function (config) { let settings = Object.assign({}, baseKarmaConfig, { files: testFiles.concat(assets), preprocessors: preprocessors, - frameworks: ["polyfill", "qunit"] + frameworks: ["polyfill", "qunit"], + singleRun: true }); config.set(settings); diff --git a/client/karma/webpack.config.unittest.js b/client/karma/webpack.config.unittest.js index 77483b5b2a3..c3d98dfec4f 100644 --- a/client/karma/webpack.config.unittest.js +++ b/client/karma/webpack.config.unittest.js @@ -3,28 +3,45 @@ * the ignore-loader for speedier testing. */ -let merge = require("webpack-merge"); -let wpConfig = require("../webpack.config"); +const merge = require("webpack-merge"); +const wpConfig = require("../webpack.config"); wpConfig.mode = "development"; wpConfig.entry = () => ({}); -// Don't need assets for unit testing, override those rules -module.exports = merge.smart(wpConfig, { - module: { - rules: [ - { - test: /\.(png|jpg|jpeg|gif|svg|woff|woff2|ttf|eot)(\?.*$|$)/, - loader: "ignore-loader" - }, - { - test: /\.css$/, - loader: "ignore-loader" - }, - { - test: /\.scss$/, - loader: "ignore-loader" - } - ] + +// Don't need any assets for unit testing + +let ignoreAssetLoaders = { + rules:[ + { + test: /\.(png|jpg|jpeg|gif|svg|woff|woff2|ttf|eot)(\?.*$|$)/, + loader: "ignore-loader" + }, + { + test: /\.css$/, + loader: "ignore-loader" + }, + { + test: /\.scss$/, + loader: "ignore-loader" + } + ] +}; + +wpConfig.module = merge.smart(wpConfig.module, ignoreAssetLoaders); + + +// Using babel-plugin-rewire to handle dependency mocking since webpack 4 +// exports immutable bindings for ES modules but we still need a way to +// overwrite dependencies during unit-testing. + +wpConfig.module.rules = wpConfig.module.rules.map(rule => { + if (rule.loader == "babel-loader") { + rule.options.plugins.push("rewire"); } + return rule; }); + + +module.exports = wpConfig; diff --git a/client/package.json b/client/package.json index cae03da10ba..8061b9dfb62 100644 --- a/client/package.json +++ b/client/package.json @@ -16,6 +16,8 @@ "@babel/polyfill": "^7.0.0", "@babel/preset-env": "^7.1.0", "@handsontable/vue": "^2.0.0-beta1", + "@johmun/vue-tags-input": "^2.0.0", + "@vue/test-utils": "^1.0.0-beta.28", "amdi18n-loader": "^0.8.0", "axios": "^0.18.0", "backbone": "1.3", @@ -44,9 +46,12 @@ "raven-js": "^3.27.0", "requirejs": "2.3.6", "rxjs": "^6.3.3", + "slug": "^0.9.3", "underscore": "^1.9.1", - "vue": "2.5.17", - "vue-router": "3.0.1" + "vue": "^2.5.22", + "vue-router": "^3.0.2", + "vue-rx": "^6.1.0", + "vuex": "^3.1.0" }, "scripts": { "watch": "gulp staging && gulp clean && gulp && yarn run save-build-hash && yarn run webpack-watch", @@ -69,13 +74,14 @@ "styleguide:build": "vue-styleguidist build", "test": "npm run test-mocha && npm run test-qunit", "test-watch": "npm run test-mocha -- --no-single-run", - "test-qunit": "GALAXY_TEST_AS_SINGLE_PACK=true karma start karma/karma.config.qunit.js", - "test-mocha": "GALAXY_TEST_AS_SINGLE_PACK=true karma start karma/karma.config.mocha.js", + "test-qunit": "karma start karma/karma.config.qunit.js", + "test-mocha": "karma start karma/karma.config.mocha.js", "jshint": "jshint --exclude='galaxy/scripts/libs/**' galaxy/scripts/**/*.js", "eslint": "eslint -c .eslintrc.js galaxy/scripts --ext .js,.vue" }, "devDependencies": { "babel-loader": "^8.0.4", + "babel-plugin-rewire": "^1.2.0", "babel-plugin-transform-inline-environment-variables": "^0.4.3", "babel-plugin-transform-vue-template": "^0.4.2", "chai": "^4.2.0", @@ -112,14 +118,14 @@ "phantomjs-prebuilt": "^2.1.7", "prettier": "^1.15.3", "qunitjs": "^2.4.1", + "raw-loader": "^1.0.0", "sass-loader": "^7.1.0", "sinon": "^4.1.2", "store": "^2.0.12", "style-loader": "^0.23.1", - "vue-loader": "^15.4.2", - "vue-style-loader": "^4.1.2", + "vue-loader": "^15.6.2", "vue-styleguidist": "^1.8.9", - "vue-template-compiler": "2.5.17", + "vue-template-compiler": "^2.5.22", "webpack": "^4.23.0", "webpack-cli": "^3.1.2", "webpack-merge": "^4.1.4", diff --git a/client/webpack.config.js b/client/webpack.config.js index 88d306b6e06..1c308fc40be 100644 --- a/client/webpack.config.js +++ b/client/webpack.config.js @@ -29,7 +29,8 @@ let buildconfig = { alias: { jquery$: `${libsBase}/jquery.custom.js`, jqueryVendor$: `${libsBase}/jquery/jquery.js`, - store$: "store/dist/store.modern.js" + store$: "store/dist/store.modern.js", + vue$: "vue/dist/vue.esm.js" } }, optimization: { @@ -64,7 +65,22 @@ let buildconfig = { libsBase ], loader: "babel-loader", - options: { babelrc: true } + options: { + cacheDirectory: true, + cacheCompression: false, + presets: [ + ["@babel/preset-env", { modules: false }] + ], + plugins: [ + "transform-vue-template", + "@babel/plugin-syntax-dynamic-import" + ], + ignore: [ + "i18n.js", + "utils/localization.js", + "nls/*" + ] + } }, { test: `${libsBase}/jquery.custom.js`, @@ -144,12 +160,15 @@ let buildconfig = { }, { loader: "sass-loader", - options: { sourceMap: true } + options: { + sourceMap: true, + includePaths: ["galaxy/style/scss"] + } } ] }, { - test: /\.tmpl$/, + test: /\.(txt|tmpl)$/, loader: "raw-loader" } ] diff --git a/client/yarn.lock b/client/yarn.lock index 23886d562b7..a810dae625a 100644 --- a/client/yarn.lock +++ b/client/yarn.lock @@ -754,6 +754,13 @@ resolved "https://registry.yarnpkg.com/@handsontable/vue/-/vue-2.0.0.tgz#28ae7d247a89738088abc32584562925dde18db0" integrity sha512-QVqJywrQWwJzoDCXibZkXrg+T91hNCTCN9qznmum4FSewL0MUtaHotv7Zc/D4scvEwpwWnKx5vpw4CY14V4OYw== +"@johmun/vue-tags-input@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@johmun/vue-tags-input/-/vue-tags-input-2.0.0.tgz#95870006ddd1320eb4979a58cec8ed24ee966c9b" + integrity sha512-SBNZm9ZtgwfnjEYUOkJ8Al7ZzUluZ3F49mO8EOTjTdYfB+ZJIQbf0R71PGYfKWkU7qdAwzW2imzlTPnJC3Wh7g== + dependencies: + vue "^2.5.16" + "@sinonjs/formatio@3.0.0": version "3.0.0" resolved "https://registry.yarnpkg.com/@sinonjs/formatio/-/formatio-3.0.0.tgz#9d282d81030a03a03fa0c5ce31fd8786a4da311a" @@ -802,6 +809,29 @@ source-map "^0.5.6" vue-template-es2015-compiler "^1.6.0" +"@vue/component-compiler-utils@^2.5.1": + version "2.5.2" + resolved "https://registry.yarnpkg.com/@vue/component-compiler-utils/-/component-compiler-utils-2.5.2.tgz#a8d57e773354ab10e4742c7d6a8dd86184d4d7be" + integrity sha512-3exq9O89GXo9E+CGKzgURCbasG15FtFMs8QRrCUVWGaKue4Egpw41MHb3Avtikv1VykKfBq3FvAnf9Nx3sdVJg== + dependencies: + consolidate "^0.15.1" + hash-sum "^1.0.2" + lru-cache "^4.1.2" + merge-source-map "^1.1.0" + postcss "^7.0.14" + postcss-selector-parser "^5.0.0" + prettier "1.16.3" + source-map "~0.6.1" + vue-template-es2015-compiler "^1.8.2" + +"@vue/test-utils@^1.0.0-beta.28": + version "1.0.0-beta.28" + resolved "https://registry.yarnpkg.com/@vue/test-utils/-/test-utils-1.0.0-beta.28.tgz#767c43413df8cde86128735e58923803e444b9a5" + integrity sha512-uVbFJG0g/H9hf2pgWUdhvQYItRGzQ44cMFf00wp0YEo85pxuvM9e3mx8QLQfx6R2CogxbK4CvV7qvkLblehXeQ== + dependencies: + dom-event-types "^1.0.0" + lodash "^4.17.4" + "@vxna/mini-html-webpack-template@^0.1.7": version "0.1.7" resolved "https://registry.yarnpkg.com/@vxna/mini-html-webpack-template/-/mini-html-webpack-template-0.1.7.tgz#2a8270e513ee14f395cc17c2ce22ced383c45d22" @@ -1839,6 +1869,11 @@ babel-plugin-espower@^2.1.0: espurify "^1.6.0" estraverse "^4.1.1" +babel-plugin-rewire@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/babel-plugin-rewire/-/babel-plugin-rewire-1.2.0.tgz#822562d72ed2c84e47c0f95ee232c920853e9d89" + integrity sha512-JBZxczHw3tScS+djy6JPLMjblchGhLI89ep15H3SyjujIzlxo5nr6Yjo7AXotdeVczeBmWs0tF8PgJWDdgzAkQ== + babel-plugin-syntax-async-functions@^6.8.0: version "6.13.0" resolved "https://registry.yarnpkg.com/babel-plugin-syntax-async-functions/-/babel-plugin-syntax-async-functions-6.13.0.tgz#cad9cad1191b5ad634bf30ae0872391e0647be95" @@ -2968,6 +3003,15 @@ chalk@^2.0.0, chalk@^2.0.1, chalk@^2.1.0, chalk@^2.3.0, chalk@^2.3.1, chalk@^2.4 escape-string-regexp "^1.0.5" supports-color "^5.3.0" +chalk@^2.4.2: + version "2.4.2" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" + integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== + dependencies: + ansi-styles "^3.2.1" + escape-string-regexp "^1.0.5" + supports-color "^5.3.0" + character-entities-html4@^1.0.0: version "1.1.2" resolved "https://registry.yarnpkg.com/character-entities-html4/-/character-entities-html4-1.1.2.tgz#c44fdde3ce66b52e8d321d6c1bf46101f0150610" @@ -3927,6 +3971,11 @@ cssesc@^1.0.1: resolved "https://registry.yarnpkg.com/cssesc/-/cssesc-1.0.1.tgz#ef7bd8d0229ed6a3a7051ff7771265fe7330e0a8" integrity sha512-S2hzrpWvE6G/rW7i7IxJfWBYn27QWfOIncUW++8Rbo1VB5zsJDSVPcnI+Q8z7rhxT6/yZeLOCja4cZnghJrNGA== +cssesc@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/cssesc/-/cssesc-2.0.0.tgz#3b13bd1bb1cb36e1bcb5a4dcd27f54c5dcb35703" + integrity sha512-MsCAG1z9lPdoO/IUMLSBWBSVxVtJ1395VGIQ+Fc2gNdkQ1hNDnQdw3YhA71WJCBW1vdwA0cAnk/DnW6bqoEUYg== + cssnano-preset-default@^4.0.2: version "4.0.3" resolved "https://registry.yarnpkg.com/cssnano-preset-default/-/cssnano-preset-default-4.0.3.tgz#9bfd1b06d4aa3991ed958ad9b9ec25a179261705" @@ -4411,6 +4460,11 @@ dom-converter@~0.2: dependencies: utila "~0.4" +dom-event-types@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/dom-event-types/-/dom-event-types-1.0.0.tgz#5830a0a29e1bf837fe50a70cd80a597232813cae" + integrity sha512-2G2Vwi2zXTHBGqXHsJ4+ak/iP0N8Ar+G8a7LiD2oup5o4sQWytwqqrZu/O6hIMV0KMID2PL69OhpshLO0n7UJQ== + dom-serialize@^2.2.0: version "2.2.1" resolved "https://registry.yarnpkg.com/dom-serialize/-/dom-serialize-2.2.1.tgz#562ae8999f44be5ea3076f5419dcd59eb43ac95b" @@ -10243,6 +10297,15 @@ postcss-selector-parser@^3.0.0, postcss-selector-parser@^3.1.1: indexes-of "^1.0.1" uniq "^1.0.1" +postcss-selector-parser@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-5.0.0.tgz#249044356697b33b64f1a8f7c80922dddee7195c" + integrity sha512-w+zLE5Jhg6Liz8+rQOWEAwtwkyqpfnmsinXjXg6cY7YIONZZtgvE0v2O0uhQBs0peNomOJwWRKt6JBfTdTd3OQ== + dependencies: + cssesc "^2.0.0" + indexes-of "^1.0.1" + uniq "^1.0.1" + postcss-selector-parser@^5.0.0-rc.3: version "5.0.0-rc.3" resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-5.0.0-rc.3.tgz#c4525dcc8eb90166c53dcbf0cb9317ceff5a15b5" @@ -10333,6 +10396,15 @@ postcss@^7.0.0, postcss@^7.0.1, postcss@^7.0.2: source-map "^0.6.1" supports-color "^5.5.0" +postcss@^7.0.14: + version "7.0.14" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-7.0.14.tgz#4527ed6b1ca0d82c53ce5ec1a2041c2346bbd6e5" + integrity sha512-NsbD6XUUMZvBxtQAJuWDJeeC4QFsmWsfozWxCJPWf3M55K9iu2iMDaKqyoOdTJ1R4usBXuxlVFAIo8rZPQD4Bg== + dependencies: + chalk "^2.4.2" + source-map "^0.6.1" + supports-color "^6.1.0" + power-assert-context-formatter@^1.0.7: version "1.2.0" resolved "https://registry.yarnpkg.com/power-assert-context-formatter/-/power-assert-context-formatter-1.2.0.tgz#8fbe72692288ec5a7203cdf215c8b838a6061d2a" @@ -10467,6 +10539,11 @@ prettier@1.13.7: resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.13.7.tgz#850f3b8af784a49a6ea2d2eaa7ed1428a34b7281" integrity sha512-KIU72UmYPGk4MujZGYMFwinB7lOf2LsDNGSOC8ufevsrPLISrZbNJlWstRi3m0AMuszbH+EFSQ/r6w56RSPK6w== +prettier@1.16.3: + version "1.16.3" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.16.3.tgz#8c62168453badef702f34b45b6ee899574a6a65d" + integrity sha512-kn/GU6SMRYPxUakNXhpP0EedT/KmaPzr0H5lIsDogrykbaxOpOfAFfk5XA7DZrJyMAv1wlMV3CPcZruGXVVUZw== + prettier@^1.15.3: version "1.15.3" resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.15.3.tgz#1feaac5bdd181237b54dbe65d874e02a1472786a" @@ -10765,6 +10842,14 @@ raw-body@2.3.3: iconv-lite "0.4.23" unpipe "1.0.0" +raw-loader@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/raw-loader/-/raw-loader-1.0.0.tgz#3f9889e73dadbda9a424bce79809b4133ad46405" + integrity sha512-Uqy5AqELpytJTRxYT4fhltcKPj0TyaEpzJDcGz7DFJi+pQOOi3GjR/DOdxTkTsF+NzhnldIoG6TORaBlInUuqA== + dependencies: + loader-utils "^1.1.0" + schema-utils "^1.0.0" + rc@^1.0.1, rc@^1.1.6, rc@^1.2.7: version "1.2.8" resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed" @@ -11889,6 +11974,13 @@ slide@^1.1.5: resolved "https://registry.yarnpkg.com/slide/-/slide-1.1.6.tgz#56eb027d65b4d2dce6cb2e2d32c4d4afc9e1d707" integrity sha1-VusCfWW00tzmyy4tMsTUr8nh1wc= +slug@^0.9.3: + version "0.9.3" + resolved "https://registry.yarnpkg.com/slug/-/slug-0.9.3.tgz#8c9c773d79367c0188733316cf49fd2b8db40f6a" + integrity sha512-DddSQQnUdAofjFOKRT+zsMNrdzdte04G5DUA+NeaUJlPAqR1bWQ22qVfayRxHFiRGR3bUV0wt5VSj4849pGKSw== + dependencies: + unicode ">= 0.3.1" + snapdragon-node@^2.0.1: version "2.1.1" resolved "https://registry.yarnpkg.com/snapdragon-node/-/snapdragon-node-2.1.1.tgz#6c175f86ff14bdb0724563e8f3c1b021a286853b" @@ -12466,6 +12558,13 @@ supports-color@^5.1.0, supports-color@^5.3.0, supports-color@^5.4.0, supports-co dependencies: has-flag "^3.0.0" +supports-color@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-6.1.0.tgz#0764abc69c63d5ac842dd4867e8d025e880df8f3" + integrity sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ== + dependencies: + has-flag "^3.0.0" + svgo@^0.7.0: version "0.7.2" resolved "https://registry.yarnpkg.com/svgo/-/svgo-0.7.2.tgz#9f5772413952135c6fefbf40afe6a4faa88b4bb5" @@ -12960,6 +13059,11 @@ unicode-property-aliases-ecmascript@^1.0.4: resolved "https://registry.yarnpkg.com/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-1.0.4.tgz#5a533f31b4317ea76f17d807fa0d116546111dd0" integrity sha512-2WSLa6OdYd2ng8oqiGIWnJqyFArvhn+5vgx5GTxMbUYjCYKUcuKS62YLFF0R/BDGlB1yzXjQOLtPAfHsgirEpg== +"unicode@>= 0.3.1": + version "11.0.1" + resolved "https://registry.yarnpkg.com/unicode/-/unicode-11.0.1.tgz#735bd422ec75cf28d396eb224d535d168d5f1db6" + integrity sha512-+cHtykLb+eF1yrSLWTwcYBrqJkTfX7Quoyg7Juhe6uylF43ZbMdxMuSHNYlnyLT8T7POAvavgBthzUF9AIaQvQ== + unified@^6.0.0: version "6.2.0" resolved "https://registry.yarnpkg.com/unified/-/unified-6.2.0.tgz#7fbd630f719126d67d40c644b7e3f617035f6dba" @@ -13377,7 +13481,7 @@ vue-hot-reload-api@^2.3.0: resolved "https://registry.yarnpkg.com/vue-hot-reload-api/-/vue-hot-reload-api-2.3.1.tgz#b2d3d95402a811602380783ea4f566eb875569a2" integrity sha512-AA86yKZ5uOKz87/q1UpngEXhbRkaYg1b7HMMVRobNV1IVKqZe8oLIzo6iMocVwZXnYitlGwf2k4ZRLOZlS8oPQ== -vue-loader@^15.2.4, vue-loader@^15.4.2: +vue-loader@^15.2.4: version "15.4.2" resolved "https://registry.yarnpkg.com/vue-loader/-/vue-loader-15.4.2.tgz#812bb26e447dd3b84c485eb634190d914ce125e2" integrity sha512-nVV27GNIA9MeoD8yQ3dkUzwlAaAsWeYSWZHsu/K04KCD339lW0Jv2sJWsjj3721SP7sl2lYdPmjcHgkWQSp5bg== @@ -13388,10 +13492,26 @@ vue-loader@^15.2.4, vue-loader@^15.4.2: vue-hot-reload-api "^2.3.0" vue-style-loader "^4.1.0" -vue-router@3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/vue-router/-/vue-router-3.0.1.tgz#d9b05ad9c7420ba0f626d6500d693e60092cc1e9" - integrity sha512-vLLoY452L+JBpALMP5UHum9+7nzR9PeIBCghU9ZtJ1eWm6ieUI8Zb/DI3MYxH32bxkjzYV1LRjNv4qr8d+uX/w== +vue-loader@^15.6.2: + version "15.6.2" + resolved "https://registry.yarnpkg.com/vue-loader/-/vue-loader-15.6.2.tgz#892741d96260936ff69e892f72ec361ba4d100d2" + integrity sha512-T6fONodj861M3PqZ1jlbUFjeezbUnPRY2bd+3eZuDvYADgkN3VFU2H5feqySNg9XBt8rcbyBGmFWTZtrOX+v5w== + dependencies: + "@vue/component-compiler-utils" "^2.5.1" + hash-sum "^1.0.2" + loader-utils "^1.1.0" + vue-hot-reload-api "^2.3.0" + vue-style-loader "^4.1.0" + +vue-router@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/vue-router/-/vue-router-3.0.2.tgz#dedc67afe6c4e2bc25682c8b1c2a8c0d7c7e56be" + integrity sha512-opKtsxjp9eOcFWdp6xLQPLmRGgfM932Tl56U9chYTnoWqKxQ8M20N7AkdEbM5beUh6wICoFGYugAX9vQjyJLFg== + +vue-rx@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/vue-rx/-/vue-rx-6.1.0.tgz#75c378ef7d58195344b9bf95379d61497c9863ae" + integrity sha512-2nfkmmoZ1C4hBZ6eC1c2vvLzc3DextyNpXIDLQcPfcSHyarQCClyEO5y5Pr/P7MDzT5mrRzfunlLQaH3/lQoXA== vue-style-loader@^3.0.1: version "3.1.2" @@ -13401,7 +13521,7 @@ vue-style-loader@^3.0.1: hash-sum "^1.0.2" loader-utils "^1.0.2" -vue-style-loader@^4.1.0, vue-style-loader@^4.1.2: +vue-style-loader@^4.1.0: version "4.1.2" resolved "https://registry.yarnpkg.com/vue-style-loader/-/vue-style-loader-4.1.2.tgz#dedf349806f25ceb4e64f3ad7c0a44fba735fcf8" integrity sha512-0ip8ge6Gzz/Bk0iHovU9XAUQaFt/G2B61bnWa2tCcqqdgfHs1lF9xXorFbE55Gmy92okFT+8bfmySuUOu13vxQ== @@ -13478,7 +13598,7 @@ vue-styleguidist@^1.8.9: webpack-dev-server "^2.11.2" webpack-merge "^4.1.3" -vue-template-compiler@2.5.17, vue-template-compiler@^2.4.2, vue-template-compiler@^2.5.16: +vue-template-compiler@^2.4.2, vue-template-compiler@^2.5.16: version "2.5.17" resolved "https://registry.yarnpkg.com/vue-template-compiler/-/vue-template-compiler-2.5.17.tgz#52a4a078c327deb937482a509ae85c06f346c3cb" integrity sha512-63uI4syCwtGR5IJvZM0LN5tVsahrelomHtCxvRkZPJ/Tf3ADm1U1wG6KWycK3qCfqR+ygM5vewUvmJ0REAYksg== @@ -13486,11 +13606,24 @@ vue-template-compiler@2.5.17, vue-template-compiler@^2.4.2, vue-template-compile de-indent "^1.0.2" he "^1.1.0" +vue-template-compiler@^2.5.22: + version "2.5.22" + resolved "https://registry.yarnpkg.com/vue-template-compiler/-/vue-template-compiler-2.5.22.tgz#c3d3c02c65f1908205c4fbd3b0ef579e51239955" + integrity sha512-1VTw/NPTUeHNiwhkq6NkFzO7gYLjFCueBN0FX8NEiQIemd5EUMQ5hxrF7O0zCPo5tae+U9S/scETPea+hIz8Eg== + dependencies: + de-indent "^1.0.2" + he "^1.1.0" + vue-template-es2015-compiler@^1.5.3, vue-template-es2015-compiler@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/vue-template-es2015-compiler/-/vue-template-es2015-compiler-1.6.0.tgz#dc42697133302ce3017524356a6c61b7b69b4a18" integrity sha512-x3LV3wdmmERhVCYy3quqA57NJW7F3i6faas++pJQWtknWT+n7k30F4TVdHvCLn48peTJFRvCpxs3UuFPqgeELg== +vue-template-es2015-compiler@^1.8.2: + version "1.8.2" + resolved "https://registry.yarnpkg.com/vue-template-es2015-compiler/-/vue-template-es2015-compiler-1.8.2.tgz#dd73e80ba58bb65dd7a8aa2aeef6089cf6116f2a" + integrity sha512-cliV19VHLJqFUYbz/XeWXe5CO6guzwd0yrrqqp0bmjlMP3ZZULY7fu8RTC4+3lmHwo6ESVDHFDsvjB15hcR5IA== + vue-webpack-loaders@^1.0.8: version "1.0.8" resolved "https://registry.yarnpkg.com/vue-webpack-loaders/-/vue-webpack-loaders-1.0.8.tgz#1476bfa1feb40401663b5eb72de83ab611f1f047" @@ -13513,11 +13646,21 @@ vue-webpack-loaders@^1.0.8: vue-style-loader "^3.0.1" vue-template-compiler "^2.5.16" -vue@2.5.17, vue@^2.4.2, vue@^2.5.16: +vue@^2.4.2, vue@^2.5.16: version "2.5.17" resolved "https://registry.yarnpkg.com/vue/-/vue-2.5.17.tgz#0f8789ad718be68ca1872629832ed533589c6ada" integrity sha512-mFbcWoDIJi0w0Za4emyLiW72Jae0yjANHbCVquMKijcavBGypqlF7zHRgMa5k4sesdv7hv2rB4JPdZfR+TPfhQ== +vue@^2.5.22: + version "2.5.22" + resolved "https://registry.yarnpkg.com/vue/-/vue-2.5.22.tgz#3bf88041af08b8539c37b268b70ca79245e9cc30" + integrity sha512-pxY3ZHlXNJMFQbkjEgGVMaMMkSV1ONpz+4qB55kZuJzyJOhn6MSy/YZdzhdnumegNzVTL/Dn3Pp4UrVBYt1j/g== + +vuex@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/vuex/-/vuex-3.1.0.tgz#634b81515cf0cfe976bd1ffe9601755e51f843b9" + integrity sha512-mdHeHT/7u4BncpUZMlxNaIdcN/HIt1GsGG5LKByArvYG/v6DvHcOxvDCts+7SRdCoIRGllK8IMZvQtQXLppDYg== + walk-back@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/walk-back/-/walk-back-3.0.0.tgz#2358787a35da91032dad5e92f80b12370d8795c5" diff --git a/lib/galaxy/web/framework/helpers/grids.py b/lib/galaxy/web/framework/helpers/grids.py index 1538ebc3d5d..33184bacc1a 100644 --- a/lib/galaxy/web/framework/helpers/grids.py +++ b/lib/galaxy/web/framework/helpers/grids.py @@ -654,7 +654,7 @@ class CommunityTagsColumn(TextColumn): def get_value(self, trans, grid, item): return trans.fill_template("/tagging_common.mako", tag_type="community", trans=trans, user=trans.get_user(), tagged_item=item, elt_context=self.grid_name, - in_form=True, input_size="20", tag_click_fn="add_tag_to_grid_filter", use_toggle_link=True) + tag_click_fn="add_tag_to_grid_filter", use_toggle_link=True) def filter(self, trans, user, query, column_filter): """ Modify query to filter model_class by tag. Multiple filters are ANDed. """ @@ -690,8 +690,6 @@ class IndividualTagsColumn(CommunityTagsColumn): user=trans.user, tagged_item=item, elt_context=self.grid_name, - in_form=True, - input_size="20", tag_click_fn="add_tag_to_grid_filter", use_toggle_link=True) diff --git a/lib/galaxy/webapps/galaxy/controllers/tag.py b/lib/galaxy/webapps/galaxy/controllers/tag.py index bc3e9dfe6ae..25e74fdf576 100644 --- a/lib/galaxy/webapps/galaxy/controllers/tag.py +++ b/lib/galaxy/webapps/galaxy/controllers/tag.py @@ -30,8 +30,6 @@ class TagsController(BaseUIController, UsesTagsMixin): user=trans.user, tagged_item=item, elt_context=elt_context, - in_form=False, - input_size="22", tag_click_fn="default_tag_click_fn", use_toggle_link=False) diff --git a/lib/galaxy/webapps/reports/framework/grids.py b/lib/galaxy/webapps/reports/framework/grids.py index 1cd9c69a135..8d40c093cad 100644 --- a/lib/galaxy/webapps/reports/framework/grids.py +++ b/lib/galaxy/webapps/reports/framework/grids.py @@ -573,7 +573,7 @@ class CommunityTagsColumn(TextColumn): def get_value(self, trans, grid, item): return trans.fill_template("/tagging_common.mako", tag_type="community", trans=trans, user=trans.get_user(), tagged_item=item, elt_context=self.grid_name, - in_form=True, input_size="20", tag_click_fn="add_tag_to_grid_filter", use_toggle_link=True) + tag_click_fn="add_tag_to_grid_filter", use_toggle_link=True) def filter(self, trans, user, query, column_filter): """ Modify query to filter model_class by tag. Multiple filters are ANDed. """ @@ -609,8 +609,6 @@ class IndividualTagsColumn(CommunityTagsColumn): user=trans.user, tagged_item=item, elt_context=self.grid_name, - in_form=True, - input_size="20", tag_click_fn="add_tag_to_grid_filter", use_toggle_link=True) diff --git a/templates/tagging_common.mako b/templates/tagging_common.mako index 303bc0a3290..5310f40f1c1 100644 --- a/templates/tagging_common.mako +++ b/templates/tagging_common.mako @@ -1,259 +1,124 @@ +<%namespace file="/display_common.mako" import="get_controller_name" /> <%! - from cgi import escape - from random import random - from math import floor - import six - from galaxy.model import Tag, ItemTagAssociation - from galaxy.web.framework.helpers import iff - from galaxy.util import unicodify + +from cgi import escape +import six +## from galaxy.model import Tag, ItemTagAssociation +from galaxy.util import unicodify + +## Build dict of tag name, values. +def tags_to_dict(item_tags): + tag_names_and_values = dict() + for tag in item_tags: + tag_name = escape( tag.user_tname ) + tag_value = "" + if tag.value is not None: + tag_value = escape( tag.user_value ) + + ## Tag names and values may be string or unicode object. + if isinstance( tag_name, six.binary_type ): + tag_names_and_values[unicodify(tag_name, 'utf-8')] = unicodify(tag_value, 'utf-8') + else: + tag_names_and_values[tag_name] = tag_value + return tag_names_and_values + %> ## Render a tagging element if there is a tagged_item. %if tagged_item is not None: %if tag_type == "individual": - ${render_individual_tagging_element( user=user, tagged_item=tagged_item, elt_context=elt_context, in_form=in_form, input_size=input_size, tag_click_fn=tag_click_fn, use_toggle_link=use_toggle_link )} + ${render_individual_tagging_element( + user=user, + tagged_item=tagged_item, + elt_context=elt_context, + tag_click_fn=tag_click_fn, + use_toggle_link=use_toggle_link + )} %elif tag_type == "community": - ${render_community_tagging_element(tagged_item=tagged_item, elt_context=elt_context, tag_click_fn=tag_click_fn)} + ${render_community_tagging_element( + tagged_item=tagged_item, + elt_context=elt_context, + tag_click_fn=tag_click_fn + )} %endif %endif -## Render HTML for a list of tags. -<%def name="render_tagging_element_html(elt_id=None, tags=None, editable=True, use_toggle_link=True, input_size='15', in_form=False, tag_type='individual', render_add_tag_button=True)"> - ## Useful attributes. + +<%def name="render_community_tagging_element( + tagged_item=None, + elt_context=None, + use_toggle_link=False, + tag_click_fn='default_tag_click_fn')"> + <% - num_tags = len( tags ) - %> - - - -## Render tool tagging elements -<%def name="render_tool_tagging_elements()"> - <% - elt_id = int ( floor ( random() * six.MAXSIZE ) ) - tags = trans.app.tag_handler.get_tool_tags() - %> - ${self.render_tagging_element_html(elt_id=elt_id, \ - tags=tags, \ - editable=False, \ - use_toggle_link=False )} - - - -## Render community tagging element. -<%def name="render_community_tagging_element(tagged_item=None, elt_context=None, use_toggle_link=False, tag_click_fn='default_tag_click_fn')"> - ## Build HTML. - <% - elt_id = int ( floor ( random() * six.MAXSIZE ) ) + controller_name = get_controller_name(tagged_item) + click_url = h.url_for( controller='/' + controller_name , action='list_published') community_tags = trans.app.tag_handler.get_community_tags( item=tagged_item, limit=5 ) %> - ${self.render_tagging_element_html(elt_id=elt_id, \ - tags=community_tags, \ - use_toggle_link=use_toggle_link, \ - editable=False, tag_type="community")} + +
- ## Set up tag click function. -## Render individual tagging element. -<%def name="render_individual_tagging_element(user=None, tagged_item=None, elt_context=None, use_toggle_link=True, in_form=False, input_size='15', tag_click_fn='default_tag_click_fn', get_toggle_link_text_fn='default_get_toggle_link_text_fn', editable=True, render_add_tag_button=True)"> - ## Useful attributes. - <% - # Useful ids. - tagged_item_id = str( trans.security.encode_id ( tagged_item.id ) ) - elt_id = int ( floor ( random() * six.MAXSIZE ) ) +<%def name="render_individual_tagging_element( + user=None, + tagged_item=None, + elt_context=None, + use_toggle_link=True, + tag_click_fn='default_tag_click_fn', + get_toggle_link_text_fn='default_get_toggle_link_text_fn', + editable=True)"> - # Get list of user's item tags. TODO: implement owner_tags for all taggable objects and use here. + <% + tagged_item_id = str( trans.security.encode_id ( tagged_item.id ) ) item_tags = [ tag for tag in tagged_item.tags if ( tag.user == user ) ] %> - ## Build HTML. - <% - if len(item_tags) > 3: - # If item has more than 3 tags show a link to see tags instead of displaying them all - use_toggle_link = True - else: - use_toggle_link = False - %> - ${self.render_tagging_element_html(elt_id=elt_id, tags=item_tags, editable=editable, use_toggle_link=use_toggle_link, input_size=input_size, in_form=in_form, render_add_tag_button=render_add_tag_button)} +
- ## Build script that augments tags using progressive javascript. - ## Use style to hide/display the tag area. - - - <%def name="community_tag_js( controller_name )"> +## TODO: Note that this function no longer has anything to do with community +## tags. the ratings code and tagging initialization were previously co-mingled +## in here. Will remove this script when we write the ratings components + ## set up comminity tag and rating handling - used for page start up / set up ## controller_name: the model controller for the item being tagged - generally gotten with get_controller_name( item ) - + diff --git a/client/galaxy/scripts/components/Tags/tagStyles.scss b/client/galaxy/scripts/components/Tags/tagStyles.scss deleted file mode 100644 index a856ec8ecf8..00000000000 --- a/client/galaxy/scripts/components/Tags/tagStyles.scss +++ /dev/null @@ -1,77 +0,0 @@ -// TODO: use general color definition file -@import "theme/blue"; -@import "scss/mixins"; - -// Puts a little graphic in place of the text-input -// when the input is not in focus -@mixin newTagHoverButton() { - .vue-tags-input .ti-tags .ti-new-tag-input-wrapper { - input { - background-color: transparent; - } - input:not(:focus) { - background: url("/static/images/fugue/tag--plus.png"); - background-repeat: no-repeat; - color: transparent; - &::placeholder { - color: transparent; - } - } - } -} - -// hides tag container edges -@mixin hideEditorBorders() { - .vue-tags-input { - // need to add yet another class to beat the scoping - &.tag-area { - background-color: transparent; - } - .ti-input { - border: none; - } - } -} - -// general style butchering -@mixin matchBootstrapStyling() { - .vue-tags-input { - @include fill(); - .ti-input { - padding: 0; - } - .ti-tag { - border-radius: 4px; - font-size: 0.8rem; - font-weight: 400; - } - } -} - -// Version of the tags that only allow clicking -// existing tags instead of the full editing UI -@mixin forDisplayOnly() { - .vue-tags-input { - .ti-actions, - .ti-new-tag-input-wrapper { - display: none; - } - } -} - -.galaxy-tags { - - // adds in a graphic in place of the text input - @include newTagHoverButton(); - - // match bootstrap tag styles/colors - @include matchBootstrapStyling(); - - // removes input borders (not sure if this happens everywhere) - @include hideEditorBorders(); - - // display-only tags - &.disabled { - @include forDisplayOnly(); - } -} From 51280fbf7d345deb6d2cc0aefd4f0bbe598a92c1 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Mon, 18 Feb 2019 15:26:31 -0500 Subject: [PATCH 0098/1016] flip a line --- lib/galaxy/datatypes/anvio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/anvio.py b/lib/galaxy/datatypes/anvio.py index 37c7a3c1256..7cdc42ff250 100644 --- a/lib/galaxy/datatypes/anvio.py +++ b/lib/galaxy/datatypes/anvio.py @@ -7,8 +7,8 @@ import logging import os import sys -from galaxy.datatypes.text import Html from galaxy.datatypes.metadata import MetadataElement +from galaxy.datatypes.text import Html log = logging.getLogger(__name__) From a22142437df527342d72cde51f2f79c667fe0da4 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 15 Feb 2019 07:18:00 +0000 Subject: [PATCH 0099/1016] Use mysqlclient instead of mysql-python as conditional dependency MySQL-python is legacy and unmaintained. xref. https://github.com/galaxyproject/starforge-recipes/issues/22 --- lib/galaxy/dependencies/__init__.py | 2 +- lib/galaxy/dependencies/conditional-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index bda5016208d..553b0a29615 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -90,7 +90,7 @@ class ConditionalDependencies(object): def check_psycopg2_binary(self): return self.config["database_connection"].startswith("postgres") - def check_mysql_python(self): + def check_mysqlclient(self): return self.config["database_connection"].startswith("mysql") def check_drmaa(self): diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index 662871a7b51..b8c19a41cf5 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -1,7 +1,7 @@ # These dependencies are only required when certain config options are set psycopg2-binary==2.7.4 weberror==0.10.3 -mysql-python +mysqlclient fluent-logger raven pbs_python From c6f1183939cf8b9897357e4506bfc6d438067f36 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 15 Feb 2019 08:13:00 +0000 Subject: [PATCH 0100/1016] Update documentation about dependencies --- client/README.md | 24 ++++++++++----------- doc/source/admin/framework_dependencies.rst | 13 ++++++----- scripts/common_startup.sh | 2 +- 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/client/README.md b/client/README.md index 1eee5fbe0c2..3094d6b69d7 100644 --- a/client/README.md +++ b/client/README.md @@ -2,32 +2,30 @@ Client Build System =================== Installs, stages, and builds the client-side scripts necessary for running the -Galaxy webapp. There's no need to use this system unless you are modifying or -developing client-side scripts, or are running the development branch of -Galaxy. When started through `run.sh` or any other method that utilizes -`common_startup.sh`, Galaxy will also (since 18.09) *automatically* build +Galaxy webapp. When started through `run.sh` or any other method that utilizes +`scripts/common_startup.sh`, Galaxy will (since 18.09) *automatically* build the client as a part of server startup, when it detects changes, unless that functionality is explicitly disabled. The base dependencies used are Node.js and Yarn. Galaxy now includes these in the virtual environment, and they can be accessed by activating that with -`source .venv/bin/activate` from the Galaxy root directory. +`. .venv/bin/activate` from the Galaxy root directory. If you'd like to install your own dependencies, on OSX the easiest way to get -set up is using homebrew and the command `brew install nodejs yarn`. More -information including instructions for other platforms is available at -nodejs.org and yarnpkg.com. +set up is using `homebrew` and the command `brew install nodejs yarn`. More +information, including instructions for other platforms, is available at +https://nodejs.org/ and https://yarnpkg.com/ . The Galaxy client build has necessarily grown more complex in the past several -years, but we're still trying to keep things as simple as possible for -developers (everyone, really). If you're having any trouble with building the -client after following the instructions below please create an issue on GitHub +years, but we are still trying to keep things as simple as possible for +everyone. If you're having any trouble with building the +client after following the instructions below, please create an issue on GitHub or reach out for help directly on Gitter at -https://gitter.im/galaxyproject/Lobby. +https://gitter.im/galaxyproject/Lobby . Complete Client Build -================================================ +===================== There are many moving parts to the client build system, but the entry point for most people is the 'client' rule in the Makefile at the root of the Galaxy diff --git a/doc/source/admin/framework_dependencies.rst b/doc/source/admin/framework_dependencies.rst index c65100ef31d..50558123768 100644 --- a/doc/source/admin/framework_dependencies.rst +++ b/doc/source/admin/framework_dependencies.rst @@ -8,9 +8,8 @@ developers have made significant effort to provide these dependencies in as simp compatible with the `Python packaging best practices`_. Thus, Galaxy's runtime setup procedure makes use of virtualenv_ for package environment isolation, pip_ for installation, and wheel_ to provide pre-built versions of dependencies. -In addition to framework dependencies, as of Galaxy 18.01, the client (UI) is no longer provided in its built format -**in the development (dev) branch of the source repository**. The built client is still provided in -``release_YY.MM`` branches and the ``master`` branch. +In addition to framework dependencies, since Galaxy 18.09, the client (UI) is no longer provided in its built format +when downloading Galaxy. For more information, see https://github.com/galaxyproject/galaxy/blob/dev/client/README.md . .. _Python module dependencies: https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt .. _Python packaging best practices: https://packaging.python.org @@ -27,12 +26,12 @@ Upon startup (with ``run.sh``), the startup scripts will: 2. Unset the ``$PYTHONPATH`` environment variable (if set) as this can interfere with installing dependencies. -3. Download and install packages from the Galaxy project wheel server, wheels.galaxyproject.org_, as well as the `Python +3. Download and install packages from the `Galaxy project wheel server`_, as well as the `Python Package Index`_ (aka PyPI) , using pip_. 4. Start Galaxy using ``.venv/bin/python``. -.. _wheels.galaxyproject.org: https://wheels.galaxyproject.org/ +.. _Galaxy project wheel server: https://wheels.galaxyproject.org/ .. _Python Package Index: https://pypi.org Options @@ -175,7 +174,7 @@ Conda These instruction apply to Galaxy release 19.01 or newer. Please consult the documentation for your version of Galaxy. -`Conda`_ and `virtualenv`_ are incompatible, unless an adapted `virtualenv_` from the `conda-forge_` channel is used. +`Conda`_ and `virtualenv`_ are incompatible, unless an adapted ``virtualenv`` package from the `conda-forge`_ channel is used. Galaxy can create a virtualenv using the adapted virtualenv package. Once a valid ``.venv`` environment exists it will be used. .. tip:: @@ -217,7 +216,7 @@ Galaxy's dependencies can be installed either "pinned" (they will be installed a Galaxy release) or "unpinned" (the latest versions of all dependencies will be installed unless there are known incompatibilities with new versions). By default, the release branches of Galaxy use pinned versions for three reasons: -1. Using pinned versions insures that the prebuilt wheels on `wheels.galaxyproject.org`_ will be installed, and no +1. Using pinned versions insures that the prebuilt wheels will be installed, and no compilation will be necessary. 2. Galaxy releases are tested with the pinned versions and this allows us to give as much assurance as possible that the diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index 8b0b94ec224..4162945bf7c 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -113,7 +113,7 @@ if [ $SET_VENV -eq 1 -a $CREATE_VENV -eq 1 ]; then set_conda_exe if [ -n "$CONDA_EXE" ]; then echo "Found Conda, will set up a virtualenv using conda." - echo "To use a virtualenv instead, create one with a non-Conda Python 2.7 at $GALAXY_VIRTUAL_ENV" + echo "To use a virtualenv instead, create one with a non-Conda Python at $GALAXY_VIRTUAL_ENV" : ${GALAXY_CONDA_ENV:="_galaxy_"} if [ "$CONDA_DEFAULT_ENV" != "$GALAXY_CONDA_ENV" ]; then if ! check_conda_env "$GALAXY_CONDA_ENV"; then From b84c61a12a4e6c2aac6c223d668dae4a722cbf60 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 13 Feb 2019 10:21:19 +0000 Subject: [PATCH 0101/1016] Remove pin on Sphinx. Update all dependencies with `make update-dependencies` using pipenv 2018.11.26 . Important updates: - Cheetah3 3.2.0 has now x86_64 wheels on PyPI - `sphinx-rtd-theme` 0.4.3 has integrated Google Analytics support that we can use for the docs website - Fix environment markers for `configparser`, `functools32` and `more-itertools` --- .../dependencies/pipfiles/default/Pipfile | 2 +- .../default/pinned-dev-requirements.txt | 14 ++++++----- .../pipfiles/default/pinned-requirements.txt | 24 +++++++++---------- .../pipfiles/flake8/pinned-requirements.txt | 3 ++- lib/galaxy/dependencies/pipfiles/update.sh | 3 +++ 5 files changed, 26 insertions(+), 20 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index 6d0b0b5001e..568a7c4428e 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -19,7 +19,7 @@ pytest-html = "*" pytest-pythonpath = "*" recommonmark = "*" selenium = "*" -Sphinx = "<1.7" # Docs do not build with sphinx 1.7 for some reason, can't find issue. +Sphinx = "*" sphinx_markdown_tables = "*" sphinx_rtd_theme = "*" testfixtures = "*" diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt index 6bfc7b9b86f..b362b630330 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt @@ -2,7 +2,7 @@ --extra-index-url https://pypi.python.org/simple alabaster==0.7.12 argh==0.26.2 -atomicwrites==1.2.1 +atomicwrites==1.3.0 attrs==18.2.0 babel==2.6.0 certifi==2018.11.29 @@ -15,13 +15,14 @@ gunicorn==19.9.0 idna==2.8 imagesize==1.1.0 jinja2==2.10 -lxml==4.3.0 +lxml==4.3.1 markdown==2.6.11 markupsafe==1.1.0 mock==2.0.0 more-itertools==5.0.0 nose==1.3.7 nosehtml==0.4.5 +packaging==19.0 pathlib2==2.3.3 ; python_version < '3.6' pathtools==0.1.2 pbr==5.1.2 @@ -29,10 +30,11 @@ pluggy==0.8.1 py==1.7.0 pygithub3==0.5.1 ; python_version < '3' pygments==2.3.1 +pyparsing==2.3.1 pytest-html==1.20.0 pytest-metadata==1.8.0 pytest-pythonpath==0.7.3 -pytest==4.2.0 +pytest==4.3.0 pytz==2018.9 pyyaml==3.13 recommonmark==0.5.0 @@ -42,10 +44,10 @@ selenium==3.141.0 six==1.11.0 snowballstemmer==1.2.1 sphinx-markdown-tables==0.0.9 -sphinx-rtd-theme==0.4.2 -sphinx==1.6.7 +sphinx-rtd-theme==0.4.3 +sphinx==1.8.4 sphinxcontrib-websupport==1.1.0 -testfixtures==6.5.0 +testfixtures==6.5.2 twill==0.9.1 ; python_version < '3' typing==3.6.6 ; python_version < '3.5' urllib3==1.24.1 diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 0f0891ed272..59c0e44ab18 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -1,7 +1,7 @@ -i https://wheels.galaxyproject.org/simple --extra-index-url https://pypi.python.org/simple adal==1.2.1 -amqp==2.4.0 +amqp==2.4.1 appdirs==1.4.3 asn1crypto==0.24.0 azure-common==1.1.14 @@ -24,7 +24,7 @@ bdbag==1.4.1 beaker==1.10.0 bioblend==0.12.0 bleach==3.1.0 -boltons==18.0.1 +boltons==19.0.1 boto3==1.7.84 boto==2.49.0 botocore==1.10.84 @@ -33,9 +33,9 @@ bx-python==0.8.2 bz2file==0.98 ; python_version < '3.3' cachetools==3.1.0 certifi==2018.11.29 -cffi==1.11.5 +cffi==1.12.1 chardet==3.0.4 -cheetah3==3.1.0 +cheetah3==3.2.0 cliff==2.14.0 cloudauthz==0.2.0 cloudbridge==1.0.2 @@ -52,7 +52,7 @@ dogpile.cache==0.7.1 enum34==1.1.6 ; python_version < '3.4' fabric3==1.14.post1 funcsigs==1.0.2 ; python_version < '3.3' -functools32==3.2.3.post2 ; python_version == '2.7' +functools32==3.2.3.post2 ; python_version < '3.2' future==0.17.1 futures==3.2.0 ; python_version == '2.6' or python_version == '2.7' galaxy-sequence-utils==1.1.3 @@ -68,7 +68,7 @@ jsonpatch==1.23 jsonpointer==2.0 jsonschema==2.6.0 keystoneauth1==3.11.2 -kombu==4.2.2.post1 +kombu==4.3.0 mako==1.0.7 markupsafe==1.1.0 mercurial==3.7.3 ; python_version < '3' @@ -82,7 +82,7 @@ netifaces==0.10.9 networkx==1.11 nodeenv==1.3.3 nose==1.3.7 -numpy==1.16.0 +numpy==1.16.1 oauthlib==3.0.1 openstacksdk==0.17.0 os-client-config==1.31.2 @@ -93,7 +93,7 @@ oslo.context==2.22.0 oslo.i18n==3.23.0 oslo.log==3.42.2 oslo.serialization==2.28.1 -oslo.utils==3.40.1 +oslo.utils==3.40.2 packaging==19.0 paramiko==2.4.2 parsley==1.3 @@ -102,7 +102,7 @@ pastedeploy==2.0.1 pastescript==3.0.0 pbr==5.1.2 prettytable==0.7.2 -psutil==5.5.0 +psutil==5.5.1 pulsar-galaxy-lib==0.8.3 pyasn1==0.4.5 pycparser==2.19 @@ -119,7 +119,7 @@ pyperclip==1.7.0 pysam==0.15.2 pysftp==0.2.9 python-cinderclient==4.0.0 -python-dateutil==2.7.5 +python-dateutil==2.8.0 python-genomespaceclient==1.1.0 python-glanceclient==2.12.0 python-keystoneclient==3.17.0 @@ -142,7 +142,7 @@ six==1.11.0 social-auth-core[openidconnect]==1.5.0 sqlalchemy-migrate==0.12.0 sqlalchemy-utils==0.33.11 -sqlalchemy==1.2.17 +sqlalchemy==1.2.18 sqlparse==0.2.4 stevedore==1.30.0 subprocess32==3.5.3 ; python_version < '3.0' @@ -153,7 +153,7 @@ typing==3.6.6 ; python_version < '3.5' tzlocal==1.5.1 unicodecsv==0.14.1 ; python_version < '3.0' urllib3==1.24.1 -uwsgi==2.0.17.1 +uwsgi==2.0.18 vine==1.2.0 warlock==1.3.0 wcwidth==0.1.7 ; sys_platform != 'win32' diff --git a/lib/galaxy/dependencies/pipfiles/flake8/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/flake8/pinned-requirements.txt index 4be76b34a6c..a71eebf35b9 100644 --- a/lib/galaxy/dependencies/pipfiles/flake8/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/flake8/pinned-requirements.txt @@ -3,7 +3,8 @@ configparser==3.7.1 ; python_version < '3.2' entrypoints==0.3 enum34==1.1.6 ; python_version < '3.4' flake8-import-order==0.18 -flake8==3.7.3 +flake8==3.7.6 +functools32==3.2.3.post2 ; python_version < '3.2' mccabe==0.6.1 pycodestyle==2.5.0 pyflakes==2.1.0 diff --git a/lib/galaxy/dependencies/pipfiles/update.sh b/lib/galaxy/dependencies/pipfiles/update.sh index 711e0595388..2f96969d9e5 100755 --- a/lib/galaxy/dependencies/pipfiles/update.sh +++ b/lib/galaxy/dependencies/pipfiles/update.sh @@ -45,10 +45,13 @@ for env in $ENVS; do sed -i.orig -e "s/^azure-storage-nspkg==\([^ ;]\{1,\}\).*$/azure-storage-nspkg==\1/" \ -e "s/^cffi==\([^ ;]\{1,\}\).*$/cffi==\1/" \ -e "s/^cmd2==\([^ ;]\{1,\}\).*$/cmd2==\1/" \ + -e "s/^configparser==\([^ ;]\{1,\}\).*$/configparser==\1 ; python_version < '3.2'/" \ -e "s/^enum34==\([^ ;]\{1,\}\).*$/enum34==\1 ; python_version < '3.4'/" \ -e "s/^funcsigs==\([^ ;]\{1,\}\).*$/funcsigs==\1 ; python_version < '3.3'/" \ + -e "s/^functools32==\([^ ;]\{1,\}\).*$/functools32==\1 ; python_version < '3.2'/" \ -e "s/^futures==\([^ ;]\{1,\}\).*$/futures==\1 ; python_version == '2.6' or python_version == '2.7'/" \ -e "s/^monotonic==\([^ ;]\{1,\}\).*$/monotonic==\1/" \ + -e "s/^more-itertools==\([^ ;]\{1,\}\).*$/more-itertools==\1/" \ -e "s/^py2-ipaddress==\([^ ;]\{1,\}\).*$/py2-ipaddress==\1 ; python_version < '3'/" \ -e "s/^pyinotify==\([^ ;]\{1,\}\).*$/pyinotify==\1 ; sys_platform != 'win32' and sys_platform != 'darwin' and sys_platform != 'sunos5'/" \ -e "s/^python-dateutil==\([^ ;]\{1,\}\).*$/python-dateutil==\1/" \ From 737bae8f9b920ec9bed43a2d25378b19ace12f58 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 19 Feb 2019 00:43:12 +0000 Subject: [PATCH 0102/1016] Reword HDF5 attribute test docs --- lib/galaxy/tools/xsd/galaxy.xsd | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/tools/xsd/galaxy.xsd b/lib/galaxy/tools/xsd/galaxy.xsd index 93fd65ca6eb..c0da4562fdd 100644 --- a/lib/galaxy/tools/xsd/galaxy.xsd +++ b/lib/galaxy/tools/xsd/galaxy.xsd @@ -1659,26 +1659,26 @@ module. - ``).]]> + ``).]]> - Comma separated list of keys to check for. + Comma-separated list of HDF5 attributes to check for. - ``).]]> + ``).]]> - Key to check H5 attribute value of. + HDF5 attribute to check value of. - Expected value of H5 attribute to check. + Expected value of HDF5 attribute to check. From 6678a6baf0c61c7538425c7c82d0fadb6f27a3ea Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 18 Feb 2019 18:10:51 +0100 Subject: [PATCH 0103/1016] Fix display of images in static/images --- lib/galaxy/tools/__init__.py | 9 --------- lib/galaxy/tools/data_manager/manager.py | 3 ++- .../galaxy/controllers/shed_tool_static.py | 16 ++++++++++------ 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index c6c39ff27a1..ca66a95bdcc 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -422,7 +422,6 @@ class Tool(Dictifiable): # tool_data_table_conf.xml entries exist. self.input_params = [] # Attributes of tools installed from Galaxy tool sheds. - self._repository_path = None self.tool_shed = None self.repository_name = None self.repository_owner = None @@ -493,14 +492,6 @@ class Tool(Dictifiable): installed_changeset_revision=self.installed_changeset_revision, repository_id=self.repository_id) - @property - def repository_path(self): - if self._repository_path is None: - repository = self.tool_shed_repository - if repository: - self._repository_path = repository.repo_path(self.app) - return self._repository_path - @property def produces_collections_with_unknown_structure(self): diff --git a/lib/galaxy/tools/data_manager/manager.py b/lib/galaxy/tools/data_manager/manager.py index 4437a8e0002..978bd9aa423 100644 --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -260,12 +260,13 @@ class DataManager(object): def id(self): return self.guid or self.declared_id # if we have a guid, we will use that as the data_manager id - def load_tool(self, tool_filename, guid=None, data_manager_id=None, tool_shed_repository_id=None): + def load_tool(self, tool_filename, guid=None, data_manager_id=None, tool_shed_repository_id=None, tool_shed_repository=None): toolbox = self.data_managers.app.toolbox tool = toolbox.load_hidden_tool(tool_filename, guid=guid, data_manager_id=data_manager_id, repository_id=tool_shed_repository_id, + tool_shed_repository=tool_shed_repository, use_cached=True) self.data_managers.app.toolbox.data_manager_tools[tool.id] = tool self.tool = tool diff --git a/lib/galaxy/webapps/galaxy/controllers/shed_tool_static.py b/lib/galaxy/webapps/galaxy/controllers/shed_tool_static.py index c6f8c719fa7..258eaed3a21 100644 --- a/lib/galaxy/webapps/galaxy/controllers/shed_tool_static.py +++ b/lib/galaxy/webapps/galaxy/controllers/shed_tool_static.py @@ -1,5 +1,5 @@ import logging -from os.path import splitext +import os from galaxy import web from galaxy.exceptions import RequestParameterInvalidException @@ -26,13 +26,17 @@ class ShedToolStatic(BaseUIController): """ guid = '/'.join([shed, 'repos', owner, repo, tool, version]) tool = trans.app.toolbox.get_tool(guid) - repo_path = tool.repository_path - path = join(repo_path, image_file) - if not safe_contains(repo_path, path): + repo_path = tool._repository_dir + if 'static/images' not in image_file: + path = join(repo_path, 'static', 'images', image_file) + else: + path = join(repo_path, image_file) + if not safe_contains(os.path.abspath(repo_path), os.path.abspath(path)): raise RequestParameterInvalidException() - ext = splitext(image_file)[-1].lstrip('.') + ext = os.path.splitext(image_file)[-1].lstrip('.') if ext: mime = trans.app.datatypes_registry.get_mimetype_by_extension(ext) if mime: trans.response.set_content_type(mime) - return open(path, 'rb') + if os.path.exists(path): + return open(path, 'rb') From 99447a9d299e8e8e52ff0367144bb9665627ccf3 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 18 Feb 2019 19:09:15 +0100 Subject: [PATCH 0104/1016] Add dummy tool shed registry --- lib/galaxy/tools/repositories.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/tools/repositories.py b/lib/galaxy/tools/repositories.py index 2766be49d79..00b82fd8d6b 100644 --- a/lib/galaxy/tools/repositories.py +++ b/lib/galaxy/tools/repositories.py @@ -30,6 +30,7 @@ class ValidationContext(object): self.config.tool_data_table_config = os.path.join(self.temporary_path, 'tool_data_table_conf.xml') self.config.shed_tool_data_table_config = os.path.join(self.temporary_path, 'shed_tool_data_table_conf.xml') self.tool_data_tables = tool_data_tables + self.tool_shed_registry = Bunch(tool_sheds={}) self.datatypes_registry = registry self.hgweb_config_manager = hgweb_config_manager self.config.len_file_path = os.path.join(self.temporary_path, 'chromlen.txt') From 0d6f4b13837e52371ddf12881375639cb198ef33 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 18 Feb 2019 19:09:45 +0100 Subject: [PATCH 0105/1016] Remove useless lock --- lib/tool_shed/util/readme_util.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/tool_shed/util/readme_util.py b/lib/tool_shed/util/readme_util.py index 778e5789b51..d3624168b2a 100644 --- a/lib/tool_shed/util/readme_util.py +++ b/lib/tool_shed/util/readme_util.py @@ -1,7 +1,6 @@ import json import logging import os -import threading from mako.template import Template @@ -50,16 +49,12 @@ def build_readme_files_dict(app, repository, changeset_revision, metadata, tool_ text_of_reasonable_length = basic_util.size_string(text) if text_of_reasonable_length.find('.. image:: ') >= 0: # Handle image display for README files that are contained in repositories in the tool shed or installed into Galaxy. - lock = threading.Lock() - lock.acquire(True) try: text_of_reasonable_length = suc.set_image_paths(app, text_of_reasonable_length, encoded_repository_id=app.security.encode_id(repository.id)) except Exception: log.exception("Exception in build_readme_files_dict, so images may not be properly displayed") - finally: - lock.release() if readme_file_name.endswith('.rst'): text_of_reasonable_length = Template(rst_to_html(text_of_reasonable_length), input_encoding='utf-8', From 1b40fa9a3313afc564e022a75a045faa4e18b02e Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 19 Feb 2019 11:47:34 +0100 Subject: [PATCH 0106/1016] verify bam: Exception strings are formatted first --- lib/galaxy/tools/verify/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tools/verify/__init__.py b/lib/galaxy/tools/verify/__init__.py index 56cef0bea28..38575960d1b 100644 --- a/lib/galaxy/tools/verify/__init__.py +++ b/lib/galaxy/tools/verify/__init__.py @@ -107,7 +107,7 @@ def verify( local_fh, temp_name = _bam_to_sam(local_name, temp_name) local_name = local_fh.name except Exception as e: - log.warning( "Conversion BAM to SAM failed for local or temp file. Will compare BAM files") + log.warning( "%s. Will compare BAM files" %e) if compare == 'diff': files_diff(local_name, temp_name, attributes=attributes) elif compare == 're_match': @@ -153,11 +153,13 @@ def _bam_to_sam(local_name, temp_name): try: pysam.view('-h', '-o%s' % temp_local.name, local_name) except Exception as e: - raise Exception("Converting local (test-data) BAM to SAM failed: %s" % e) + msg = "Converting local (test-data) BAM to SAM failed: %s" % e + raise Exception(msg) try: pysam.view('-h', '-o%s' % temp_temp, temp_name) except Exception as e: - raise Exception("Converting history BAM to SAM failed: %s" % e) + msg = "Converting history BAM to SAM failed: %s" % e + raise Exception(msg) os.remove(temp_name) return temp_local, temp_temp From 0b6e9cfd7c55f9eedde9a37719fa477ac8839fd8 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 19 Feb 2019 10:03:57 +0100 Subject: [PATCH 0107/1016] Mock the mock shed registry --- test/unit/unittest_utils/galaxy_mock.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/unittest_utils/galaxy_mock.py b/test/unit/unittest_utils/galaxy_mock.py index 9a28c8e4174..06aff87ffa1 100644 --- a/test/unit/unittest_utils/galaxy_mock.py +++ b/test/unit/unittest_utils/galaxy_mock.py @@ -76,6 +76,7 @@ class MockApp(object): self.dataset_collections_service = None self.container_finder = NullContainerFinder() self._toolbox_lock = MockLock() + self.tool_shed_registry = Bunch(tool_sheds={}) self.genome_builds = GenomeBuilds(self) self.job_manager = NoopManager() self.application_stack = ApplicationStack() From 694dde1b07099fdf4a1b400eb5054fee59335ae7 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 19 Feb 2019 10:15:21 +0100 Subject: [PATCH 0108/1016] Fix set_image_path for tools in tool shed --- lib/galaxy/tools/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index ca66a95bdcc..6793787d9ba 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1208,9 +1208,9 @@ class Tool(Dictifiable): help_text = tool_source.parse_help() if help_text is not None: try: - if self.tool_shed_repository and help_text.find('.. image:: ') >= 0: + if help_text.find('.. image:: ') >= 0 and (self.tool_shed_repository or self.repository_id): help_text = tool_shed.util.shed_util_common.set_image_paths( - self.app, help_text, tool_shed_repository=self.tool_shed_repository, tool_id=self.old_id, tool_version=self.version + self.app, help_text, encoded_repository_id=self.repository_id, tool_shed_repository=self.tool_shed_repository, tool_id=self.old_id, tool_version=self.version ) except Exception: log.exception("Exception in parse_help, so images may not be properly displayed") From 8b9239172acc43969946b00c8f3fb94e403803d9 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 19 Feb 2019 10:59:02 +0100 Subject: [PATCH 0109/1016] Simplify getting changeset revision from path --- lib/galaxy/tools/toolbox/base.py | 48 ++++++++++---------------------- 1 file changed, 14 insertions(+), 34 deletions(-) diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index 470a66ce3af..e06e51bb186 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -40,8 +40,8 @@ log = logging.getLogger(__name__) ToolConfRepository = namedtuple( 'ToolConfRepository', ( - 'tool_shed', 'name', 'owner', 'installed_changeset_revision', 'changeset_revision', 'repo_path', - 'tool_dependencies_installed_or_in_error', 'id' + 'tool_shed', 'name', 'owner', 'installed_changeset_revision', 'changeset_revision', + 'tool_dependencies_installed_or_in_error', ) ) @@ -570,7 +570,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): # In that case recreating the tool will correct the cached version. from_cache = False if guid and not from_cache: # tool was not in cache and is a tool shed tool - tool_shed_repository = self.get_tool_repository_from_xml_item(item, path, concrete_path) + tool_shed_repository = self.get_tool_repository_from_xml_item(item, path) if tool_shed_repository: if hasattr(tool_shed_repository, 'deleted'): # The shed tool is in the install database @@ -605,7 +605,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): except Exception: log.exception("Error reading tool from path: %s", path) - def get_tool_repository_from_xml_item(self, item, path, concrete_path): + def get_tool_repository_from_xml_item(self, item, path): tool_shed = item.elem.find("tool_shed").text repository_name = item.elem.find("repository_name").text repository_owner = item.elem.find("repository_owner").text @@ -613,33 +613,8 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): if installed_changeset_revision_elem is None: # Backward compatibility issue - the tag used to be named 'changeset_revision'. installed_changeset_revision_elem = item.elem.find("changeset_revision") - installed_changeset_revision = installed_changeset_revision_elem.text + installed_changeset_revision = changeset_revision = installed_changeset_revision_elem.text if "/repos/" in path: # The only time "/repos/" should not be in path is during testing! - try: - tool_shed_path, reduced_path = path.split('/repos/', 1) - splitted_path = reduced_path.split('/') - assert tool_shed_path == tool_shed - assert splitted_path[0] == repository_owner - assert splitted_path[1] == repository_name - if splitted_path[2] != installed_changeset_revision: - # This can happen if the Tool Shed repository has been - # updated to a new revision and the installed_changeset_revision - # element in shed_tool_conf.xml file has been updated too - log.debug("The installed_changeset_revision for tool %s is %s, using %s instead", path, - installed_changeset_revision, splitted_path[2]) - installed_changeset_revision = splitted_path[2] - except AssertionError: - log.debug("Error while loading tool %s", path) - pass - repository = self._get_tool_shed_repository(tool_shed=tool_shed, - name=repository_name, - owner=repository_owner, - installed_changeset_revision=installed_changeset_revision) - if not repository: - msg = "Attempted to load tool shed tool, but the repository with name '%s' from owner '%s' was not found " \ - "in database. Tool will be loaded without install database." - log.warning(msg, repository_name, repository_owner) - # Pull the changeset_revision from the path, and determine the base repo path. pre = '{shed}/repos/{owner}/{name}/'.format( shed=tool_shed, owner=repository_owner, @@ -649,11 +624,16 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): changeset_revision, name2 = path[path.index(pre) + len(pre):].split('/')[0:2] except ValueError as exc: raise Exception("Cannot determine changeset revision from path '%s': %s" % (path, exc)) - pre = '/'.join([pre.rstrip('/'), changeset_revision, name2]) - repo_path = concrete_path[:concrete_path.index(pre) + len(pre)] + repository = self._get_tool_shed_repository(tool_shed=tool_shed, + name=repository_name, + owner=repository_owner, + installed_changeset_revision=installed_changeset_revision) + if not repository: + msg = "Attempted to load tool shed tool, but the repository with name '%s' from owner '%s' was not found " \ + "in database. Tool will be loaded without install database." + log.warning(msg, repository_name, repository_owner) repository = ToolConfRepository( - tool_shed, repository_name, repository_owner, installed_changeset_revision, changeset_revision, - repo_path, None, None + tool_shed, repository_name, repository_owner, installed_changeset_revision, changeset_revision, None, ) self.app.tool_shed_repository_cache.add_local_repository(repository) return repository From 2e57760086bba24df25b1c0e2650be3c90e90ecb Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 19 Feb 2019 11:48:13 +0100 Subject: [PATCH 0110/1016] Simplify some more --- lib/galaxy/tools/toolbox/base.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index e06e51bb186..0f9337340ee 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -613,17 +613,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): if installed_changeset_revision_elem is None: # Backward compatibility issue - the tag used to be named 'changeset_revision'. installed_changeset_revision_elem = item.elem.find("changeset_revision") - installed_changeset_revision = changeset_revision = installed_changeset_revision_elem.text - if "/repos/" in path: # The only time "/repos/" should not be in path is during testing! - pre = '{shed}/repos/{owner}/{name}/'.format( - shed=tool_shed, - owner=repository_owner, - name=repository_name - ) - try: - changeset_revision, name2 = path[path.index(pre) + len(pre):].split('/')[0:2] - except ValueError as exc: - raise Exception("Cannot determine changeset revision from path '%s': %s" % (path, exc)) + installed_changeset_revision = installed_changeset_revision_elem.text repository = self._get_tool_shed_repository(tool_shed=tool_shed, name=repository_name, owner=repository_owner, @@ -633,7 +623,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): "in database. Tool will be loaded without install database." log.warning(msg, repository_name, repository_owner) repository = ToolConfRepository( - tool_shed, repository_name, repository_owner, installed_changeset_revision, changeset_revision, None, + tool_shed, repository_name, repository_owner, installed_changeset_revision, installed_changeset_revision, None, ) self.app.tool_shed_repository_cache.add_local_repository(repository) return repository From eca238d7837928c7ee962f10435fd667d61ccc4c Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 30 Oct 2018 16:28:28 +0100 Subject: [PATCH 0111/1016] Make pykube a conditional requirement --- config/job_conf.xml.sample_advanced | 5 ----- lib/galaxy/dependencies/__init__.py | 3 +++ lib/galaxy/dependencies/conditional-requirements.txt | 3 +++ 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/config/job_conf.xml.sample_advanced b/config/job_conf.xml.sample_advanced index a1b7a7db096..fb8a24b2c70 100644 --- a/config/job_conf.xml.sample_advanced +++ b/config/job_conf.xml.sample_advanced @@ -120,11 +120,6 @@ /path/to/kubeconfig @@ -191,14 +191,12 @@ Kubernetes 1.2 and on. Changing this a much newer version in the future might require changes to the plugin runner code. Value extensions/v1beta1 is also supported for pre 1.2 legacy installations. --> - galaxy_pvc - + galaxy_pvc1:/mount_point1,galaxy_pvc2:/mount_point2 + + Typical mount paths are the file_path, job_working_directory, all paths containing tools and scripts + and all library paths set in galaxy.yml (or equivalent general galaxy config file). --> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 564d0d7f9584f09c2ebf1af1f24c61c4226132c5 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 21 Feb 2019 14:08:00 -0500 Subject: [PATCH 0162/1016] choose better exceptions class Co-Authored-By: martenson --- lib/galaxy/webapps/galaxy/api/users.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/users.py b/lib/galaxy/webapps/galaxy/api/users.py index 93e82287f98..54c93ca08fc 100644 --- a/lib/galaxy/webapps/galaxy/api/users.py +++ b/lib/galaxy/webapps/galaxy/api/users.py @@ -817,7 +817,7 @@ class UserAPIController(BaseAPIController, UsesTagsMixin, CreatesApiKeysMixin, B if len_type not in ['file', 'fasta', 'text'] or not len_value: raise exceptions.RequestParameterInvalidException('Please specify a valid data source type.') if not name or not key: - raise exceptions.RequestParameterInvalidException('You must specify values for all the fields.') + raise exceptions.RequestParameterMissingException('You must specify values for all the fields.') elif key in dbkeys: raise exceptions.Conflict('There is already a custom build with that key. Delete it first if you want to replace it.') else: From c459d9a98ac1101438bc0b3cf747b84958b69e0c Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 21 Feb 2019 14:08:46 -0500 Subject: [PATCH 0163/1016] choose better exceptions class Co-Authored-By: martenson --- lib/galaxy/webapps/galaxy/api/users.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/users.py b/lib/galaxy/webapps/galaxy/api/users.py index 54c93ca08fc..765a8e9f073 100644 --- a/lib/galaxy/webapps/galaxy/api/users.py +++ b/lib/galaxy/webapps/galaxy/api/users.py @@ -871,7 +871,7 @@ class UserAPIController(BaseAPIController, UsesTagsMixin, CreatesApiKeysMixin, B build_dict['len'] = new_len.id build_dict['linecount'] = new_linecount.id except Exception: - raise exceptions.InternalServerError('Failed to convert dataset.') + raise exceptions.ToolExecutionError('Failed to convert dataset.') dbkeys[key] = build_dict user.preferences['dbkeys'] = json.dumps(dbkeys) trans.sa_session.flush() From 420e92917ee33672cf65f7406c285bb48968d255 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 21 Feb 2019 14:08:55 -0500 Subject: [PATCH 0164/1016] Filter kwds for admin client bootstrapping. --- lib/galaxy/webapps/galaxy/controllers/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py index 3fe64d79101..e55d9afed65 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -552,7 +552,7 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP 'installing_repository_ids': repository_util.get_ids_of_tool_shed_repositories_being_installed(trans.app, as_string=True), 'is_tool_shed_installed': bool(trans.app.tool_shed_registry and trans.app.tool_shed_registry.tool_sheds) } - return self._bootstrapped_client(trans, app_name='admin', settings=settings, message=message, status=status, **kwd) + return self._bootstrapped_client(trans, app_name='admin', settings=settings, message=message, status=status) @web.expose @web.json From 09a3802602c512ed51e02c243447db0776921f56 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 21 Feb 2019 14:09:34 -0500 Subject: [PATCH 0165/1016] choose better exceptions class Co-Authored-By: martenson --- lib/galaxy/webapps/galaxy/api/users.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/users.py b/lib/galaxy/webapps/galaxy/api/users.py index 765a8e9f073..bb9adf9f469 100644 --- a/lib/galaxy/webapps/galaxy/api/users.py +++ b/lib/galaxy/webapps/galaxy/api/users.py @@ -819,7 +819,7 @@ class UserAPIController(BaseAPIController, UsesTagsMixin, CreatesApiKeysMixin, B if not name or not key: raise exceptions.RequestParameterMissingException('You must specify values for all the fields.') elif key in dbkeys: - raise exceptions.Conflict('There is already a custom build with that key. Delete it first if you want to replace it.') + raise exceptions.DuplicatedIdentifierException('There is already a custom build with that key. Delete it first if you want to replace it.') else: # Have everything needed; create new build. build_dict = {'name': name} From b197fedf60b8e82f68bdb61e05d2f6d2bad33a7b Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Thu, 21 Feb 2019 14:10:22 -0500 Subject: [PATCH 0166/1016] improve exception message per @nsoranzo --- lib/galaxy/exceptions/error_codes.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/exceptions/error_codes.json b/lib/galaxy/exceptions/error_codes.json index 11908515a1d..5b8c2bbce2a 100644 --- a/lib/galaxy/exceptions/error_codes.json +++ b/lib/galaxy/exceptions/error_codes.json @@ -27,7 +27,7 @@ { "name": "USER_OBJECT_ATTRIBUTE_MISSING", "code": 400005, - "message": "Attempted to create object without required attribute." + "message": "Attempted to create or update object without required attribute." }, { "name": "USER_SLUG_DUPLICATE", @@ -108,7 +108,7 @@ "name": "ADMIN_REQUIRED", "code": 403006, "message": "Action requires admin account." - }, + }, { "name": "USER_ACTIVATION_REQUIRED", "code": 403007, From 96b4d1efc23c22c2085a7810f1b62d8bdd2c5cc8 Mon Sep 17 00:00:00 2001 From: guerler Date: Thu, 21 Feb 2019 15:18:58 -0500 Subject: [PATCH 0167/1016] Fix setting remaining setting for label column placement --- config/plugins/visualizations/nvd3/nvd3_pie/config/nvd3_pie.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/plugins/visualizations/nvd3/nvd3_pie/config/nvd3_pie.xml b/config/plugins/visualizations/nvd3/nvd3_pie/config/nvd3_pie.xml index 14ca88a3cef..093543e2559 100644 --- a/config/plugins/visualizations/nvd3/nvd3_pie/config/nvd3_pie.xml +++ b/config/plugins/visualizations/nvd3/nvd3_pie/config/nvd3_pie.xml @@ -93,7 +93,7 @@ key - label_outside + outside Would you like to show labels outside the donut? select From f00fb3e4add155268aaf7e17228699734b0bfeab Mon Sep 17 00:00:00 2001 From: guerler Date: Thu, 21 Feb 2019 15:59:10 -0500 Subject: [PATCH 0168/1016] Place oidc login button under regular login form --- client/galaxy/scripts/components/login/Login.vue | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/client/galaxy/scripts/components/login/Login.vue b/client/galaxy/scripts/components/login/Login.vue index e49f8dd3222..22ed39aafdf 100644 --- a/client/galaxy/scripts/components/login/Login.vue +++ b/client/galaxy/scripts/components/login/Login.vue @@ -24,12 +24,10 @@ + + Sign in with Google + - - - Log in with Google - -
From 87261f625368f55b2d7ba7aa1711fd462e2afcb6 Mon Sep 17 00:00:00 2001 From: guerler Date: Thu, 21 Feb 2019 16:05:29 -0500 Subject: [PATCH 0169/1016] Hide oidc options based on config setting --- client/galaxy/scripts/components/login/Login.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/galaxy/scripts/components/login/Login.vue b/client/galaxy/scripts/components/login/Login.vue index 22ed39aafdf..55018a5bfe4 100644 --- a/client/galaxy/scripts/components/login/Login.vue +++ b/client/galaxy/scripts/components/login/Login.vue @@ -24,7 +24,7 @@ - + Sign in with Google @@ -71,7 +71,7 @@ export default { messageShow() { return this.messageText != null; }, - showOIDC() { + oidcShow() { let Galaxy = getGalaxyInstance(); return Galaxy.config.enable_oidc == true; } From 61a5c3b14f7e606b7d0513b2ae1cec3acf6b75b0 Mon Sep 17 00:00:00 2001 From: guerler Date: Thu, 21 Feb 2019 16:07:28 -0500 Subject: [PATCH 0170/1016] Use lower-case galaxy local variable name --- client/galaxy/scripts/components/login/Login.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/galaxy/scripts/components/login/Login.vue b/client/galaxy/scripts/components/login/Login.vue index 55018a5bfe4..7e908a6a735 100644 --- a/client/galaxy/scripts/components/login/Login.vue +++ b/client/galaxy/scripts/components/login/Login.vue @@ -72,8 +72,8 @@ export default { return this.messageText != null; }, oidcShow() { - let Galaxy = getGalaxyInstance(); - return Galaxy.config.enable_oidc == true; + let galaxy = getGalaxyInstance(); + return galaxy.config.enable_oidc == true; } }, methods: { From 66b01d2b32181b6ae526f90e0c4cbde85f19c4c3 Mon Sep 17 00:00:00 2001 From: guerler Date: Thu, 21 Feb 2019 16:15:55 -0500 Subject: [PATCH 0171/1016] Avoid computing oidc setting --- client/galaxy/scripts/components/login/Login.vue | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/client/galaxy/scripts/components/login/Login.vue b/client/galaxy/scripts/components/login/Login.vue index 7e908a6a735..88d1e6348ef 100644 --- a/client/galaxy/scripts/components/login/Login.vue +++ b/client/galaxy/scripts/components/login/Login.vue @@ -24,7 +24,7 @@ - + Sign in with Google @@ -64,16 +64,13 @@ export default { messageText: null, messageVariant: null, redirect: galaxy.params.redirect, - session_csrf_token: galaxy.session_csrf_token + session_csrf_token: galaxy.session_csrf_token, + enable_oidc: galaxy.config.enable_oidc }; }, computed: { messageShow() { return this.messageText != null; - }, - oidcShow() { - let galaxy = getGalaxyInstance(); - return galaxy.config.enable_oidc == true; } }, methods: { From 351c188830793946ce6fa71dfaab95fe3af7cf7a Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 22 Feb 2019 12:07:16 +0000 Subject: [PATCH 0172/1016] Remove ``filter-with`` and update ``cookie_path`` options also for Tool Shed and Reports --- config/galaxy.yml.sample | 10 +++--- config/reports.yml.sample | 26 +++++++-------- config/tool_shed.yml.sample | 17 +++++----- doc/source/admin/galaxy_options.rst | 2 +- doc/source/admin/reports_options.rst | 33 +++++++------------ lib/galaxy/webapps/galaxy/config_schema.yml | 2 +- lib/galaxy/webapps/reports/config_schema.yml | 24 +++++--------- .../webapps/tool_shed/config_schema.yml | 10 +++--- 8 files changed, 51 insertions(+), 73 deletions(-) diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample index cfed6347d90..bc73ee43045 100644 --- a/config/galaxy.yml.sample +++ b/config/galaxy.yml.sample @@ -98,11 +98,11 @@ uwsgi: galaxy: - # If you serve Galaxy at a URL prefix and you're running more than one - # Galaxy instance behind one hostname, you will want to set this to - # the same path as the prefix in the mount uWSGI configuration above. - # This value becomes the "path" attribute set in the cookie so the - # cookies from each instance will not clobber each other. + # If Galaxy is served at a URL prefix and you are running more than + # one Galaxy instance behind one hostname, you will want to set this + # to the same path as the prefix in the mount uWSGI configuration + # above. This value becomes the "path" attribute set in the cookie so + # the cookies from each instance will not clobber each other. #cookie_path: '' # By default, Galaxy uses a SQLite database at diff --git a/config/reports.yml.sample b/config/reports.yml.sample index b4288fa4ca6..06b315cf134 100644 --- a/config/reports.yml.sample +++ b/config/reports.yml.sample @@ -80,34 +80,30 @@ uwsgi: reports: - # If running behind a proxy server and Galaxy is served from a - # subdirectory, enable the proxy-prefix filter and set the prefix in - # the [filter:proxy-prefix] section above. - #filter-with: proxy-prefix - - # If proxy-prefix is enabled and you're running more than one Galaxy - # instance behind one hostname, you will want to set this to the same - # path as the prefix in the filter above. This value becomes the - # "path" attribute set in the cookie so the cookies from each instance - # will not clobber each other. + # If Galaxy Reports is served at a URL prefix and you are running more + # than one Galaxy Reports instance behind one hostname, you will want + # to set this to the same path as the prefix in the mount uWSGI + # configuration above. This value becomes the "path" attribute set in + # the cookie so the cookies from each instance will not clobber each + # other. #cookie_path: null # Verbosity of console log messages. Acceptable values can be found # here: https://docs.python.org/2/library/logging.html#logging-levels #log_level: DEBUG - # Database connection Galaxy reports are intended for production + # Database connection. Galaxy Reports are intended for production # Galaxy instances, so sqlite (and the default value below) is not # supported. An SQLAlchemy connection string should be used specify an # external database. #database_connection: 'sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE' - # Where dataset files are saved Temporary storage for additional - # datasets, this should be shared through the cluster + # Where dataset files are saved. Temporary storage for additional + # datasets, this should be shared through the cluster. #file_path: database/files - # Where dataset files are saved Temporary storage for additional - # datasets, this should be shared through the cluster + # Where dataset files are saved. Temporary storage for additional + # datasets, this should be shared through the cluster. #new_file_path: database/tmp # Mako templates are compiled as needed and cached for reuse, this diff --git a/config/tool_shed.yml.sample b/config/tool_shed.yml.sample index 857869709a8..90dbb628abd 100644 --- a/config/tool_shed.yml.sample +++ b/config/tool_shed.yml.sample @@ -95,7 +95,7 @@ tool_shed: # installation directory. #hgweb_config_dir: null - # Where tool shed repositories are stored. + # Where Tool Shed repositories are stored. #file_path: database/community_files # Temporary storage for additional datasets, this should be shared @@ -233,10 +233,10 @@ tool_shed: # Allow administrators to delete accounts. #allow_user_deletion: false - # For use by email messages sent from the tool shed + # For use by email messages sent from the Tool Shed. #smtp_server: smtp.your_tool_shed_server - # For use by email messages sent from the tool shed + # For use by email messages sent from the Tool Shed. #email_from: your_tool_shed_email@server # If your SMTP server requires a username and password, you can @@ -351,11 +351,12 @@ tool_shed: # store this information. #citation_cache_lock_dir: database/citations/lock - # If proxy-prefix is enabled and you're running more than one Galaxy - # instance behind one hostname, you will want to set this to the same - # path as the prefix in the filter above. This value becomes the - # "path" attribute set in the cookie so the cookies from each instance - # will not clobber each other. + # If the Tool Shed is served at a URL prefix and you are running more + # than one Tool Shed instance behind one hostname, you will want to + # set this to the same path as the prefix in the mount uWSGI + # configuration above. This value becomes the "path" attribute set in + # the cookie so the cookies from each instance will not clobber each + # other. #cookie_path: null # Enable authentication via OpenID. Allows users to log in to their diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 3383515e1f6..f3e49bab46c 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -3,7 +3,7 @@ ~~~~~~~~~~~~~~~ :Description: - If you serve Galaxy at a URL prefix and you're running more than + If Galaxy is served at a URL prefix and you are running more than one Galaxy instance behind one hostname, you will want to set this to the same path as the prefix in the mount uWSGI configuration above. This value becomes the "path" attribute set in the cookie diff --git a/doc/source/admin/reports_options.rst b/doc/source/admin/reports_options.rst index d3cc03e683e..d3512952d51 100644 --- a/doc/source/admin/reports_options.rst +++ b/doc/source/admin/reports_options.rst @@ -1,25 +1,14 @@ -~~~~~~~~~~~~~~~ -``filter-with`` -~~~~~~~~~~~~~~~ - -:Description: - If running behind a proxy server and Galaxy is served from a - subdirectory, enable the proxy-prefix filter and set the prefix in - the [filter:proxy-prefix] section above. -:Default: ``proxy-prefix`` -:Type: str - - ~~~~~~~~~~~~~~~ ``cookie_path`` ~~~~~~~~~~~~~~~ :Description: - If proxy-prefix is enabled and you're running more than one Galaxy - instance behind one hostname, you will want to set this to the - same path as the prefix in the filter above. This value becomes - the "path" attribute set in the cookie so the cookies from each - instance will not clobber each other. + If Galaxy Reports is served at a URL prefix and you are running + more than one Galaxy Reports instance behind one hostname, you + will want to set this to the same path as the prefix in the mount + uWSGI configuration above. This value becomes the "path" attribute + set in the cookie so the cookies from each instance will not + clobber each other. :Default: ``None`` :Type: str @@ -41,7 +30,7 @@ ~~~~~~~~~~~~~~~~~~~~~~~ :Description: - Database connection Galaxy reports are intended for production + Database connection. Galaxy Reports are intended for production Galaxy instances, so sqlite (and the default value below) is not supported. An SQLAlchemy connection string should be used specify an external database. @@ -54,8 +43,8 @@ ~~~~~~~~~~~~~ :Description: - Where dataset files are saved Temporary storage for additional - datasets, this should be shared through the cluster + Where dataset files are saved. Temporary storage for additional + datasets, this should be shared through the cluster. :Default: ``database/files`` :Type: str @@ -65,8 +54,8 @@ ~~~~~~~~~~~~~~~~~ :Description: - Where dataset files are saved Temporary storage for additional - datasets, this should be shared through the cluster + Where dataset files are saved. Temporary storage for additional + datasets, this should be shared through the cluster. :Default: ``database/tmp`` :Type: str diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 58380cca392..761a8fc5973 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -30,7 +30,7 @@ mapping: default: '' required: false desc: | - If you serve Galaxy at a URL prefix and you're running more than one Galaxy instance + If Galaxy is served at a URL prefix and you are running more than one Galaxy instance behind one hostname, you will want to set this to the same path as the prefix in the mount uWSGI configuration above. This value becomes the "path" attribute set in the cookie so the cookies from each instance will not clobber each other. diff --git a/lib/galaxy/webapps/reports/config_schema.yml b/lib/galaxy/webapps/reports/config_schema.yml index c02b31d9e06..62003f5aebb 100644 --- a/lib/galaxy/webapps/reports/config_schema.yml +++ b/lib/galaxy/webapps/reports/config_schema.yml @@ -7,20 +7,12 @@ mapping: desc: | Galaxy Reports configuration options. mapping: - filter-with: - type: str - default: proxy-prefix - desc: | - If running behind a proxy server and Galaxy is served from a subdirectory, - enable the proxy-prefix filter and set the prefix in the - [filter:proxy-prefix] section above. - cookie_path: type: str desc: | - If proxy-prefix is enabled and you're running more than one Galaxy instance + If Galaxy Reports is served at a URL prefix and you are running more than one Galaxy Reports instance behind one hostname, you will want to set this to the same path as the prefix - in the filter above. This value becomes the "path" attribute set in the + in the mount uWSGI configuration above. This value becomes the "path" attribute set in the cookie so the cookies from each instance will not clobber each other. log_level: @@ -34,8 +26,8 @@ mapping: type: str default: sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE desc: | - Database connection - Galaxy reports are intended for production Galaxy instances, so sqlite (and the default value + Database connection. + Galaxy Reports are intended for production Galaxy instances, so sqlite (and the default value below) is not supported. An SQLAlchemy connection string should be used specify an external database. @@ -43,15 +35,15 @@ mapping: type: str default: database/files desc: | - Where dataset files are saved - Temporary storage for additional datasets, this should be shared through the cluster + Where dataset files are saved. + Temporary storage for additional datasets, this should be shared through the cluster. new_file_path: type: str default: database/tmp desc: | - Where dataset files are saved - Temporary storage for additional datasets, this should be shared through the cluster + Where dataset files are saved. + Temporary storage for additional datasets, this should be shared through the cluster. template_cache_path: type: str diff --git a/lib/galaxy/webapps/tool_shed/config_schema.yml b/lib/galaxy/webapps/tool_shed/config_schema.yml index 03f6eb74666..86e6212a105 100644 --- a/lib/galaxy/webapps/tool_shed/config_schema.yml +++ b/lib/galaxy/webapps/tool_shed/config_schema.yml @@ -37,7 +37,7 @@ mapping: default: database/community_files required: false desc: | - Where tool shed repositories are stored. + Where Tool Shed repositories are stored. new_file_path: type: str @@ -293,14 +293,14 @@ mapping: default: smtp.your_tool_shed_server required: false desc: | - For use by email messages sent from the tool shed + For use by email messages sent from the Tool Shed. email_from: type: str default: your_tool_shed_email@server required: false desc: | - For use by email messages sent from the tool shed + For use by email messages sent from the Tool Shed. smtp_username: type: str @@ -521,9 +521,9 @@ mapping: default: null required: false desc: | - If proxy-prefix is enabled and you're running more than one Galaxy instance + If the Tool Shed is served at a URL prefix and you are running more than one Tool Shed instance behind one hostname, you will want to set this to the same path as the prefix - in the filter above. This value becomes the "path" attribute set in the + in the mount uWSGI configuration above. This value becomes the "path" attribute set in the cookie so the cookies from each instance will not clobber each other. enable_openid: From 2a05323daa84db46479f5ee9ca756dba51da45e7 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Fri, 22 Feb 2019 10:08:34 -0500 Subject: [PATCH 0173/1016] Client formatting. --- client/galaxy/scripts/components/login/Login.vue | 2 +- client/galaxy/scripts/entry/panels/tool-panel.js | 4 +++- client/galaxy/scripts/layout/menu.js | 14 ++++++-------- client/galaxy/scripts/mvc/tool/tool-form-base.js | 10 ++++++---- client/galaxy/scripts/mvc/tool/tools.js | 6 +++--- client/galaxy/scripts/mvc/upload/upload-button.js | 6 ++---- client/galaxy/scripts/mvc/user/user-model.js | 10 +++++----- 7 files changed, 26 insertions(+), 26 deletions(-) diff --git a/client/galaxy/scripts/components/login/Login.vue b/client/galaxy/scripts/components/login/Login.vue index 88d1e6348ef..fc7d84f9c2f 100644 --- a/client/galaxy/scripts/components/login/Login.vue +++ b/client/galaxy/scripts/components/login/Login.vue @@ -25,7 +25,7 @@ - Sign in with Google + Sign in with Google
diff --git a/client/galaxy/scripts/entry/panels/tool-panel.js b/client/galaxy/scripts/entry/panels/tool-panel.js index 5d1f7008f78..49b36eb9414 100644 --- a/client/galaxy/scripts/entry/panels/tool-panel.js +++ b/client/galaxy/scripts/entry/panels/tool-panel.js @@ -50,7 +50,9 @@ var ToolPanel = Backbone.View.extend({ title: _l("Show favorites"), icon: "fa fa-star-o", onclick: e => { - $("#tool-search-query").val("#favorites").trigger("change"); + $("#tool-search-query") + .val("#favorites") + .trigger("change"); } }); diff --git a/client/galaxy/scripts/layout/menu.js b/client/galaxy/scripts/layout/menu.js index 01923a2f804..286e4150d99 100644 --- a/client/galaxy/scripts/layout/menu.js +++ b/client/galaxy/scripts/layout/menu.js @@ -14,14 +14,12 @@ function logoutClick() { let galaxy = getGalaxyInstance(); let session_csrf_token = galaxy.session_csrf_token; let url = `${galaxy.root}user/logout?session_csrf_token=${session_csrf_token}`; - axios - .get(url) - .then(() => { - if (galaxy.user) { - galaxy.user.clearSessionStorage(); - } - window.top.location.href = `${galaxy.root}login`; - }); + axios.get(url).then(() => { + if (galaxy.user) { + galaxy.user.clearSessionStorage(); + } + window.top.location.href = `${galaxy.root}login`; + }); } var Collection = Backbone.Collection.extend({ diff --git a/client/galaxy/scripts/mvc/tool/tool-form-base.js b/client/galaxy/scripts/mvc/tool/tool-form-base.js index 7be2efe4e97..32300172f55 100644 --- a/client/galaxy/scripts/mvc/tool/tool-form-base.js +++ b/client/galaxy/scripts/mvc/tool/tool-form-base.js @@ -125,11 +125,11 @@ export default FormBase.extend({ visible: !Galaxy.user.isAnonymous() && !in_favorites, onclick: () => { axios - .put(`${Galaxy.root}api/users/${Galaxy.user.id}/favorites/tools`, {'object_id': options.id}) + .put(`${Galaxy.root}api/users/${Galaxy.user.id}/favorites/tools`, { object_id: options.id }) .then(response => { favorite_button.hide(); remove_favorite_button.show(); - Galaxy.user.updateFavorites("tools", response.data) + Galaxy.user.updateFavorites("tools", response.data); }); } }); @@ -141,11 +141,13 @@ export default FormBase.extend({ visible: !Galaxy.user.isAnonymous() && in_favorites, onclick: () => { axios - .delete(`${Galaxy.root}api/users/${Galaxy.user.id}/favorites/tools/${encodeURIComponent(options.id)}`) + .delete( + `${Galaxy.root}api/users/${Galaxy.user.id}/favorites/tools/${encodeURIComponent(options.id)}` + ) .then(response => { remove_favorite_button.hide(); favorite_button.show(); - Galaxy.user.updateFavorites("tools", response.data) + Galaxy.user.updateFavorites("tools", response.data); }); } }); diff --git a/client/galaxy/scripts/mvc/tool/tools.js b/client/galaxy/scripts/mvc/tool/tools.js index dab611c4ba6..034b0e5c2d2 100644 --- a/client/galaxy/scripts/mvc/tool/tools.js +++ b/client/galaxy/scripts/mvc/tool/tools.js @@ -344,7 +344,7 @@ _.extend(ToolSection.prototype, VisibilityMixin); * query. */ var ToolSearch = Backbone.Model.extend({ - SEARCH_RESERVED_TERMS_FAVORITES: ['#favs', '#favorites', '#favourites'], + SEARCH_RESERVED_TERMS_FAVORITES: ["#favs", "#favorites", "#favourites"], defaults: { search_hint_string: "search tools", @@ -383,9 +383,9 @@ var ToolSearch = Backbone.Model.extend({ clearTimeout(this.timer); } // Catch reserved words - if (this.SEARCH_RESERVED_TERMS_FAVORITES.indexOf(q) >= 0){ + if (this.SEARCH_RESERVED_TERMS_FAVORITES.indexOf(q) >= 0) { this.set("results", Galaxy.user.getFavorites().tools); - } else{ + } else { // Start a new ajax-request in X ms $("#search-clear-btn").hide(); $("#search-spinner").show(); diff --git a/client/galaxy/scripts/mvc/upload/upload-button.js b/client/galaxy/scripts/mvc/upload/upload-button.js index 5788c505d55..7efabca5dd1 100644 --- a/client/galaxy/scripts/mvc/upload/upload-button.js +++ b/client/galaxy/scripts/mvc/upload/upload-button.js @@ -45,16 +45,14 @@ var View = Backbone.View.extend({ /** Template */ _template: function() { - return ( - `
+ return `
-
` - ); +
`; } }); export default { View: View }; diff --git a/client/galaxy/scripts/mvc/user/user-model.js b/client/galaxy/scripts/mvc/user/user-model.js index ba86489ea4b..8de2b2e1343 100644 --- a/client/galaxy/scripts/mvc/user/user-model.js +++ b/client/galaxy/scripts/mvc/user/user-model.js @@ -60,21 +60,21 @@ var User = Backbone.Model.extend(baseMVC.LoggableMixin).extend( this.preferences = preferences; }, - getFavorites: function(){ + getFavorites: function() { let preferences = this.get("preferences"); - if (preferences && preferences.favorites){ + if (preferences && preferences.favorites) { return JSON.parse(preferences.favorites); } else { return { - "tools": [] + tools: [] }; } }, - updateFavorites: function(object_type, new_favorites){ + updateFavorites: function(object_type, new_favorites) { let favorites = this.getFavorites(); favorites[object_type] = new_favorites[object_type]; - this.updatePreferences("favorites", favorites) + this.updatePreferences("favorites", favorites); }, /** Load a user with the API using an id. From a83eb67775442f37b22ff20203b88a234ab11bec Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Fri, 22 Feb 2019 11:31:44 -0500 Subject: [PATCH 0174/1016] Slight rewording of the cookie_path option description, and corrections to file_path and new_file_path descriptions. --- config/galaxy.yml.sample | 17 ++++++++++------- config/reports.yml.sample | 17 +++++++---------- config/tool_shed.yml.sample | 14 ++++++-------- doc/source/admin/galaxy_options.rst | 18 +++++++++++------- doc/source/admin/reports_options.rst | 18 ++++++++---------- lib/galaxy/webapps/galaxy/config_schema.yml | 15 +++++++++------ lib/galaxy/webapps/reports/config_schema.yml | 15 +++++++-------- lib/galaxy/webapps/tool_shed/config_schema.yml | 12 ++++++------ 8 files changed, 64 insertions(+), 62 deletions(-) diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample index bc73ee43045..6de3271e013 100644 --- a/config/galaxy.yml.sample +++ b/config/galaxy.yml.sample @@ -98,11 +98,11 @@ uwsgi: galaxy: - # If Galaxy is served at a URL prefix and you are running more than - # one Galaxy instance behind one hostname, you will want to set this - # to the same path as the prefix in the mount uWSGI configuration - # above. This value becomes the "path" attribute set in the cookie so - # the cookies from each instance will not clobber each other. + # When running multiple Galaxy instances under separate URL prefixes + # on a single hostname, you will want to set this to the same path as + # the prefix set in the uWSGI "mount" configuration option above. This + # value becomes the "path" attribute set in the cookie so the cookies + # from one instance will not clobber those from another. #cookie_path: '' # By default, Galaxy uses a SQLite database at @@ -170,10 +170,13 @@ galaxy: # not recommended for production use. #database_auto_migrate: false - # Dataset files are stored in this directory. + # Where dataset files are stored. It must accessible at the same path + # on any cluster nodes that will run Galaxy jobs, unless using Pulsar. #file_path: database/files - # Temporary files are stored in this directory. + # Where temporary files are stored. It must accessible at the same + # path on any cluster nodes that will run Galaxy jobs, unless using + # Pulsar. #new_file_path: database/tmp # Tool config files, defines what tools are available in Galaxy. Tools diff --git a/config/reports.yml.sample b/config/reports.yml.sample index 06b315cf134..fa9e533bcfd 100644 --- a/config/reports.yml.sample +++ b/config/reports.yml.sample @@ -80,12 +80,11 @@ uwsgi: reports: - # If Galaxy Reports is served at a URL prefix and you are running more - # than one Galaxy Reports instance behind one hostname, you will want - # to set this to the same path as the prefix in the mount uWSGI - # configuration above. This value becomes the "path" attribute set in - # the cookie so the cookies from each instance will not clobber each - # other. + # When running multiple Galaxy Reports instances under separate URL + # prefixes on a single hostname, you will want to set this to the same + # path as the prefix set in the uWSGI "mount" configuration option + # above. This value becomes the "path" attribute set in the cookie so + # the cookies from one instance will not clobber those from another. #cookie_path: null # Verbosity of console log messages. Acceptable values can be found @@ -98,12 +97,10 @@ reports: # external database. #database_connection: 'sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE' - # Where dataset files are saved. Temporary storage for additional - # datasets, this should be shared through the cluster. + # Where dataset files are stored. #file_path: database/files - # Where dataset files are saved. Temporary storage for additional - # datasets, this should be shared through the cluster. + # Where temporary files are stored. #new_file_path: database/tmp # Mako templates are compiled as needed and cached for reuse, this diff --git a/config/tool_shed.yml.sample b/config/tool_shed.yml.sample index 90dbb628abd..fc709bc0c01 100644 --- a/config/tool_shed.yml.sample +++ b/config/tool_shed.yml.sample @@ -98,8 +98,7 @@ tool_shed: # Where Tool Shed repositories are stored. #file_path: database/community_files - # Temporary storage for additional datasets, this should be shared - # through the cluster + # Where temporary files are stored. #new_file_path: database/tmp # File containing old-style genome builds @@ -351,12 +350,11 @@ tool_shed: # store this information. #citation_cache_lock_dir: database/citations/lock - # If the Tool Shed is served at a URL prefix and you are running more - # than one Tool Shed instance behind one hostname, you will want to - # set this to the same path as the prefix in the mount uWSGI - # configuration above. This value becomes the "path" attribute set in - # the cookie so the cookies from each instance will not clobber each - # other. + # When running multiple Tool Shed instances under separate URL + # prefixes on a single hostname, you will want to set this to the same + # path as the prefix set in the uWSGI "mount" configuration option + # above. This value becomes the "path" attribute set in the cookie so + # the cookies from one instance will not clobber those from another. #cookie_path: null # Enable authentication via OpenID. Allows users to log in to their diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index f3e49bab46c..d95bcd682a1 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -3,11 +3,11 @@ ~~~~~~~~~~~~~~~ :Description: - If Galaxy is served at a URL prefix and you are running more than - one Galaxy instance behind one hostname, you will want to set this - to the same path as the prefix in the mount uWSGI configuration - above. This value becomes the "path" attribute set in the cookie - so the cookies from each instance will not clobber each other. + When running multiple Galaxy instances under separate URL prefixes + on a single hostname, you will want to set this to the same path + as the prefix set in the uWSGI "mount" configuration option above. + This value becomes the "path" attribute set in the cookie so the + cookies from one instance will not clobber those from another. :Default: ```` :Type: str @@ -161,7 +161,9 @@ ~~~~~~~~~~~~~ :Description: - Dataset files are stored in this directory. + Where dataset files are stored. It must accessible at the same + path on any cluster nodes that will run Galaxy jobs, unless using + Pulsar. :Default: ``database/files`` :Type: str @@ -171,7 +173,9 @@ ~~~~~~~~~~~~~~~~~ :Description: - Temporary files are stored in this directory. + Where temporary files are stored. It must accessible at the same + path on any cluster nodes that will run Galaxy jobs, unless using + Pulsar. :Default: ``database/tmp`` :Type: str diff --git a/doc/source/admin/reports_options.rst b/doc/source/admin/reports_options.rst index d3512952d51..1a0a7b83032 100644 --- a/doc/source/admin/reports_options.rst +++ b/doc/source/admin/reports_options.rst @@ -3,12 +3,12 @@ ~~~~~~~~~~~~~~~ :Description: - If Galaxy Reports is served at a URL prefix and you are running - more than one Galaxy Reports instance behind one hostname, you - will want to set this to the same path as the prefix in the mount - uWSGI configuration above. This value becomes the "path" attribute - set in the cookie so the cookies from each instance will not - clobber each other. + When running multiple Galaxy Reports instances under separate URL + prefixes on a single hostname, you will want to set this to the + same path as the prefix set in the uWSGI "mount" configuration + option above. This value becomes the "path" attribute set in the + cookie so the cookies from one instance will not clobber those + from another. :Default: ``None`` :Type: str @@ -43,8 +43,7 @@ ~~~~~~~~~~~~~ :Description: - Where dataset files are saved. Temporary storage for additional - datasets, this should be shared through the cluster. + Where dataset files are stored. :Default: ``database/files`` :Type: str @@ -54,8 +53,7 @@ ~~~~~~~~~~~~~~~~~ :Description: - Where dataset files are saved. Temporary storage for additional - datasets, this should be shared through the cluster. + Where temporary files are stored. :Default: ``database/tmp`` :Type: str diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 761a8fc5973..ace6732a500 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -30,10 +30,11 @@ mapping: default: '' required: false desc: | - If Galaxy is served at a URL prefix and you are running more than one Galaxy instance - behind one hostname, you will want to set this to the same path as the prefix - in the mount uWSGI configuration above. This value becomes the "path" attribute set in the - cookie so the cookies from each instance will not clobber each other. + When running multiple Galaxy instances under separate URL prefixes on a + single hostname, you will want to set this to the same path as the prefix set + in the uWSGI "mount" configuration option above. This value becomes the "path" + attribute set in the cookie so the cookies from one instance will not clobber + those from another. database_connection: type: str @@ -143,14 +144,16 @@ mapping: default: database/files required: false desc: | - Dataset files are stored in this directory. + Where dataset files are stored. It must accessible at the same path on any cluster + nodes that will run Galaxy jobs, unless using Pulsar. new_file_path: type: str default: database/tmp required: false desc: | - Temporary files are stored in this directory. + Where temporary files are stored. It must accessible at the same path on any cluster + nodes that will run Galaxy jobs, unless using Pulsar. tool_config_file: type: str diff --git a/lib/galaxy/webapps/reports/config_schema.yml b/lib/galaxy/webapps/reports/config_schema.yml index 62003f5aebb..91be030da36 100644 --- a/lib/galaxy/webapps/reports/config_schema.yml +++ b/lib/galaxy/webapps/reports/config_schema.yml @@ -10,10 +10,11 @@ mapping: cookie_path: type: str desc: | - If Galaxy Reports is served at a URL prefix and you are running more than one Galaxy Reports instance - behind one hostname, you will want to set this to the same path as the prefix - in the mount uWSGI configuration above. This value becomes the "path" attribute set in the - cookie so the cookies from each instance will not clobber each other. + When running multiple Galaxy Reports instances under separate URL prefixes on a + single hostname, you will want to set this to the same path as the prefix set + in the uWSGI "mount" configuration option above. This value becomes the "path" + attribute set in the cookie so the cookies from one instance will not clobber + those from another. log_level: type: str @@ -35,15 +36,13 @@ mapping: type: str default: database/files desc: | - Where dataset files are saved. - Temporary storage for additional datasets, this should be shared through the cluster. + Where dataset files are stored. new_file_path: type: str default: database/tmp desc: | - Where dataset files are saved. - Temporary storage for additional datasets, this should be shared through the cluster. + Where temporary files are stored. template_cache_path: type: str diff --git a/lib/galaxy/webapps/tool_shed/config_schema.yml b/lib/galaxy/webapps/tool_shed/config_schema.yml index 86e6212a105..71309d85c89 100644 --- a/lib/galaxy/webapps/tool_shed/config_schema.yml +++ b/lib/galaxy/webapps/tool_shed/config_schema.yml @@ -44,8 +44,7 @@ mapping: default: database/tmp required: false desc: | - Temporary storage for additional datasets, - this should be shared through the cluster + Where temporary files are stored. builds_file_path: type: str @@ -521,10 +520,11 @@ mapping: default: null required: false desc: | - If the Tool Shed is served at a URL prefix and you are running more than one Tool Shed instance - behind one hostname, you will want to set this to the same path as the prefix - in the mount uWSGI configuration above. This value becomes the "path" attribute set in the - cookie so the cookies from each instance will not clobber each other. + When running multiple Tool Shed instances under separate URL prefixes on a + single hostname, you will want to set this to the same path as the prefix set + in the uWSGI "mount" configuration option above. This value becomes the "path" + attribute set in the cookie so the cookies from one instance will not clobber + those from another. enable_openid: type: bool From 656b3c2f87a0dfd49284e113d7df6f2b88b43464 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 4 Dec 2018 10:37:57 -0500 Subject: [PATCH 0175/1016] Refactor dataset fetching in get/set dataset method in web controller. --- .../webapps/galaxy/controllers/dataset.py | 56 ++++++++++--------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py index 6b1b009aaad..a29c277b1c1 100644 --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -251,27 +251,11 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE @web.expose_api_anonymous def get_edit(self, trans, dataset_id=None, **kwd): """Produces the input definitions available to modify dataset attributes""" - message = None status = None - if dataset_id is not None: - id = self.decode_id(dataset_id) - data = trans.sa_session.query(self.app.model.HistoryDatasetAssociation).get(id) - else: - trans.log_event("dataset_id is None, cannot load a dataset to edit.") - return self.message_exception(trans, 'You must provide a dataset id to edit attributes.') - if data is None: - trans.log_event("Problem retrieving dataset id (%s)." % dataset_id) - return self.message_exception(trans, 'The dataset id is invalid.') - if dataset_id is not None and data.history.user is not None and data.history.user != trans.user: - trans.log_event("User attempted to edit a dataset they do not own (encoded: %s, decoded: %s)." % (dataset_id, id)) - return self.message_exception(trans, 'The dataset id is invalid.') - if data.history.user and not data.dataset.has_manage_permissions_roles(trans): - # Permission setting related to DATASET_MANAGE_PERMISSIONS was broken for a period of time, - # so it is possible that some Datasets have no roles associated with the DATASET_MANAGE_PERMISSIONS - # permission. In this case, we'll reset this permission to the hda user's private role. - manage_permissions_action = trans.app.security_agent.get_action(trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS.action) - permissions = {manage_permissions_action : [trans.app.security_agent.get_private_user_role(data.history.user)]} - trans.app.security_agent.set_dataset_permission(data.dataset, permissions) + data, message = self._get_dataset_for_edit(trans, dataset_id) + if message: + return message + if self._can_access_dataset(trans, data): if data.state == trans.model.Dataset.states.UPLOAD: return self.message_exception(trans, 'Please wait until this dataset finishes uploading before attempting to edit its metadata.') @@ -416,13 +400,13 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE return False return True - message = None status = 'success' - dataset_id = payload.get('dataset_id') operation = payload.get('operation') - if dataset_id is not None: - id = self.decode_id(dataset_id) - data = trans.sa_session.query(self.app.model.HistoryDatasetAssociation).get(id) + dataset_id = payload.get('dataset_id') + data, message = self._get_dataset_for_edit(trans, dataset_id) + if message: + return message + if operation == 'attributes': # The user clicked the Save button on the 'Edit Attributes' form data.name = payload.get('name') @@ -510,6 +494,28 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE return self.message_exception(trans, 'Invalid operation identifier (%s).' % operation) return {'status': status, 'message': sanitize_text(message)} + def _get_dataset_for_edit(self, trans, dataset_id): + if dataset_id is not None: + id = self.decode_id(dataset_id) + data = trans.sa_session.query(self.app.model.HistoryDatasetAssociation).get(id) + else: + trans.log_event("dataset_id is None, cannot load a dataset to edit.") + return None, self.message_exception(trans, 'You must provide a dataset id to edit attributes.') + if data is None: + trans.log_event("Problem retrieving dataset id (%s)." % dataset_id) + return None, self.message_exception(trans, 'The dataset id is invalid.') + if dataset_id is not None and data.history.user is not None and data.history.user != trans.user: + trans.log_event("User attempted to edit a dataset they do not own (encoded: %s, decoded: %s)." % (dataset_id, id)) + return None, self.message_exception(trans, 'The dataset id is invalid.') + if data.history.user and not data.dataset.has_manage_permissions_roles(trans): + # Permission setting related to DATASET_MANAGE_PERMISSIONS was broken for a period of time, + # so it is possible that some Datasets have no roles associated with the DATASET_MANAGE_PERMISSIONS + # permission. In this case, we'll reset this permission to the hda user's private role. + manage_permissions_action = trans.app.security_agent.get_action(trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS.action) + permissions = {manage_permissions_action : [trans.app.security_agent.get_private_user_role(data.history.user)]} + trans.app.security_agent.set_dataset_permission(data.dataset, permissions) + return data, None + @web.expose @web.json @web.require_login("see all available datasets") From 3bc90381a535aa4ca42c772e1bfca92765230f04 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 22 Feb 2019 13:32:32 -0500 Subject: [PATCH 0176/1016] Add a brief security note for 19.01. --- doc/source/releases/19.01_announce.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/source/releases/19.01_announce.rst b/doc/source/releases/19.01_announce.rst index e6f6d07fd8d..d8df49a5cf0 100644 --- a/doc/source/releases/19.01_announce.rst +++ b/doc/source/releases/19.01_announce.rst @@ -67,6 +67,16 @@ To update an existing Galaxy repository run: See the `community hub `__ for additional details regarding the source code locations. +Security +======== + +Permissions Not Checked to Modify Dataset Metadata +-------------------------------------------------- + +Jelle Scholtalbers reported an issue where permissions were not being checked when modifying dataset +metadata. This has been resolved with `Pull Request 7322 +`__ and the fix backported through 18.01. + Release Notes =========================================================== From 4372b493aa3781857c30926933e0324ac30f8517 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 13 Feb 2019 17:37:00 -0500 Subject: [PATCH 0177/1016] Simplify tag handling in history export. Not yet used in import so we might as well simplify our lives and synchronize this with API exports before we actually start using them. --- lib/galaxy/tools/imp_exp/__init__.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/tools/imp_exp/__init__.py b/lib/galaxy/tools/imp_exp/__init__.py index 0456ac22fcc..2f6ff79a0f6 100644 --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -275,15 +275,6 @@ class JobExportHistoryArchiveWrapper: # Helper methods/classes. # - def get_item_tag_dict(item): - """ Create dictionary of an item's tags. """ - tags = {} - for tag in item.tags: - tag_user_tname = unicodify(tag.user_tname) - tag_user_value = unicodify(tag.user_value) - tags[tag_user_tname] = tag_user_value - return tags - def prepare_metadata(metadata): """ Prepare metatdata for exporting. """ for name, value in list(metadata.items()): @@ -316,7 +307,7 @@ class JobExportHistoryArchiveWrapper: "visible": obj.visible, "uuid": (lambda uuid: str(uuid) if uuid else None)(obj.dataset.uuid), "annotation": unicodify(getattr(obj, 'annotation', '')), - "tags": get_item_tag_dict(obj) + "tags": obj.make_tag_string_list() } try: @@ -352,7 +343,7 @@ class JobExportHistoryArchiveWrapper: "hid_counter": history.hid_counter, "genome_build": history.genome_build, "annotation": unicodify(get_item_annotation_str(trans.sa_session, history.user, history)), - "tags": get_item_tag_dict(history) + "tags": history.make_tag_string_list() } history_attrs_filename = tempfile.NamedTemporaryFile(dir=temp_output_dir).name history_attrs_out = open(history_attrs_filename, 'w') From 07806f3a028542c2481a89bb12fe6ca1953985e7 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 12 Dec 2017 09:20:41 -0500 Subject: [PATCH 0178/1016] Simplify path tracking related to history exports. This is a simpler model to think about and prevents us from needing to track new fields in the database for extensions such as collections. --- lib/galaxy/model/__init__.py | 9 +++++---- lib/galaxy/model/mapping.py | 4 +--- lib/galaxy/tools/imp_exp/__init__.py | 9 +-------- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index d83b17ce21c..10707240552 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1254,15 +1254,16 @@ class JobExternalOutputMetadata(RepresentById): class JobExportHistoryArchive(RepresentById): def __init__(self, job=None, history=None, dataset=None, compressed=False, - history_attrs_filename=None, datasets_attrs_filename=None, - jobs_attrs_filename=None): + history_attrs_filename=None): self.job = job self.history = history self.dataset = dataset self.compressed = compressed self.history_attrs_filename = history_attrs_filename - self.datasets_attrs_filename = datasets_attrs_filename - self.jobs_attrs_filename = jobs_attrs_filename + + @property + def temp_directory(self): + return os.path.split(self.history_attrs_filename)[0] @property def up_to_date(self): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 63f4daa52d9..55df76e797e 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -656,9 +656,7 @@ model.JobExportHistoryArchive.table = Table( Column("history_id", Integer, ForeignKey("history.id"), index=True), Column("dataset_id", Integer, ForeignKey("dataset.id"), index=True), Column("compressed", Boolean, index=True, default=False), - Column("history_attrs_filename", TEXT), - Column("datasets_attrs_filename", TEXT), - Column("jobs_attrs_filename", TEXT)) + Column("history_attrs_filename", TEXT)) model.JobImportHistoryArchive.table = Table( "job_import_history_archive", metadata, diff --git a/lib/galaxy/tools/imp_exp/__init__.py b/lib/galaxy/tools/imp_exp/__init__.py index 0456ac22fcc..49298c6b8d7 100644 --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -376,7 +376,6 @@ class JobExportHistoryArchiveWrapper: datasets_attrs_out = open(datasets_attrs_filename, 'w') datasets_attrs_out.write(dumps(datasets_attrs, cls=HistoryDatasetAssociationEncoder)) datasets_attrs_out.close() - jeha.datasets_attrs_filename = datasets_attrs_filename provenance_attrs_out = open(datasets_attrs_filename + ".provenance", 'w') provenance_attrs_out.write(dumps(provenance_attrs, cls=HistoryDatasetAssociationEncoder)) @@ -454,7 +453,6 @@ class JobExportHistoryArchiveWrapper: jobs_attrs_out = open(jobs_attrs_filename, 'w') jobs_attrs_out.write(dumps(jobs_attrs, cls=HistoryDatasetAssociationEncoder)) jobs_attrs_out.close() - jeha.jobs_attrs_filename = jobs_attrs_filename # # Create and return command line for running tool. @@ -471,12 +469,7 @@ class JobExportHistoryArchiveWrapper: # Get jeha for job. jeha = db_session.query(model.JobExportHistoryArchive).filter_by(job_id=self.job_id).first() if jeha: - for filename in [jeha.history_attrs_filename, jeha.datasets_attrs_filename, jeha.jobs_attrs_filename]: - try: - os.remove(filename) - except Exception as e: - log.debug('Failed to cleanup attributes file (%s): %s' % (filename, e)) - temp_dir = os.path.split(jeha.history_attrs_filename)[0] + temp_dir = jeha.temp_directory try: shutil.rmtree(temp_dir) except Exception as e: From db78c26c74a29b02f0e5dad48409cd1ac4d73f0a Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 7 Jan 2019 11:41:52 -0500 Subject: [PATCH 0179/1016] Rework export history argument parsing. Other changes in #4532 would have caused export jobs that were created before an upgrade that added that PR to fail after the upgrade. This wouldn't have been a huge deal but the new format of the arguments is a bit cleaner and future proof anyway. --- lib/galaxy/tools/imp_exp/__init__.py | 23 ++++++++++++---------- lib/galaxy/tools/imp_exp/export_history.py | 10 +++++++++- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/tools/imp_exp/__init__.py b/lib/galaxy/tools/imp_exp/__init__.py index 49298c6b8d7..04261b2a08f 100644 --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -17,6 +17,10 @@ from galaxy.util import unicodify log = logging.getLogger(__name__) +ATTRS_FILENAME_HISTORY = 'history_attrs.txt' +ATTRS_FILENAME_DATASETS = 'datasets_attrs.txt' +ATTRS_FILENAME_JOBS = 'jobs_attrs.txt' + class JobImportHistoryArchiveWrapper: """ @@ -49,7 +53,7 @@ class JobImportHistoryArchiveWrapper: user = jiha.job.user # Bioblend previous to 17.01 exported histories with an extra subdir. - if not os.path.exists(os.path.join(archive_dir, 'history_attrs.txt')): + if not os.path.exists(os.path.join(archive_dir, ATTRS_FILENAME_HISTORY)): for d in os.listdir(archive_dir): if os.path.isdir(os.path.join(archive_dir, d)): archive_dir = os.path.join(archive_dir, d) @@ -58,7 +62,7 @@ class JobImportHistoryArchiveWrapper: # # Create history. # - history_attr_file_name = os.path.join(archive_dir, 'history_attrs.txt') + history_attr_file_name = os.path.join(archive_dir, ATTRS_FILENAME_HISTORY) history_attrs = load(open(history_attr_file_name)) # Create history. @@ -83,7 +87,7 @@ class JobImportHistoryArchiveWrapper: # # Create datasets. # - datasets_attrs_file_name = os.path.join(archive_dir, 'datasets_attrs.txt') + datasets_attrs_file_name = os.path.join(archive_dir, ATTRS_FILENAME_DATASETS) datasets_attrs = load(open(datasets_attrs_file_name)) provenance_file_name = datasets_attrs_file_name + ".provenance" @@ -165,7 +169,7 @@ class JobImportHistoryArchiveWrapper: return self.sa_session.query(model.HistoryDatasetAssociation) \ .filter_by(history=new_history, hid=obj_dct['hid']).first() return obj_dct - jobs_attr_file_name = os.path.join(archive_dir, 'jobs_attrs.txt') + jobs_attr_file_name = os.path.join(archive_dir, ATTRS_FILENAME_JOBS) jobs_attrs = load(open(jobs_attr_file_name), object_hook=as_hda) # Create each job. @@ -354,7 +358,7 @@ class JobExportHistoryArchiveWrapper: "annotation": unicodify(get_item_annotation_str(trans.sa_session, history.user, history)), "tags": get_item_tag_dict(history) } - history_attrs_filename = tempfile.NamedTemporaryFile(dir=temp_output_dir).name + history_attrs_filename = os.path.join(temp_output_dir, ATTRS_FILENAME_HISTORY) history_attrs_out = open(history_attrs_filename, 'w') history_attrs_out.write(dumps(history_attrs)) history_attrs_out.close() @@ -372,7 +376,8 @@ class JobExportHistoryArchiveWrapper: else: datasets_attrs.append(dataset) included_datasets.append(dataset) - datasets_attrs_filename = tempfile.NamedTemporaryFile(dir=temp_output_dir).name + + datasets_attrs_filename = os.path.join(temp_output_dir, ATTRS_FILENAME_DATASETS) datasets_attrs_out = open(datasets_attrs_filename, 'w') datasets_attrs_out.write(dumps(datasets_attrs, cls=HistoryDatasetAssociationEncoder)) datasets_attrs_out.close() @@ -449,7 +454,7 @@ class JobExportHistoryArchiveWrapper: jobs_attrs.append(job_attrs) - jobs_attrs_filename = tempfile.NamedTemporaryFile(dir=temp_output_dir).name + jobs_attrs_filename = os.path.join(temp_output_dir, ATTRS_FILENAME_JOBS) jobs_attrs_out = open(jobs_attrs_filename, 'w') jobs_attrs_out.write(dumps(jobs_attrs, cls=HistoryDatasetAssociationEncoder)) jobs_attrs_out.close() @@ -460,9 +465,7 @@ class JobExportHistoryArchiveWrapper: options = "" if jeha.compressed: options = "-G" - return "%s %s %s %s" % (options, history_attrs_filename, - datasets_attrs_filename, - jobs_attrs_filename) + return "%s %s" % (options, temp_output_dir) def cleanup_after_job(self, db_session): """ Remove temporary directory and attribute files generated during setup for this job. """ diff --git a/lib/galaxy/tools/imp_exp/export_history.py b/lib/galaxy/tools/imp_exp/export_history.py index ac70477db47..aa65c897e52 100644 --- a/lib/galaxy/tools/imp_exp/export_history.py +++ b/lib/galaxy/tools/imp_exp/export_history.py @@ -100,7 +100,15 @@ def main(): parser.add_option('-G', '--gzip', dest='gzip', action="store_true", help='Compress archive using gzip.') (options, args) = parser.parse_args() gzip = bool(options.gzip) - history_attrs, dataset_attrs, job_attrs, out_file = args + if len(args) == 2: + # We have a 19.0X directory argument instead of individual arguments. + temp_directory, out_file = args + history_attrs = os.path.join(temp_directory, 'history_attrs.txt') + dataset_attrs = os.path.join(temp_directory, 'datasets_attrs.txt') + job_attrs = os.path.join(temp_directory, 'jobs_attrs.txt') + else: + # This job was created pre 18.0X with old argument style. + history_attrs, dataset_attrs, job_attrs, out_file = args # Create archive. create_archive(history_attrs, dataset_attrs, job_attrs, out_file, gzip) From 0fe769e2e4d9105460e6873d40e00507da9b0db4 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 13 Feb 2019 15:25:09 -0500 Subject: [PATCH 0180/1016] Add --galaxy_version argument to export_history script. --- lib/galaxy/tools/imp_exp/__init__.py | 6 ++++-- lib/galaxy/tools/imp_exp/export_history.py | 17 ++++++++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/tools/imp_exp/__init__.py b/lib/galaxy/tools/imp_exp/__init__.py index 04261b2a08f..1666f5f3cdc 100644 --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -9,11 +9,13 @@ from json import dumps, load from sqlalchemy.orm import eagerload_all from sqlalchemy.sql import expression + from galaxy import model from galaxy.exceptions import MalformedContents from galaxy.exceptions import ObjectNotFound from galaxy.model.item_attrs import add_item_annotation, get_item_annotation_str from galaxy.util import unicodify +from galaxy.version import VERSION_MAJOR log = logging.getLogger(__name__) @@ -462,9 +464,9 @@ class JobExportHistoryArchiveWrapper: # # Create and return command line for running tool. # - options = "" + options = "--galaxy-version '%s'" % VERSION_MAJOR if jeha.compressed: - options = "-G" + options += " -G" return "%s %s" % (options, temp_output_dir) def cleanup_after_job(self, db_session): diff --git a/lib/galaxy/tools/imp_exp/export_history.py b/lib/galaxy/tools/imp_exp/export_history.py index aa65c897e52..cb35a888abe 100644 --- a/lib/galaxy/tools/imp_exp/export_history.py +++ b/lib/galaxy/tools/imp_exp/export_history.py @@ -98,17 +98,24 @@ def main(): # Parse command line. parser = optparse.OptionParser() parser.add_option('-G', '--gzip', dest='gzip', action="store_true", help='Compress archive using gzip.') + parser.add_option('--galaxy-version', dest='galaxy_version', help='Galaxy version that initiated the command.', default=None) (options, args) = parser.parse_args() + galaxy_version = options.galaxy_version + if galaxy_version is None: + galaxy_version = "19.01" if len(args) == 4 else "19.05" + gzip = bool(options.gzip) - if len(args) == 2: + if galaxy_version == "19.01": + # This job was created pre 18.0X with old argument style. + history_attrs, dataset_attrs, job_attrs, out_file = args + else: + assert len(args) >= 2 # We have a 19.0X directory argument instead of individual arguments. - temp_directory, out_file = args + temp_directory = args[0] + out_file = args[1] history_attrs = os.path.join(temp_directory, 'history_attrs.txt') dataset_attrs = os.path.join(temp_directory, 'datasets_attrs.txt') job_attrs = os.path.join(temp_directory, 'jobs_attrs.txt') - else: - # This job was created pre 18.0X with old argument style. - history_attrs, dataset_attrs, job_attrs, out_file = args # Create archive. create_archive(history_attrs, dataset_attrs, job_attrs, out_file, gzip) From 47ed81feb95c092587c68deb07e247c3a4d29bff Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 26 Feb 2019 12:06:02 +0100 Subject: [PATCH 0181/1016] Remove unreachable print statement I don't think an exception could ever occur here. --- tools/data_source/upload.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index f8afc8e93c4..0c6ecf8325e 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -178,10 +178,7 @@ def add_composite_file(dataset, registry, output_path, files_path): # Find data type if dataset.file_type is not None: - try: - datatype = registry.get_datatype_by_extension(dataset.file_type) - except Exception: - print("Unable to instantiate the datatype object for the file type '%s'" % dataset.file_type) + datatype = registry.get_datatype_by_extension(dataset.file_type) def to_path(path_or_url): is_url = path_or_url.find('://') != -1 # todo fixme From ed375993ebc43112e03f58779d7c8c104e67956a Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 26 Feb 2019 12:22:46 +0100 Subject: [PATCH 0182/1016] Fix analyze75 hdr file upload when convert newlines is on --- lib/galaxy/datatypes/proteomics.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py index edd636500bb..ee6c4bc8d7b 100644 --- a/lib/galaxy/datatypes/proteomics.py +++ b/lib/galaxy/datatypes/proteomics.py @@ -488,7 +488,7 @@ class Analyze75(Binary): self.add_composite_file( 'hdr', description='The Analyze75 header file.', - is_binary=False) + is_binary=True) """The image file. Image data, whose data type and ordering are described by the header file.""" self.add_composite_file( @@ -500,7 +500,8 @@ class Analyze75(Binary): self.add_composite_file( 't2m', description='The Analyze75 t2m file.', - optional='True', is_binary=True) + optional=True, + is_binary=True) def generate_primary_file(self, dataset=None): rval = ['Analyze75 Composite Dataset.

'] From 8e82fc71910025ddc77827a9078e25ecfeb0cc30 Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Tue, 26 Feb 2019 08:36:30 -0700 Subject: [PATCH 0183/1016] SQUASH ME: rename suggestion from @bgruening --- lib/galaxy/tools/verify/asserts/zip.py | 2 +- lib/galaxy/tools/xsd/galaxy.xsd | 4 +-- test/functional/tools/validation_zip.xml | 38 ++++++++++++------------ 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/lib/galaxy/tools/verify/asserts/zip.py b/lib/galaxy/tools/verify/asserts/zip.py index 23dc9194044..56cc922acbd 100644 --- a/lib/galaxy/tools/verify/asserts/zip.py +++ b/lib/galaxy/tools/verify/asserts/zip.py @@ -3,7 +3,7 @@ import re import zipfile -def assert_has_zip_member(output_bytes, path, verify_assertions_function, children): +def assert_has_archive_member(output_bytes, path, verify_assertions_function, children): """ Recursively checks the specified assertions against the text of the first element matching the specified path found within the zip archive.""" output_temp = io.BytesIO(output_bytes) diff --git a/lib/galaxy/tools/xsd/galaxy.xsd b/lib/galaxy/tools/xsd/galaxy.xsd index f7dc98345df..fa42a2da76f 100644 --- a/lib/galaxy/tools/xsd/galaxy.xsd +++ b/lib/galaxy/tools/xsd/galaxy.xsd @@ -1655,10 +1655,10 @@ module. - + - ``).]]> + ``).]]> diff --git a/test/functional/tools/validation_zip.xml b/test/functional/tools/validation_zip.xml index 210299984bc..6e512074540 100644 --- a/test/functional/tools/validation_zip.xml +++ b/test/functional/tools/validation_zip.xml @@ -18,43 +18,43 @@ - + - + - - + + - - + + - + - + - - + + - - + + - - + + - - + + - + - + - + From 14f3ce3b5092467e3d9dde9bd94fdab5f6b604b2 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 26 Feb 2019 11:12:22 -0500 Subject: [PATCH 0184/1016] Fix workflow interface drop to import --- client/galaxy/scripts/mvc/workflow/workflow.js | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/client/galaxy/scripts/mvc/workflow/workflow.js b/client/galaxy/scripts/mvc/workflow/workflow.js index dcc9f6e3dd6..e096b65472f 100644 --- a/client/galaxy/scripts/mvc/workflow/workflow.js +++ b/client/galaxy/scripts/mvc/workflow/workflow.js @@ -195,13 +195,14 @@ const WorkflowListView = Backbone.View.extend({ }, events: { - dragleave: "unhighlightDropZone", - drop: "drop", - dragover: function(ev) { - $(".hidden_description_layer").addClass("dragover"); - $(".menubutton").addClass("background-none"); - ev.preventDefault(); - } + dragover: "highlightDropZone", + dragleave: "unhighlightDropZone" + }, + + highlightDropZone: function(ev) { + $(".hidden_description_layer").addClass("dragover"); + $(".menubutton").addClass("background-none"); + ev.preventDefault(); }, unhighlightDropZone: function() { @@ -272,6 +273,7 @@ const WorkflowListView = Backbone.View.extend({ this.searchWorkflow(this.$(".search-wf"), this.$(".workflow-search tr"), minQueryLength); this.adjustActiondropdown(); this._showArgErrors(); + this.$(".hidden_description_layer").get(0).addEventListener('drop', _.bind(this.drop, this)); return this; }, From 1583eecfdb338aae17fbb199160f7937e3b0ec01 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 26 Feb 2019 11:14:36 -0500 Subject: [PATCH 0185/1016] Client formatting sync for 19.01 --- client/galaxy/scripts/apps/extended.js | 2 +- .../scripts/components/admin/ErrorStack.vue | 10 ++++---- .../galaxy.interactive_environments.js | 4 +-- .../scripts/mvc/tool/tool-form-composite.js | 6 +++-- .../galaxy/scripts/mvc/workflow/workflow.js | 4 ++- client/galaxy/style/scss/base.scss | 25 +++++++++++-------- 6 files changed, 28 insertions(+), 23 deletions(-) diff --git a/client/galaxy/scripts/apps/extended.js b/client/galaxy/scripts/apps/extended.js index 50580cb7500..5523700bc1f 100644 --- a/client/galaxy/scripts/apps/extended.js +++ b/client/galaxy/scripts/apps/extended.js @@ -82,4 +82,4 @@ export function multiHistory(options) { } // Used in common.mako -export { default as store } from "store"; \ No newline at end of file +export { default as store } from "store"; diff --git a/client/galaxy/scripts/components/admin/ErrorStack.vue b/client/galaxy/scripts/components/admin/ErrorStack.vue index d594074daed..b1266d222ab 100644 --- a/client/galaxy/scripts/components/admin/ErrorStack.vue +++ b/client/galaxy/scripts/components/admin/ErrorStack.vue @@ -12,7 +12,7 @@

diff --git a/client/galaxy/scripts/components/Tags/StandardTags.test.js b/client/galaxy/scripts/components/Tags/StandardTags.test.js index b0b11edc444..1d0f45d65e0 100644 --- a/client/galaxy/scripts/components/Tags/StandardTags.test.js +++ b/client/galaxy/scripts/components/Tags/StandardTags.test.js @@ -1,5 +1,5 @@ import sinon from "sinon"; -import { mount, createLocalVue} from "@vue/test-utils"; +import { mount, createLocalVue } from "@vue/test-utils"; import StandardTags from "./StandardTags"; import store from "../../store"; import _l from "utils/localization"; @@ -7,9 +7,7 @@ import _l from "utils/localization"; let mockRedirect = sinon.stub(); StandardTags.__Rewire__("redirectToUrl", mockRedirect); - describe("Tags/StandardTags.vue", () => { - let localVue = createLocalVue(); localVue.filter("localize", value => _l(value)); @@ -18,18 +16,16 @@ describe("Tags/StandardTags.vue", () => { function clickFirstTag() { let firstTag = wrapper.find(".ti-tag-center > div"); - firstTag.trigger('click'); + firstTag.trigger("click"); } - + // TODO: put this in a store-specific test file it("the searchTags in the store should be a Set object", () => { let searchTags = store.state.gridSearch.searchTags; // this is a Set() assert(searchTags instanceof Set, "searchTags wrong variable type, should be Set()"); - }) - + }); describe("grid tags (tagClickFn: add_tag_to_grid_filter)", () => { - let propsData = { tags: Array.from(testTags), tagClickFn: "add_tag_to_grid_filter", @@ -37,55 +33,53 @@ describe("Tags/StandardTags.vue", () => { itemClass: "fakeItemClass" }; - beforeEach(function () { - wrapper = mount(StandardTags, { - store, + beforeEach(function() { + wrapper = mount(StandardTags, { + store, propsData, localVue }); - }) + }); afterEach(function() { mockRedirect.reset(); store.dispatch("clearSearchTags"); - }) + }); it("clicking on a search tag should put that tag in the global store", () => { clickFirstTag(); let searchTags = store.state.gridSearch.searchTags; // this is a Set() assert(searchTags.has(testTags[0]), "clicked tag not in store"); assert(searchTags.size == 1, `wrong number of searchTags in store: ${searchTags.size}`); - }) + }); it("clicking the same tag twice should add and remove it from the global store", () => { clickFirstTag(); clickFirstTag(); let searchTags = store.state.gridSearch.searchTags; assert(!searchTags.has(testTags[0]), "clicked tag shouldn't be in store"); - }) + }); it("clicking any odd number of times should put the tag in the store", () => { let oddNumber = 2 * Math.floor(Math.random() * 10) + 1; - for(let i=0; i < oddNumber; i++) { + for (let i = 0; i < oddNumber; i++) { clickFirstTag(); } let searchTags = store.state.gridSearch.searchTags; assert(searchTags.has(testTags[0]), "clicked tag not in store"); - }) + }); it("clicking any even number of times should remove the tag from the store", () => { let evenNumber = 2 * Math.floor(Math.random() * 10); - for(let i=0; i < evenNumber; i++) { + for (let i = 0; i < evenNumber; i++) { clickFirstTag(); } let searchTags = store.state.gridSearch.searchTags; assert(!searchTags.has(testTags[0]), "clicked tag shouldn't be in store"); - }) - - }) + }); + }); describe("community tag (tagClickFn: community_tag_click)", () => { - let propsData = { tags: ["abc", "def"], tagClickFn: "community_tag_click", @@ -94,26 +88,24 @@ describe("Tags/StandardTags.vue", () => { itemClass: "fakeItemClass" }; - beforeEach(function () { - wrapper = mount(StandardTags, { + beforeEach(function() { + wrapper = mount(StandardTags, { store, propsData, localVue }); - }) + }); afterEach(function() { mockRedirect.reset(); store.dispatch("clearSearchTags"); - }) + }); // tagClickFn=community_tag_click, clickUrl=something it("should try to redirect when you click a 'community tag'", () => { clickFirstTag(); let expectedUrl = `${propsData.clickUrl}?f-tags=${testTags[0]}`; assert(mockRedirect.calledWith(expectedUrl), "requested wrong url"); - }) - - }) - -}) + }); + }); +}); diff --git a/client/galaxy/scripts/components/Tags/StandardTags.vue b/client/galaxy/scripts/components/Tags/StandardTags.vue index 9d2e88a3420..08a82675e05 100644 --- a/client/galaxy/scripts/components/Tags/StandardTags.vue +++ b/client/galaxy/scripts/components/Tags/StandardTags.vue @@ -9,7 +9,8 @@ parameters here. --> +
No search results found for: {{ this.filter }} Date: Mon, 18 Mar 2019 14:06:03 -0400 Subject: [PATCH 0507/1016] Fix dependency cycle between galaxy.security and galaxy.model (1/2) ```python >>> from galaxy.security import get_permitted_actions Traceback (most recent call last): File "", line 1, in File "galaxy/security/__init__.py", line 12, in import galaxy.model File "galaxy/model/__init__.py", line 51, in from galaxy.security import get_permitted_actions ImportError: cannot import name get_permitted_actions ``` Previously galaxy.model import galaxy.security and galaxy.security import galaxy.model. This prevents certain seemingly reasonable imports from working properly (see above). The new break down of code is objectively less circular and matches the better pattern we've taken more recently of seperating interfaces from implementations to allow cleaner imports and dependencies. The fix is to use galaxy.security to define the interface for security operations on Galaxy components and moving the actual model dependent code into galaxy.model.security. Now galaxy.security has no dependencies on galaxy.model, galaxy.model (the raw model definitions) only depends on the interface and utilities methods, and the actual implementation of the security agent is found in galaxy.model.security that is effectively injected at runtime from galaxy.model.mapping. --- lib/galaxy/model/mapping.py | 2 +- lib/galaxy/{security/__init__.py => model/security.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename lib/galaxy/{security/__init__.py => model/security.py} (100%) diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index e750e757668..70341658519 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -40,7 +40,7 @@ from galaxy.model.base import ModelMapping from galaxy.model.custom_types import JSONType, MetadataType, TrimmedString, UUIDType from galaxy.model.orm.engine_factory import build_engine from galaxy.model.orm.now import now -from galaxy.security import GalaxyRBACAgent +from galaxy.model.security import GalaxyRBACAgent log = logging.getLogger(__name__) diff --git a/lib/galaxy/security/__init__.py b/lib/galaxy/model/security.py similarity index 100% rename from lib/galaxy/security/__init__.py rename to lib/galaxy/model/security.py From 96723959eaf4a1a4869aefb1fd2c7e64152f6d99 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 18 Mar 2019 16:07:46 -0400 Subject: [PATCH 0508/1016] Refactor security (2/2). Breaking one commit into two to try to preserve history. --- lib/galaxy/model/security.py | 145 +------------------------------- lib/galaxy/security/__init__.py | 144 +++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+), 143 deletions(-) create mode 100644 lib/galaxy/security/__init__.py diff --git a/lib/galaxy/model/security.py b/lib/galaxy/model/security.py index 44327ecfb68..ce86991244e 100644 --- a/lib/galaxy/model/security.py +++ b/lib/galaxy/model/security.py @@ -1,7 +1,3 @@ -""" -Galaxy Security - -""" import logging import socket from datetime import datetime, timedelta @@ -10,142 +6,14 @@ from sqlalchemy import and_, false, not_, or_ from sqlalchemy.orm import eagerload_all import galaxy.model +from galaxy.security import Action, RBACAgent from galaxy.util import listify from galaxy.util.bunch import Bunch + log = logging.getLogger(__name__) -class Action(object): - def __init__(self, action, description, model): - self.action = action - self.description = description - self.model = model - - -class RBACAgent(object): - """Class that handles galaxy security""" - permitted_actions = Bunch( - DATASET_MANAGE_PERMISSIONS=Action("manage permissions", "Users having associated role can manage the roles associated with permissions on this dataset.", "grant"), - DATASET_ACCESS=Action("access", "Users having associated role can import this dataset into their history for analysis.", "restrict"), - LIBRARY_ACCESS=Action("access library", "Restrict access to this library to only users having associated role", "restrict"), - LIBRARY_ADD=Action("add library item", "Users having associated role can add library items to this library item", "grant"), - LIBRARY_MODIFY=Action("modify library item", "Users having associated role can modify this library item", "grant"), - LIBRARY_MANAGE=Action("manage library permissions", "Users having associated role can manage roles associated with permissions on this library item", "grant") - ) - - def get_action(self, name, default=None): - """Get a permitted action by its dict key or action name""" - for k, v in self.permitted_actions.items(): - if k == name or v.action == name: - return v - return default - - def get_actions(self): - """Get all permitted actions as a list of Action objects""" - return list(self.permitted_actions.__dict__.values()) - - def get_item_actions(self, action, item): - raise Exception('No valid method of retrieving action (%s) for item %s.' % (action, item)) - - def guess_derived_permissions_for_datasets(self, datasets=[]): - raise Exception("Unimplemented Method") - - def can_access_dataset(self, roles, dataset): - raise Exception("Unimplemented Method") - - def can_manage_dataset(self, roles, dataset): - raise Exception("Unimplemented Method") - - def can_access_library(self, roles, library): - raise Exception("Unimplemented Method") - - def can_add_library_item(self, roles, item): - raise Exception("Unimplemented Method") - - def can_modify_library_item(self, roles, item): - raise Exception("Unimplemented Method") - - def can_manage_library_item(self, roles, item): - raise Exception("Unimplemented Method") - - def associate_components(self, **kwd): - raise Exception('No valid method of associating provided components: %s' % kwd) - - def create_private_user_role(self, user): - raise Exception("Unimplemented Method") - - def get_private_user_role(self, user): - raise Exception("Unimplemented Method") - - def user_set_default_permissions(self, user, permissions={}, history=False, dataset=False): - raise Exception("Unimplemented Method") - - def history_set_default_permissions(self, history, permissions=None, dataset=False, bypass_manage_permission=False): - raise Exception("Unimplemented Method") - - def set_all_dataset_permissions(self, dataset, permissions, new=False): - raise Exception("Unimplemented Method") - - def set_dataset_permission(self, dataset, permission): - raise Exception("Unimplemented Method") - - def set_all_library_permissions(self, trans, dataset, permissions): - raise Exception("Unimplemented Method") - - def set_library_item_permission(self, library_item, permission): - raise Exception("Unimplemented Method") - - def library_is_public(self, library): - raise Exception("Unimplemented Method") - - def make_library_public(self, library): - raise Exception("Unimplemented Method") - - def get_accessible_libraries(self, trans, user): - raise Exception("Unimplemented Method") - - def get_permitted_libraries(self, trans, user, actions): - raise Exception("Unimplemented Method") - - def folder_is_public(self, library): - raise Exception("Unimplemented Method") - - def make_folder_public(self, folder, count=0): - raise Exception("Unimplemented Method") - - def dataset_is_public(self, dataset): - raise Exception("Unimplemented Method") - - def make_dataset_public(self, dataset): - raise Exception("Unimplemented Method") - - def get_permissions(self, library_dataset): - raise Exception("Unimplemented Method") - - def get_all_roles(self, trans, cntrller): - raise Exception("Unimplemented Method") - - def get_legitimate_roles(self, trans, item, cntrller): - raise Exception("Unimplemented Method") - - def derive_roles_from_access(self, trans, item_id, cntrller, library=False, **kwd): - raise Exception("Unimplemented Method") - - def get_component_associations(self, **kwd): - raise Exception("Unimplemented Method") - - def components_are_associated(self, **kwd): - return bool(self.get_component_associations(**kwd)) - - def convert_permitted_action_strings(self, permitted_action_strings): - """ - When getting permitted actions from an untrusted source like a - form, ensure that they match our actual permitted actions. - """ - return [_ for _ in [self.permitted_actions.get(action_string) for action_string in permitted_action_strings] if _ is not None] - - class GalaxyRBACAgent(RBACAgent): def __init__(self, model, permitted_actions=None): self.model = model @@ -1588,12 +1456,3 @@ class HostAgent(RBACAgent): hdadaa = self.model.HistoryDatasetAssociationDisplayAtAuthorization(hda=hda, user=user, site=site) self.sa_session.add(hdadaa) self.sa_session.flush() - - -def get_permitted_actions(filter=None): - '''Utility method to return a subset of RBACAgent's permitted actions''' - if filter is None: - return RBACAgent.permitted_actions - tmp_bunch = Bunch() - [tmp_bunch.__dict__.__setitem__(k, v) for k, v in RBACAgent.permitted_actions.items() if k.startswith(filter)] - return tmp_bunch diff --git a/lib/galaxy/security/__init__.py b/lib/galaxy/security/__init__.py new file mode 100644 index 00000000000..d1efb9c7a6e --- /dev/null +++ b/lib/galaxy/security/__init__.py @@ -0,0 +1,144 @@ +""" +Galaxy Security + +""" +from galaxy.util.bunch import Bunch + + +class Action(object): + def __init__(self, action, description, model): + self.action = action + self.description = description + self.model = model + + +class RBACAgent(object): + """Class that handles galaxy security""" + permitted_actions = Bunch( + DATASET_MANAGE_PERMISSIONS=Action("manage permissions", "Users having associated role can manage the roles associated with permissions on this dataset.", "grant"), + DATASET_ACCESS=Action("access", "Users having associated role can import this dataset into their history for analysis.", "restrict"), + LIBRARY_ACCESS=Action("access library", "Restrict access to this library to only users having associated role", "restrict"), + LIBRARY_ADD=Action("add library item", "Users having associated role can add library items to this library item", "grant"), + LIBRARY_MODIFY=Action("modify library item", "Users having associated role can modify this library item", "grant"), + LIBRARY_MANAGE=Action("manage library permissions", "Users having associated role can manage roles associated with permissions on this library item", "grant") + ) + + def get_action(self, name, default=None): + """Get a permitted action by its dict key or action name""" + for k, v in self.permitted_actions.items(): + if k == name or v.action == name: + return v + return default + + def get_actions(self): + """Get all permitted actions as a list of Action objects""" + return list(self.permitted_actions.__dict__.values()) + + def get_item_actions(self, action, item): + raise Exception('No valid method of retrieving action (%s) for item %s.' % (action, item)) + + def guess_derived_permissions_for_datasets(self, datasets=[]): + raise Exception("Unimplemented Method") + + def can_access_dataset(self, roles, dataset): + raise Exception("Unimplemented Method") + + def can_manage_dataset(self, roles, dataset): + raise Exception("Unimplemented Method") + + def can_access_library(self, roles, library): + raise Exception("Unimplemented Method") + + def can_add_library_item(self, roles, item): + raise Exception("Unimplemented Method") + + def can_modify_library_item(self, roles, item): + raise Exception("Unimplemented Method") + + def can_manage_library_item(self, roles, item): + raise Exception("Unimplemented Method") + + def associate_components(self, **kwd): + raise Exception('No valid method of associating provided components: %s' % kwd) + + def create_private_user_role(self, user): + raise Exception("Unimplemented Method") + + def get_private_user_role(self, user): + raise Exception("Unimplemented Method") + + def user_set_default_permissions(self, user, permissions={}, history=False, dataset=False): + raise Exception("Unimplemented Method") + + def history_set_default_permissions(self, history, permissions=None, dataset=False, bypass_manage_permission=False): + raise Exception("Unimplemented Method") + + def set_all_dataset_permissions(self, dataset, permissions, new=False): + raise Exception("Unimplemented Method") + + def set_dataset_permission(self, dataset, permission): + raise Exception("Unimplemented Method") + + def set_all_library_permissions(self, trans, dataset, permissions): + raise Exception("Unimplemented Method") + + def set_library_item_permission(self, library_item, permission): + raise Exception("Unimplemented Method") + + def library_is_public(self, library): + raise Exception("Unimplemented Method") + + def make_library_public(self, library): + raise Exception("Unimplemented Method") + + def get_accessible_libraries(self, trans, user): + raise Exception("Unimplemented Method") + + def get_permitted_libraries(self, trans, user, actions): + raise Exception("Unimplemented Method") + + def folder_is_public(self, library): + raise Exception("Unimplemented Method") + + def make_folder_public(self, folder, count=0): + raise Exception("Unimplemented Method") + + def dataset_is_public(self, dataset): + raise Exception("Unimplemented Method") + + def make_dataset_public(self, dataset): + raise Exception("Unimplemented Method") + + def get_permissions(self, library_dataset): + raise Exception("Unimplemented Method") + + def get_all_roles(self, trans, cntrller): + raise Exception("Unimplemented Method") + + def get_legitimate_roles(self, trans, item, cntrller): + raise Exception("Unimplemented Method") + + def derive_roles_from_access(self, trans, item_id, cntrller, library=False, **kwd): + raise Exception("Unimplemented Method") + + def get_component_associations(self, **kwd): + raise Exception("Unimplemented Method") + + def components_are_associated(self, **kwd): + return bool(self.get_component_associations(**kwd)) + + def convert_permitted_action_strings(self, permitted_action_strings): + """ + When getting permitted actions from an untrusted source like a + form, ensure that they match our actual permitted actions. + """ + return [_ for _ in [self.permitted_actions.get(action_string) for action_string in permitted_action_strings] if _ is not None] + + +def get_permitted_actions(filter=None): + '''Utility method to return a subset of RBACAgent's permitted actions''' + if filter is None: + return RBACAgent.permitted_actions + tmp_bunch = Bunch() + [tmp_bunch.__dict__.__setitem__(k, v) for k, v in RBACAgent.permitted_actions.items() if k.startswith(filter)] + return tmp_bunch From 78eed4fcd55ad288287092e7fa05e875ea87aba0 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 18 Mar 2019 16:11:54 -0400 Subject: [PATCH 0509/1016] Another fix for security/model cycle fix. --- lib/galaxy/app.py | 3 ++- lib/galaxy/model/security.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 1e48845314d..ca62d1dedbf 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -6,6 +6,7 @@ import sys import time import galaxy.model +import galaxy.model.security import galaxy.queues import galaxy.quota import galaxy.security @@ -154,7 +155,7 @@ class UniverseApplication(config.ConfiguresGalaxyMixin): self.webhooks_registry = WebhooksRegistry(self.config.webhooks_dirs) # Load security policy. self.security_agent = self.model.security_agent - self.host_security_agent = galaxy.security.HostAgent( + self.host_security_agent = galaxy.model.security.HostAgent( model=self.security_agent.model, permitted_actions=self.security_agent.permitted_actions) # Load quota management. diff --git a/lib/galaxy/model/security.py b/lib/galaxy/model/security.py index ce86991244e..698942a7d00 100644 --- a/lib/galaxy/model/security.py +++ b/lib/galaxy/model/security.py @@ -6,7 +6,7 @@ from sqlalchemy import and_, false, not_, or_ from sqlalchemy.orm import eagerload_all import galaxy.model -from galaxy.security import Action, RBACAgent +from galaxy.security import Action, get_permitted_actions, RBACAgent from galaxy.util import listify from galaxy.util.bunch import Bunch From deb8f02104610899dd6dd458b5b4ac7c7413f9f9 Mon Sep 17 00:00:00 2001 From: guerler Date: Tue, 19 Mar 2019 02:37:05 -0400 Subject: [PATCH 0510/1016] Properly link filter to pagination --- client/galaxy/scripts/components/DataDialog.vue | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/client/galaxy/scripts/components/DataDialog.vue b/client/galaxy/scripts/components/DataDialog.vue index d93bfedd8c5..b165792a8f5 100644 --- a/client/galaxy/scripts/components/DataDialog.vue +++ b/client/galaxy/scripts/components/DataDialog.vue @@ -44,10 +44,10 @@ -
@@ -139,9 +139,6 @@ export default { } } return this.items; - }, - rows() { - return this.items.length } }, created: function() { @@ -153,6 +150,7 @@ export default { }, filtered: function(items) { this.nItems = items.length; + this.currentPage = 1 }, clicked: function(record) { if (this.isDataset(record)) { @@ -208,6 +206,7 @@ export default { .then(response => { this.items = []; this.stack = [response.data]; + this.filter = null; while (this.stack.length > 0) { let root = this.stack.pop(); if (Array.isArray(root)) { From 38e888b87a7a4d1ed50b79219c0927c930075558 Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Tue, 19 Mar 2019 09:56:08 +0100 Subject: [PATCH 0511/1016] Improve linking --- config/job_conf.xml.sample_advanced | 3 +++ doc/source/admin/scaling.md | 12 ++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/config/job_conf.xml.sample_advanced b/config/job_conf.xml.sample_advanced index 97a308576b8..7f9715ad843 100644 --- a/config/job_conf.xml.sample_advanced +++ b/config/job_conf.xml.sample_advanced @@ -329,6 +329,9 @@ + https://.../auth/realms/.../protocol/openid-connect/auth + https://.../auth/realms/.../protocol/openid-connect/token + https://.../auth/realms/.../protocol/openid-connect/userinfo + ... + + diff --git a/lib/galaxy/authnz/keycloak_authnz.py b/lib/galaxy/authnz/keycloak_authnz.py index 024128fd93c..3bb4e175cac 100644 --- a/lib/galaxy/authnz/keycloak_authnz.py +++ b/lib/galaxy/authnz/keycloak_authnz.py @@ -22,6 +22,7 @@ class KeycloakAuthnz(IdentityProvider): self.config['authorization_endpoint'] = oidc_backend_config['authorization_endpoint'] self.config['token_endpoint'] = oidc_backend_config['token_endpoint'] self.config['userinfo_endpoint'] = oidc_backend_config['userinfo_endpoint'] + self.config['idp_hint'] = oidc_backend_config.get('idp_hint', None) def authenticate(self, trans): client_id = self.config['client_id'] @@ -30,8 +31,11 @@ class KeycloakAuthnz(IdentityProvider): oauth2_session = OAuth2Session( client_id, scope=('openid', 'email', 'profile'), redirect_uri=redirect_uri) nonce = generate_nonce() + extra_params = {"nonce": nonce} + if self.config['idp_hint']: + extra_params['kc_idp_hint'] = self.config['idp_hint'] authorization_url, state = oauth2_session.authorization_url( - base_authorize_url, nonce=nonce) + base_authorize_url, **extra_params) # store state and nonce in database trans.sa_session.add(KeycloakAuthRequest(nonce, state)) trans.sa_session.flush() diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index a76a565911c..1dabfd2a173 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -118,6 +118,8 @@ class AuthnzManager(object): 'authorization_endpoint': config_xml.find('authorization_endpoint').text, 'token_endpoint': config_xml.find('token_endpoint').text, 'userinfo_endpoint': config_xml.find('userinfo_endpoint').text} + if config_xml.find('idp_hint') is not None: + rtv['idp_hint'] = config_xml.find('idp_hint').text return rtv def _unify_provider_implementation_names(self, provider, implementation): From bb4fd407345c7e664d629907b644c2e11d46a60c Mon Sep 17 00:00:00 2001 From: Marcus Christie Date: Fri, 4 Jan 2019 16:33:05 -0500 Subject: [PATCH 0532/1016] Fix state parameter validation and store in cookie --- lib/galaxy/authnz/keycloak_authnz.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/authnz/keycloak_authnz.py b/lib/galaxy/authnz/keycloak_authnz.py index 3bb4e175cac..cea97c1d69f 100644 --- a/lib/galaxy/authnz/keycloak_authnz.py +++ b/lib/galaxy/authnz/keycloak_authnz.py @@ -36,6 +36,7 @@ class KeycloakAuthnz(IdentityProvider): extra_params['kc_idp_hint'] = self.config['idp_hint'] authorization_url, state = oauth2_session.authorization_url( base_authorize_url, **extra_params) + trans.set_cookie(value=state, name='keycloakauth-state') # store state and nonce in database trans.sa_session.add(KeycloakAuthRequest(nonce, state)) trans.sa_session.flush() @@ -47,10 +48,14 @@ class KeycloakAuthnz(IdentityProvider): redirect_uri = self.config['redirect_uri'] token_endpoint = self.config['token_endpoint'] userinfo_endpoint = self.config['userinfo_endpoint'] + # Take state value to validate from token. OAuth2Session.fetch_token + # will validate that the state query parameter value on the URL matches + # this value. + state_cookie = trans.get_cookie(name='keycloakauth-state') oauth2_session = OAuth2Session(client_id, scope=('openid', 'email', 'profile'), redirect_uri=redirect_uri, - state=state_token) + state=state_cookie) token = oauth2_session.fetch_token( token_endpoint, client_secret=client_secret, authorization_response=trans.request.url) From 80aaaab4a5e63021e156a8935b0ad3e298f9a3dc Mon Sep 17 00:00:00 2001 From: Marcus Christie Date: Fri, 4 Jan 2019 16:54:40 -0500 Subject: [PATCH 0533/1016] Checking nonce; storing nonce and state in cookies instead of db --- lib/galaxy/authnz/keycloak_authnz.py | 32 ++++++++++++------- lib/galaxy/model/__init__.py | 7 ---- lib/galaxy/model/mapping.py | 8 ----- .../versions/0147_add_keycloak_auth_tables.py | 14 ++------ 4 files changed, 23 insertions(+), 38 deletions(-) diff --git a/lib/galaxy/authnz/keycloak_authnz.py b/lib/galaxy/authnz/keycloak_authnz.py index cea97c1d69f..df377ca40f3 100644 --- a/lib/galaxy/authnz/keycloak_authnz.py +++ b/lib/galaxy/authnz/keycloak_authnz.py @@ -1,12 +1,14 @@ +import hashlib import json import logging from datetime import datetime, timedelta +import jwt from oauthlib.common import generate_nonce from requests_oauthlib import OAuth2Session -from galaxy.model import KeycloakAccessToken, KeycloakAuthRequest, User +from galaxy.model import KeycloakAccessToken, User from ..authnz import IdentityProvider log = logging.getLogger(__name__) @@ -31,15 +33,14 @@ class KeycloakAuthnz(IdentityProvider): oauth2_session = OAuth2Session( client_id, scope=('openid', 'email', 'profile'), redirect_uri=redirect_uri) nonce = generate_nonce() - extra_params = {"nonce": nonce} + nonce_hash = hashlib.sha256(nonce).hexdigest() + extra_params = {"nonce": nonce_hash} if self.config['idp_hint']: extra_params['kc_idp_hint'] = self.config['idp_hint'] authorization_url, state = oauth2_session.authorization_url( base_authorize_url, **extra_params) trans.set_cookie(value=state, name='keycloakauth-state') - # store state and nonce in database - trans.sa_session.add(KeycloakAuthRequest(nonce, state)) - trans.sa_session.flush() + trans.set_cookie(value=nonce, name='keycloakauth-nonce') return authorization_url def callback(self, state_token, authz_code, trans, login_redirect_url): @@ -60,18 +61,25 @@ class KeycloakAuthnz(IdentityProvider): token_endpoint, client_secret=client_secret, authorization_response=trans.request.url) log.debug("token={}".format(json.dumps(token, indent=True))) - # TODO: get nonce from token['id_token'] and validate - # TODO: delete the nonce record once it is validated - userinfo = oauth2_session.get(userinfo_endpoint).json() - log.debug("userinfo={}".format(json.dumps(userinfo, indent=True))) - username = userinfo["preferred_username"] - email = userinfo["email"] - user = self._get_user(trans.sa_session, username, email) access_token = token['access_token'] id_token = token['id_token'] refresh_token = token['refresh_token'] expiration_time = datetime.now() + timedelta(seconds=token['expires_in']) refresh_expiration_time = datetime.now() + timedelta(seconds=token['refresh_expires_in']) + # get nonce from token['id_token'] and validate + id_token_decoded = jwt.decode(id_token, verify=False) + nonce_hash = id_token_decoded['nonce'] + nonce_cookie = trans.get_cookie(name='keycloakauth-nonce') + # Delete the nonce cookie + trans.set_cookie('', name='keycloakauth-nonce', age=-1) + nonce_cookie_hash = hashlib.sha256(nonce_cookie).hexdigest() + if nonce_hash != nonce_cookie_hash: + raise Exception("Nonce mismatch!") + userinfo = oauth2_session.get(userinfo_endpoint).json() + log.debug("userinfo={}".format(json.dumps(userinfo, indent=True))) + username = userinfo["preferred_username"] + email = userinfo["email"] + user = self._get_user(trans.sa_session, username, email) keycloak_access_token = KeycloakAccessToken(user=user, access_token=access_token, id_token=id_token, diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index f0a300d2c36..6b38d6e3f1e 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -5099,13 +5099,6 @@ class UserAuthnzToken(UserMixin, RepresentById): return instance -class KeycloakAuthRequest(RepresentById): - - def __init__(self, nonce=None, state=None): - self.nonce = nonce - self.state = state - - class KeycloakAccessToken(RepresentById): def __init__(self, user, access_token, id_token, refresh_token, expiration_time, refresh_expiration_time, raw_token): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 629307811d1..c39252494a3 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -123,12 +123,6 @@ model.UserAuthnzToken.table = Table( Column('lifetime', Integer), Column('assoc_type', VARCHAR(64))) -model.KeycloakAuthRequest.table = Table( - "keycloak_auth_request", metadata, - Column('nonce', String(255), primary_key=True), - Column('state', String(64)) -) - model.KeycloakAccessToken.table = Table( "keycloak_access_token", metadata, Column('id', Integer, primary_key=True), @@ -1529,8 +1523,6 @@ mapper(model.UserAuthnzToken, model.UserAuthnzToken.table, properties=dict( backref='social_auth') )) -mapper(model.KeycloakAuthRequest, model.KeycloakAuthRequest.table, properties=None) - mapper(model.KeycloakAccessToken, model.KeycloakAccessToken.table, properties=dict( user=relation(model.User, primaryjoin=(model.KeycloakAccessToken.table.c.user_id == model.User.table.c.id), diff --git a/lib/galaxy/model/migrate/versions/0147_add_keycloak_auth_tables.py b/lib/galaxy/model/migrate/versions/0147_add_keycloak_auth_tables.py index 58da237c94f..29f9c6ee944 100644 --- a/lib/galaxy/model/migrate/versions/0147_add_keycloak_auth_tables.py +++ b/lib/galaxy/model/migrate/versions/0147_add_keycloak_auth_tables.py @@ -1,24 +1,18 @@ """ -Migration for adding keycloak_auth_request, keycloak_access_token tables. +Migration for adding keycloak_access_token table. """ from __future__ import print_function import logging -from sqlalchemy import (Column, DateTime, ForeignKey, Integer, MetaData, - String, Table, Text) +from sqlalchemy import (Column, DateTime, ForeignKey, Integer, MetaData, Table, + Text) from galaxy.model.custom_types import JSONType log = logging.getLogger(__name__) metadata = MetaData() -KeycloakAuthRequest_table = Table( - "keycloak_auth_request", metadata, - Column('nonce', String(255), primary_key=True), - Column('state', String(64)) -) - KeycloakAccessToken_table = Table( "keycloak_access_token", metadata, Column('id', Integer, primary_key=True), @@ -38,7 +32,6 @@ def upgrade(migrate_engine): metadata.reflect() try: - KeycloakAuthRequest_table.create() KeycloakAccessToken_table.create() except Exception: log.exception("Failed to create Keycloak auth tables") @@ -49,7 +42,6 @@ def downgrade(migrate_engine): metadata.reflect() try: - KeycloakAuthRequest_table.drop() KeycloakAccessToken_table.drop() except Exception: log.exception("Failed to drop Keycloak auth tables") From cbdc408f5deb048220c8023341fc288ccddc0878 Mon Sep 17 00:00:00 2001 From: Marcus Christie Date: Mon, 7 Jan 2019 14:57:05 -0500 Subject: [PATCH 0534/1016] Some small refactorings --- lib/galaxy/authnz/keycloak_authnz.py | 37 +++++++++++++++++++--------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/authnz/keycloak_authnz.py b/lib/galaxy/authnz/keycloak_authnz.py index df377ca40f3..ae023749dd5 100644 --- a/lib/galaxy/authnz/keycloak_authnz.py +++ b/lib/galaxy/authnz/keycloak_authnz.py @@ -12,6 +12,8 @@ from galaxy.model import KeycloakAccessToken, User from ..authnz import IdentityProvider log = logging.getLogger(__name__) +STATE_COOKIE_NAME = 'keycloakauth-state' +NONCE_COOKIE_NAME = 'keycloakauth-nonce' class KeycloakAuthnz(IdentityProvider): @@ -33,14 +35,14 @@ class KeycloakAuthnz(IdentityProvider): oauth2_session = OAuth2Session( client_id, scope=('openid', 'email', 'profile'), redirect_uri=redirect_uri) nonce = generate_nonce() - nonce_hash = hashlib.sha256(nonce).hexdigest() + nonce_hash = self._hash_nonce(nonce) extra_params = {"nonce": nonce_hash} if self.config['idp_hint']: extra_params['kc_idp_hint'] = self.config['idp_hint'] authorization_url, state = oauth2_session.authorization_url( base_authorize_url, **extra_params) - trans.set_cookie(value=state, name='keycloakauth-state') - trans.set_cookie(value=nonce, name='keycloakauth-nonce') + trans.set_cookie(value=state, name=STATE_COOKIE_NAME) + trans.set_cookie(value=nonce, name=NONCE_COOKIE_NAME) return authorization_url def callback(self, state_token, authz_code, trans, login_redirect_url): @@ -52,7 +54,7 @@ class KeycloakAuthnz(IdentityProvider): # Take state value to validate from token. OAuth2Session.fetch_token # will validate that the state query parameter value on the URL matches # this value. - state_cookie = trans.get_cookie(name='keycloakauth-state') + state_cookie = trans.get_cookie(name=STATE_COOKIE_NAME) oauth2_session = OAuth2Session(client_id, scope=('openid', 'email', 'profile'), redirect_uri=redirect_uri, @@ -66,15 +68,15 @@ class KeycloakAuthnz(IdentityProvider): refresh_token = token['refresh_token'] expiration_time = datetime.now() + timedelta(seconds=token['expires_in']) refresh_expiration_time = datetime.now() + timedelta(seconds=token['refresh_expires_in']) - # get nonce from token['id_token'] and validate + + # Get nonce from token['id_token'] and validate. 'nonce' in the + # id_token is a hash of the nonce stored in the NONCE_COOKIE_NAME + # cookie. id_token_decoded = jwt.decode(id_token, verify=False) nonce_hash = id_token_decoded['nonce'] - nonce_cookie = trans.get_cookie(name='keycloakauth-nonce') - # Delete the nonce cookie - trans.set_cookie('', name='keycloakauth-nonce', age=-1) - nonce_cookie_hash = hashlib.sha256(nonce_cookie).hexdigest() - if nonce_hash != nonce_cookie_hash: - raise Exception("Nonce mismatch!") + self._validate_nonce(trans, nonce_hash) + + # Get userinfo and lookup/create Galaxy user record userinfo = oauth2_session.get(userinfo_endpoint).json() log.debug("userinfo={}".format(json.dumps(userinfo, indent=True))) username = userinfo["preferred_username"] @@ -92,7 +94,7 @@ class KeycloakAuthnz(IdentityProvider): return login_redirect_url, user def disconnect(self, provider, trans, disconnect_redirect_url=None, association_id=None): - # TODO: implement + # TODO: implement? pass def _get_user(self, sa_session, username, email): @@ -107,3 +109,14 @@ class KeycloakAuthnz(IdentityProvider): sa_session.add(user) sa_session.flush() return user + + def _hash_nonce(self, nonce): + return hashlib.sha256(nonce).hexdigest() + + def _validate_nonce(self, trans, nonce_hash): + nonce_cookie = trans.get_cookie(name=NONCE_COOKIE_NAME) + # Delete the nonce cookie + trans.set_cookie('', name=NONCE_COOKIE_NAME, age=-1) + nonce_cookie_hash = self._hash_nonce(nonce_cookie) + if nonce_hash != nonce_cookie_hash: + raise Exception("Nonce mismatch!") From 7e7f53c6e090f3022553d84870e62a321ae23e56 Mon Sep 17 00:00:00 2001 From: Marcus Christie Date: Mon, 7 Jan 2019 15:47:22 -0500 Subject: [PATCH 0535/1016] Simplifying configuration by using .well-known config url --- config/oidc_backends_config.xml.sample | 12 +++++------- lib/galaxy/authnz/keycloak_authnz.py | 14 +++++++++++--- lib/galaxy/authnz/managers.py | 4 +--- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/config/oidc_backends_config.xml.sample b/config/oidc_backends_config.xml.sample index 3609f3e8503..9c2a72a669b 100644 --- a/config/oidc_backends_config.xml.sample +++ b/config/oidc_backends_config.xml.sample @@ -36,15 +36,13 @@ ... ... http://localhost:8000/authnz/keycloak-auth/keycloak/callback - - https://.../auth/realms/.../protocol/openid-connect/auth - https://.../auth/realms/.../protocol/openid-connect/token - https://.../auth/realms/.../protocol/openid-connect/userinfo + https://.../auth/realms/.../.well-known/openid-configuration ... - https://.../auth/realms/.../.well-known/openid-configuration - ... - + http://localhost:8000/authnz/google/oidc/callback + + + https://.../.well-known/openid-configuration + + + + + + + + + + + + + + + diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 60b2898b13a..f31b64fb938 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -114,10 +114,21 @@ class AuthnzManager(object): rtv = { 'client_id': config_xml.find('client_id').text, 'client_secret': config_xml.find('client_secret').text, - 'redirect_uri': config_xml.find('redirect_uri').text, - 'well_known_oidc_config_uri': config_xml.find('well_known_oidc_config_uri').text} - if config_xml.find('idp_hint') is not None: - rtv['idp_hint'] = config_xml.find('idp_hint').text + 'redirect_uri': config_xml.find('redirect_uri').text} + if config_xml.find('well_known_oidc_config_uri') is not None: + rtv['well_known_oidc_config_uri'] = config_xml.find('well_known_oidc_config_uri').text + else: + rtv['authorization_endpoint'] = config_xml.find('authorization_endpoint').text + rtv['token_endpoint'] = config_xml.find('token_endpoint').text + rtv['userinfo_endpoint'] = config_xml.find('userinfo_endpoint').text + if config_xml.find('extra_params') is not None: + rtv['extra_params'] = {} + for extra_param in config_xml.find('extra_params'): + rtv['extra_params'][extra_param.attrib['name']] = extra_param.attrib['value'] + if config_xml.find('userinfo_claim_mappings') is not None: + rtv['userinfo_claim_mappings'] = {} + for claim_mapping in config_xml.find('userinfo_claim_mappings'): + rtv['userinfo_claim_mappings'][claim_mapping.tag] = claim_mapping.text return rtv def _unify_provider_implementation_names(self, provider, implementation): diff --git a/lib/galaxy/authnz/oidc_authnz.py b/lib/galaxy/authnz/oidc_authnz.py index 9a72be34e31..9de0a9fd906 100644 --- a/lib/galaxy/authnz/oidc_authnz.py +++ b/lib/galaxy/authnz/oidc_authnz.py @@ -15,6 +15,9 @@ from ..authnz import IdentityProvider log = logging.getLogger(__name__) STATE_COOKIE_NAME = 'oidc-state' NONCE_COOKIE_NAME = 'oidc-nonce' +DEFAULT_CLAIM_USERNAME = 'preferred_username' +DEFAULT_CLAIM_EMAIL = 'email' +DEFAULT_CLAIM_ID = 'sub' class OIDCAuthnz(IdentityProvider): @@ -36,7 +39,8 @@ class OIDCAuthnz(IdentityProvider): self.config['authorization_endpoint'] = oidc_backend_config['authorization_endpoint'] self.config['token_endpoint'] = oidc_backend_config['token_endpoint'] self.config['userinfo_endpoint'] = oidc_backend_config['userinfo_endpoint'] - self.config['idp_hint'] = oidc_backend_config.get('idp_hint', None) + self.config['extra_params'] = oidc_backend_config.get('extra_params', None) + self.config['userinfo_claim_mappings'] = oidc_backend_config.get('userinfo_claim_mappings', None) def authenticate(self, trans): base_authorize_url = self.config['authorization_endpoint'] @@ -44,8 +48,8 @@ class OIDCAuthnz(IdentityProvider): nonce = generate_nonce() nonce_hash = self._hash_nonce(nonce) extra_params = {"nonce": nonce_hash} - if self.config['idp_hint']: - extra_params['kc_idp_hint'] = self.config['idp_hint'] + if self.config['extra_params']: + extra_params.update(self.config['extra_params']) authorization_url, state = oauth2_session.authorization_url( base_authorize_url, **extra_params) trans.set_cookie(value=state, name=STATE_COOKIE_NAME) @@ -76,16 +80,17 @@ class OIDCAuthnz(IdentityProvider): # Get userinfo and lookup/create Galaxy user record userinfo = self._get_userinfo(oauth2_session) log.debug("userinfo={}".format(json.dumps(userinfo, indent=True))) - username = userinfo["preferred_username"] - email = userinfo["email"] - keycloak_user_id = userinfo["sub"] + claim_mappings = self._get_claim_mappings() + username = userinfo[claim_mappings['username']] + email = userinfo[claim_mappings['email']] + user_id = userinfo[claim_mappings['id']] # Create or update keycloak_access_token record - keycloak_access_token = self._get_keycloak_access_token(trans.sa_session, keycloak_user_id) + keycloak_access_token = self._get_keycloak_access_token(trans.sa_session, user_id) if keycloak_access_token is None: user = self._create_user(trans.sa_session, username, email) keycloak_access_token = KeycloakAccessToken(user=user, - keycloak_user_id=keycloak_user_id, + keycloak_user_id=user_id, access_token=access_token, id_token=id_token, refresh_token=refresh_token, @@ -126,10 +131,20 @@ class OIDCAuthnz(IdentityProvider): def _get_userinfo(self, oauth2_session): userinfo_endpoint = self.config['userinfo_endpoint'] return oauth2_session.get(userinfo_endpoint).json() + + def _get_claim_mappings(self): + userinfo_claim_mappings = { + 'username': DEFAULT_CLAIM_USERNAME, + 'email': DEFAULT_CLAIM_EMAIL, + 'id': DEFAULT_CLAIM_ID + } + if self.config['userinfo_claim_mappings']: + userinfo_claim_mappings.update(self.config['userinfo_claim_mappings']) + return userinfo_claim_mappings - def _get_keycloak_access_token(self, sa_session, keycloak_user_id): + def _get_keycloak_access_token(self, sa_session, user_id): return sa_session.query(KeycloakAccessToken).filter_by( - keycloak_user_id=keycloak_user_id).one_or_none() + keycloak_user_id=user_id).one_or_none() def _create_user(self, sa_session, username, email): user = User(email=email, username=username) diff --git a/test/unit/authnz/test_oidc_authnz.py b/test/unit/authnz/test_oidc_authnz.py index 2c48b525ef6..26cabffeeaf 100644 --- a/test/unit/authnz/test_oidc_authnz.py +++ b/test/unit/authnz/test_oidc_authnz.py @@ -32,23 +32,32 @@ class OIDCAuthnzTestCase(unittest.TestCase): 'client_id': 'test-client-id', 'client_secret': 'test-client-secret', 'redirect_uri': 'https://test-redirect-uri', - 'well_known_oidc_config_uri': 'https://test-well-known-oidc-config-uri' + 'well_known_oidc_config_uri': 'https://test-well-known-oidc-config-uri', + 'extra_params': { + 'param1': 'value1' + } }) - self.mock_fetch_token(self.oidc_authnz) - self.mock_get_userinfo(self.oidc_authnz) - self.trans = self.mockTrans() + self.setupMocks() self.test_state = "abc123" self.test_nonce = "4662892146306485421546981092" self.test_nonce_hash = hashlib.sha256(self.test_nonce).hexdigest() self.test_code = "test-code" self.test_username = "test-username" self.test_email = "test-email" + self.test_alt_username = "test-alt-username" + self.test_alt_email = "test-alt-email" self.test_access_token = "test_access_token" self.test_refresh_token = "test_refresh_token" self.test_expires_in = 30 self.test_refresh_expires_in = 1800 - self.test_keycloak_user_id = str(uuid.uuid4()) - self.trans.request.url = "https://localhost:8000/authnz/galaxy-auth/keycloak/callback?state={test_state}&code={test_code}".format(test_state=self.test_state, test_code=self.test_code) + self.test_user_id = str(uuid.uuid4()) + self.test_alt_user_id = str(uuid.uuid4()) + self.trans.request.url = "https://localhost:8000/authnz/google/oidc/callback?state={test_state}&code={test_code}".format(test_state=self.test_state, test_code=self.test_code) + + def setupMocks(self): + self.mock_fetch_token(self.oidc_authnz) + self.mock_get_userinfo(self.oidc_authnz) + self.trans = self.mockTrans() @property def test_id_token(self): @@ -82,7 +91,10 @@ class OIDCAuthnzTestCase(unittest.TestCase): return { "preferred_username": self.test_username, "email": self.test_email, - "sub": self.test_keycloak_user_id + "sub": self.test_user_id, + "alt_username": self.test_alt_username, + "alt_email": self.test_alt_email, + "alt_id": self.test_alt_user_id } oidc_authnz._get_userinfo = get_userinfo @@ -175,6 +187,28 @@ class OIDCAuthnzTestCase(unittest.TestCase): self.assertEqual(self.oidc_authnz.config['authorization_endpoint'], 'https://test-auth-endpoint') self.assertEqual(self.oidc_authnz.config['token_endpoint'], 'https://test-token-endpoint') self.assertEqual(self.oidc_authnz.config['userinfo_endpoint'], 'https://test-userinfo-endpoint') + self.assertIn('param1', self.oidc_authnz.config['extra_params']) + self.assertEqual(self.oidc_authnz.config['extra_params']['param1'], 'value1') + + def test_parse_config_without_well_known_config(self): + self.oidc_authnz = oidc_authnz.OIDCAuthnz('Google', { + 'VERIFY_SSL': True + }, { + 'client_id': 'test-client-id2', + 'client_secret': 'test-client-secret2', + 'redirect_uri': 'https://test-redirect-uri2', + "authorization_endpoint": "https://test-auth-endpoint2", + "token_endpoint": "https://test-token-endpoint2", + "userinfo_endpoint": "https://test-userinfo-endpoint2" + }) + self.assertTrue(self.oidc_authnz.config['verify_ssl']) + self.assertEqual(self.oidc_authnz.config['client_id'], 'test-client-id2') + self.assertEqual(self.oidc_authnz.config['client_secret'], 'test-client-secret2') + self.assertEqual(self.oidc_authnz.config['redirect_uri'], 'https://test-redirect-uri2') + self.assertNotIn('well_known_oidc_config_uri', self.oidc_authnz.config) + self.assertEqual(self.oidc_authnz.config['authorization_endpoint'], 'https://test-auth-endpoint2') + self.assertEqual(self.oidc_authnz.config['token_endpoint'], 'https://test-token-endpoint2') + self.assertEqual(self.oidc_authnz.config['userinfo_endpoint'], 'https://test-userinfo-endpoint2') def test_authenticate_set_state_cookie(self): """Verify that authenticate() sets a state cookie.""" @@ -192,6 +226,13 @@ class OIDCAuthnzTestCase(unittest.TestCase): hashed_nonce = self.oidc_authnz._hash_nonce(nonce_in_cookie) self.assertEqual(hashed_nonce, hashed_nonce_in_url) + def test_authenticate_adds_extra_params(self): + """Verify that authenticate() adds configured extra params.""" + authorization_url = self.oidc_authnz.authenticate(self.trans) + parsed = urlparse(authorization_url) + param1_value = parse_qs(parsed.query)['param1'][0] + self.assertEqual(param1_value, 'value1') + def test_callback_verify_with_state_cookie(self): """Verify that state from cookie is passed to OAuth2Session constructor.""" self.trans.set_cookie(value=self.test_state, name=oidc_authnz.STATE_COOKIE_NAME) @@ -233,7 +274,7 @@ class OIDCAuthnzTestCase(unittest.TestCase): self.trans.set_cookie(value=self.test_state, name=oidc_authnz.STATE_COOKIE_NAME) self.trans.set_cookie(value=self.test_nonce, name=oidc_authnz.NONCE_COOKIE_NAME) - self.assertIsNone(self.trans.sa_session.query(KeycloakAccessToken).filter_by(keycloak_user_id=self.test_keycloak_user_id).one_or_none()) + self.assertIsNone(self.trans.sa_session.query(KeycloakAccessToken).filter_by(keycloak_user_id=self.test_user_id).one_or_none()) self.assertEqual(0, len(self.trans.sa_session.items)) login_redirect_url, user = self.oidc_authnz.callback( state_token="xxx", @@ -262,6 +303,34 @@ class OIDCAuthnzTestCase(unittest.TestCase): self.assertTrue(refresh_expiration_timedelta.total_seconds() < 1) self.assertEqual(self._raw_token, added_keycloak_access_token.raw_token) self.assertTrue(self.trans.sa_session.flush_called) + + def test_callback_galaxy_user_created_with_alt_claim_mapping(self): + self.oidc_authnz = oidc_authnz.OIDCAuthnz('Google', { + 'VERIFY_SSL': True + }, { + 'client_id': 'test-client-id', + 'client_secret': 'test-client-secret', + 'redirect_uri': 'https://test-redirect-uri', + 'well_known_oidc_config_uri': 'https://test-well-known-oidc-config-uri', + 'userinfo_claim_mappings': { + 'email': 'alt_email', + 'username': 'alt_username', + 'id': 'alt_id' + } + }) + self.setupMocks() + + self.trans.set_cookie(value=self.test_state, name=oidc_authnz.STATE_COOKIE_NAME) + self.trans.set_cookie(value=self.test_nonce, name=oidc_authnz.NONCE_COOKIE_NAME) + login_redirect_url, user = self.oidc_authnz.callback( + state_token="xxx", + authz_code=self.test_code, trans=self.trans, + login_redirect_url="http://localhost:8000/") + self.assertEqual(self.test_alt_username, user.username) + self.assertEqual(self.test_alt_email, user.email) + self.assertEqual(2, len(self.trans.sa_session.items)) + added_keycloak_access_token = self.trans.sa_session.items[1] + self.assertEqual(self.test_alt_user_id, added_keycloak_access_token.keycloak_user_id) def test_callback_galaxy_user_not_created_when_keycloak_access_token_exists(self): self.trans.set_cookie(value=self.test_state, name=oidc_authnz.STATE_COOKIE_NAME) @@ -274,7 +343,7 @@ class OIDCAuthnzTestCase(unittest.TestCase): old_raw_token = "{}" existing_keycloak_access_token = KeycloakAccessToken( user=User(email=self.test_email, username=self.test_username), - keycloak_user_id=self.test_keycloak_user_id, + keycloak_user_id=self.test_user_id, access_token=old_access_token, id_token=old_id_token, refresh_token=old_refresh_token, @@ -285,7 +354,7 @@ class OIDCAuthnzTestCase(unittest.TestCase): self.trans.sa_session._query.keycloak_access_token = existing_keycloak_access_token - self.assertIsNotNone(self.trans.sa_session.query(KeycloakAccessToken).filter_by(keycloak_user_id=self.test_keycloak_user_id).one_or_none()) + self.assertIsNotNone(self.trans.sa_session.query(KeycloakAccessToken).filter_by(keycloak_user_id=self.test_user_id).one_or_none()) self.assertEqual(0, len(self.trans.sa_session.items)) login_redirect_url, user = self.oidc_authnz.callback( state_token="xxx", From 8970f1559f8bf9649867e5643ebe0849e8e7516b Mon Sep 17 00:00:00 2001 From: Marcus Christie Date: Fri, 8 Feb 2019 20:44:08 -0500 Subject: [PATCH 0543/1016] Removed Keycloak from data model --- lib/galaxy/authnz/managers.py | 2 +- lib/galaxy/authnz/oidc_authnz.py | 47 ++++---- lib/galaxy/model/__init__.py | 7 +- lib/galaxy/model/mapping.py | 17 +-- ...py => 0147_add_oidc_access_token_table.py} | 23 ++-- test/unit/authnz/test_oidc_authnz.py | 105 ++++++++++-------- 6 files changed, 113 insertions(+), 88 deletions(-) rename lib/galaxy/model/migrate/versions/{0147_add_keycloak_auth_tables.py => 0147_add_oidc_access_token_table.py} (55%) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index f31b64fb938..a09484170da 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -110,7 +110,7 @@ class AuthnzManager(object): rtv['prompt'] = config_xml.find('prompt').text return rtv - def _parse_keycloak_config(self, config_xml): + def _parse_generic_oidc_backend_config(self, config_xml): rtv = { 'client_id': config_xml.find('client_id').text, 'client_secret': config_xml.find('client_secret').text, diff --git a/lib/galaxy/authnz/oidc_authnz.py b/lib/galaxy/authnz/oidc_authnz.py index 9de0a9fd906..60f0d43007e 100644 --- a/lib/galaxy/authnz/oidc_authnz.py +++ b/lib/galaxy/authnz/oidc_authnz.py @@ -9,7 +9,7 @@ import requests from oauthlib.common import generate_nonce from requests_oauthlib import OAuth2Session -from galaxy.model import KeycloakAccessToken, User +from galaxy.model import OIDCAccessToken, User from ..authnz import IdentityProvider log = logging.getLogger(__name__) @@ -85,28 +85,29 @@ class OIDCAuthnz(IdentityProvider): email = userinfo[claim_mappings['email']] user_id = userinfo[claim_mappings['id']] - # Create or update keycloak_access_token record - keycloak_access_token = self._get_keycloak_access_token(trans.sa_session, user_id) - if keycloak_access_token is None: + # Create or update oidc_access_token record + oidc_access_token = self._get_oidc_access_token(trans.sa_session, user_id, self.config['provider']) + if oidc_access_token is None: user = self._create_user(trans.sa_session, username, email) - keycloak_access_token = KeycloakAccessToken(user=user, - keycloak_user_id=user_id, - access_token=access_token, - id_token=id_token, - refresh_token=refresh_token, - expiration_time=expiration_time, - refresh_expiration_time=refresh_expiration_time, - raw_token=token) + oidc_access_token = OIDCAccessToken(user=user, + external_user_id=user_id, + provider=self.config['provider'], + access_token=access_token, + id_token=id_token, + refresh_token=refresh_token, + expiration_time=expiration_time, + refresh_expiration_time=refresh_expiration_time, + raw_token=token) else: - keycloak_access_token.access_token = access_token - keycloak_access_token.id_token = id_token - keycloak_access_token.refresh_token = refresh_token - keycloak_access_token.expiration_time = expiration_time - keycloak_access_token.refresh_expiration_time = refresh_expiration_time - keycloak_access_token.raw_token = token - trans.sa_session.add(keycloak_access_token) + oidc_access_token.access_token = access_token + oidc_access_token.id_token = id_token + oidc_access_token.refresh_token = refresh_token + oidc_access_token.expiration_time = expiration_time + oidc_access_token.refresh_expiration_time = refresh_expiration_time + oidc_access_token.raw_token = token + trans.sa_session.add(oidc_access_token) trans.sa_session.flush() - return login_redirect_url, keycloak_access_token.user + return login_redirect_url, oidc_access_token.user def disconnect(self, provider, trans, disconnect_redirect_url=None, association_id=None): # TODO: implement? @@ -142,9 +143,9 @@ class OIDCAuthnz(IdentityProvider): userinfo_claim_mappings.update(self.config['userinfo_claim_mappings']) return userinfo_claim_mappings - def _get_keycloak_access_token(self, sa_session, user_id): - return sa_session.query(KeycloakAccessToken).filter_by( - keycloak_user_id=user_id).one_or_none() + def _get_oidc_access_token(self, sa_session, user_id, provider): + return sa_session.query(OIDCAccessToken).filter_by( + external_user_id=user_id, provider=provider).one_or_none() def _create_user(self, sa_session, username, email): user = User(email=email, username=username) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 72526fb53f6..f4d79684319 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -5099,13 +5099,14 @@ class UserAuthnzToken(UserMixin, RepresentById): return instance -class KeycloakAccessToken(RepresentById): +class OIDCAccessToken(RepresentById): - def __init__(self, user, keycloak_user_id, access_token, id_token, refresh_token, expiration_time, refresh_expiration_time, raw_token): + def __init__(self, user, external_user_id, provider, access_token, id_token, refresh_token, expiration_time, refresh_expiration_time, raw_token): self.id = None # 'user' is a backref to model.User self.user = user - self.keycloak_user_id = keycloak_user_id + self.external_user_id = external_user_id + self.provider = provider self.access_token = access_token self.id_token = id_token self.refresh_token = refresh_token diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 950124f2fdb..e131113c077 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -123,17 +123,20 @@ model.UserAuthnzToken.table = Table( Column('lifetime', Integer), Column('assoc_type', VARCHAR(64))) -model.KeycloakAccessToken.table = Table( - "keycloak_access_token", metadata, +model.OIDCAccessToken.table = Table( + "oidc_access_token", metadata, Column('id', Integer, primary_key=True), - Column('user_id', Integer, ForeignKey("galaxy_user.id"), unique=True, index=True), - Column('keycloak_user_id', String(64), unique=True, index=True), + Column('user_id', Integer, ForeignKey("galaxy_user.id")), + Column('external_user_id', String(64)), + Column('provider', String(255)), Column('access_token', Text), Column('id_token', Text), Column('refresh_token', Text), Column("expiration_time", DateTime), Column("refresh_expiration_time", DateTime), Column('raw_token', JSONType, nullable=True), + UniqueConstraint("user_id", "external_user_id", "provider"), + UniqueConstraint("external_user_id", "provider"), ) model.CloudAuthz.table = Table( @@ -1524,10 +1527,10 @@ mapper(model.UserAuthnzToken, model.UserAuthnzToken.table, properties=dict( backref='social_auth') )) -mapper(model.KeycloakAccessToken, model.KeycloakAccessToken.table, properties=dict( +mapper(model.OIDCAccessToken, model.OIDCAccessToken.table, properties=dict( user=relation(model.User, - primaryjoin=(model.KeycloakAccessToken.table.c.user_id == model.User.table.c.id), - backref='keycloak_auth') + primaryjoin=(model.OIDCAccessToken.table.c.user_id == model.User.table.c.id), + backref='oidc_auth') )) mapper(model.CloudAuthz, model.CloudAuthz.table, properties=dict( diff --git a/lib/galaxy/model/migrate/versions/0147_add_keycloak_auth_tables.py b/lib/galaxy/model/migrate/versions/0147_add_oidc_access_token_table.py similarity index 55% rename from lib/galaxy/model/migrate/versions/0147_add_keycloak_auth_tables.py rename to lib/galaxy/model/migrate/versions/0147_add_oidc_access_token_table.py index f1acbe6eef5..a448033f449 100644 --- a/lib/galaxy/model/migrate/versions/0147_add_keycloak_auth_tables.py +++ b/lib/galaxy/model/migrate/versions/0147_add_oidc_access_token_table.py @@ -1,29 +1,32 @@ """ -Migration for adding keycloak_access_token table. +Migration for adding oidc_access_token table. """ from __future__ import print_function import logging from sqlalchemy import (Column, DateTime, ForeignKey, Integer, MetaData, - String, Table, Text) + String, Table, Text, UniqueConstraint) from galaxy.model.custom_types import JSONType log = logging.getLogger(__name__) metadata = MetaData() -KeycloakAccessToken_table = Table( - "keycloak_access_token", metadata, +OIDCAccessToken_table = Table( + "oidc_access_token", metadata, Column('id', Integer, primary_key=True), - Column('user_id', Integer, ForeignKey("galaxy_user.id"), unique=True, index=True), - Column('keycloak_user_id', String(64), unique=True, index=True), + Column('user_id', Integer, ForeignKey("galaxy_user.id")), + Column('external_user_id', String(64)), + Column('provider', String(255)), Column('access_token', Text), Column('id_token', Text), Column('refresh_token', Text), Column("expiration_time", DateTime), Column("refresh_expiration_time", DateTime), Column('raw_token', JSONType, nullable=True), + UniqueConstraint("user_id", "external_user_id", "provider"), + UniqueConstraint("external_user_id", "provider"), ) @@ -33,9 +36,9 @@ def upgrade(migrate_engine): metadata.reflect() try: - KeycloakAccessToken_table.create() + OIDCAccessToken_table.create() except Exception: - log.exception("Failed to create Keycloak auth tables") + log.exception("Failed to create oidc_access_token table") def downgrade(migrate_engine): @@ -43,6 +46,6 @@ def downgrade(migrate_engine): metadata.reflect() try: - KeycloakAccessToken_table.drop() + OIDCAccessToken_table.drop() except Exception: - log.exception("Failed to drop Keycloak auth tables") + log.exception("Failed to drop oidc_access_token table") diff --git a/test/unit/authnz/test_oidc_authnz.py b/test/unit/authnz/test_oidc_authnz.py index 26cabffeeaf..1c456964272 100644 --- a/test/unit/authnz/test_oidc_authnz.py +++ b/test/unit/authnz/test_oidc_authnz.py @@ -9,7 +9,7 @@ import requests from six.moves.urllib.parse import parse_qs, urlparse from galaxy.authnz import oidc_authnz -from galaxy.model import KeycloakAccessToken, User +from galaxy.model import OIDCAccessToken, User class OIDCAuthnzTestCase(unittest.TestCase): @@ -136,13 +136,15 @@ class OIDCAuthnzTestCase(unittest.TestCase): raise Exception("More than one result!") class Query: - keycloak_user_id = None - keycloak_access_token = None + external_user_id = None + provider = None + oidc_access_token = None - def filter_by(self, keycloak_user_id=None): - self.keycloak_user_id = keycloak_user_id - if self.keycloak_access_token: - return QueryResult([self.keycloak_access_token]) + def filter_by(self, external_user_id=None, provider=None): + self.external_user_id = external_user_id + self.provider = provider + if self.oidc_access_token: + return QueryResult([self.oidc_access_token]) else: return QueryResult() @@ -270,11 +272,16 @@ class OIDCAuthnzTestCase(unittest.TestCase): self.assertTrue(self._fetch_token_called) self.assertFalse(self._get_userinfo_called) - def test_callback_galaxy_user_created_when_no_access_token_exists(self): + def test_callback_galaxy_user_created_when_no_oidc_access_token_exists(self): self.trans.set_cookie(value=self.test_state, name=oidc_authnz.STATE_COOKIE_NAME) self.trans.set_cookie(value=self.test_nonce, name=oidc_authnz.NONCE_COOKIE_NAME) - self.assertIsNone(self.trans.sa_session.query(KeycloakAccessToken).filter_by(keycloak_user_id=self.test_user_id).one_or_none()) + self.assertIsNone( + self.trans.sa_session.query(OIDCAccessToken) + .filter_by(external_user_id=self.test_user_id, + provider=self.oidc_authnz.config['provider']) + .one_or_none() + ) self.assertEqual(0, len(self.trans.sa_session.items)) login_redirect_url, user = self.oidc_authnz.callback( state_token="xxx", @@ -282,26 +289,27 @@ class OIDCAuthnzTestCase(unittest.TestCase): login_redirect_url="http://localhost:8000/") self.assertTrue(self._fetch_token_called) self.assertTrue(self._get_userinfo_called) - self.assertEqual(2, len(self.trans.sa_session.items), "Session has new User and new KeycloakAccessToken") + self.assertEqual(2, len(self.trans.sa_session.items), "Session has new User and new OIDCAccessToken") added_user = self.trans.sa_session.items[0] self.assertIsInstance(added_user, User) self.assertEqual(self.test_username, added_user.username) self.assertEqual(self.test_email, added_user.email) self.assertIsNotNone(added_user.password) - # Verify keycloak_access_token_record - added_keycloak_access_token = self.trans.sa_session.items[1] - self.assertIsInstance(added_keycloak_access_token, KeycloakAccessToken) - self.assertIs(user, added_keycloak_access_token.user) - self.assertEqual(self.test_access_token, added_keycloak_access_token.access_token) - self.assertEqual(self.test_id_token, added_keycloak_access_token.id_token) - self.assertEqual(self.test_refresh_token, added_keycloak_access_token.refresh_token) + # Verify added_oidc_access_token + added_oidc_access_token = self.trans.sa_session.items[1] + self.assertIsInstance(added_oidc_access_token, OIDCAccessToken) + self.assertIs(user, added_oidc_access_token.user) + self.assertEqual(self.test_access_token, added_oidc_access_token.access_token) + self.assertEqual(self.test_id_token, added_oidc_access_token.id_token) + self.assertEqual(self.test_refresh_token, added_oidc_access_token.refresh_token) expected_expiration_time = datetime.now() + timedelta(seconds=self.test_expires_in) - expiration_timedelta = expected_expiration_time - added_keycloak_access_token.expiration_time + expiration_timedelta = expected_expiration_time - added_oidc_access_token.expiration_time self.assertTrue(expiration_timedelta.total_seconds() < 1) expected_refresh_expiration_time = datetime.now() + timedelta(seconds=self.test_refresh_expires_in) - refresh_expiration_timedelta = expected_refresh_expiration_time - added_keycloak_access_token.refresh_expiration_time + refresh_expiration_timedelta = expected_refresh_expiration_time - added_oidc_access_token.refresh_expiration_time self.assertTrue(refresh_expiration_timedelta.total_seconds() < 1) - self.assertEqual(self._raw_token, added_keycloak_access_token.raw_token) + self.assertEqual(self._raw_token, added_oidc_access_token.raw_token) + self.assertEqual(self.oidc_authnz.config['provider'], added_oidc_access_token.provider) self.assertTrue(self.trans.sa_session.flush_called) def test_callback_galaxy_user_created_with_alt_claim_mapping(self): @@ -329,10 +337,10 @@ class OIDCAuthnzTestCase(unittest.TestCase): self.assertEqual(self.test_alt_username, user.username) self.assertEqual(self.test_alt_email, user.email) self.assertEqual(2, len(self.trans.sa_session.items)) - added_keycloak_access_token = self.trans.sa_session.items[1] - self.assertEqual(self.test_alt_user_id, added_keycloak_access_token.keycloak_user_id) + added_oidc_access_token = self.trans.sa_session.items[1] + self.assertEqual(self.test_alt_user_id, added_oidc_access_token.external_user_id) - def test_callback_galaxy_user_not_created_when_keycloak_access_token_exists(self): + def test_callback_galaxy_user_not_created_when_oidc_access_token_exists(self): self.trans.set_cookie(value=self.test_state, name=oidc_authnz.STATE_COOKIE_NAME) self.trans.set_cookie(value=self.test_nonce, name=oidc_authnz.NONCE_COOKIE_NAME) old_access_token = "old-access-token" @@ -341,9 +349,10 @@ class OIDCAuthnzTestCase(unittest.TestCase): old_expiration_time = datetime.now() - timedelta(days=1) old_refresh_expiration_time = datetime.now() - timedelta(hours=3) old_raw_token = "{}" - existing_keycloak_access_token = KeycloakAccessToken( + existing_oidc_access_token = OIDCAccessToken( user=User(email=self.test_email, username=self.test_username), - keycloak_user_id=self.test_user_id, + external_user_id=self.test_user_id, + provider=self.oidc_authnz.config['provider'], access_token=old_access_token, id_token=old_id_token, refresh_token=old_refresh_token, @@ -352,9 +361,14 @@ class OIDCAuthnzTestCase(unittest.TestCase): raw_token=old_raw_token, ) - self.trans.sa_session._query.keycloak_access_token = existing_keycloak_access_token + self.trans.sa_session._query.oidc_access_token = existing_oidc_access_token - self.assertIsNotNone(self.trans.sa_session.query(KeycloakAccessToken).filter_by(keycloak_user_id=self.test_user_id).one_or_none()) + self.assertIsNotNone( + self.trans.sa_session.query(OIDCAccessToken) + .filter_by(external_user_id=self.test_user_id, + provider=self.oidc_authnz.config['provider']) + .one_or_none() + ), self.assertEqual(0, len(self.trans.sa_session.items)) login_redirect_url, user = self.oidc_authnz.callback( state_token="xxx", @@ -362,25 +376,28 @@ class OIDCAuthnzTestCase(unittest.TestCase): login_redirect_url="http://localhost:8000/") self.assertTrue(self._fetch_token_called) self.assertTrue(self._get_userinfo_called) - self.assertEqual(1, len(self.trans.sa_session.items), "Session has updated KeycloakAccessToken") - session_keycloak_access_token = self.trans.sa_session.items[0] - self.assertIsInstance(session_keycloak_access_token, KeycloakAccessToken) - self.assertIs(existing_keycloak_access_token, session_keycloak_access_token, "existing KeycloakAccessToken should be updated") - # Verify both that existing keycloak_access_token has the correct values and different values than before - self.assertEqual(self.test_access_token, session_keycloak_access_token.access_token) - self.assertNotEqual(old_access_token, session_keycloak_access_token.access_token) - self.assertEqual(self.test_id_token, session_keycloak_access_token.id_token) - self.assertNotEqual(old_id_token, session_keycloak_access_token.id_token) - self.assertEqual(self.test_refresh_token, session_keycloak_access_token.refresh_token) - self.assertNotEqual(old_refresh_token, session_keycloak_access_token.refresh_token) + # Make sure query was called with correct parameters + self.assertEqual(self.test_user_id, self.trans.sa_session._query.external_user_id) + self.assertEqual(self.oidc_authnz.config['provider'], self.trans.sa_session._query.provider) + self.assertEqual(1, len(self.trans.sa_session.items), "Session has updated OIDCAccessToken") + session_oidc_access_token = self.trans.sa_session.items[0] + self.assertIsInstance(session_oidc_access_token, OIDCAccessToken) + self.assertIs(existing_oidc_access_token, session_oidc_access_token, "existing OIDCAccessToken should be updated") + # Verify both that existing oidc_access_token has the correct values and different values than before + self.assertEqual(self.test_access_token, session_oidc_access_token.access_token) + self.assertNotEqual(old_access_token, session_oidc_access_token.access_token) + self.assertEqual(self.test_id_token, session_oidc_access_token.id_token) + self.assertNotEqual(old_id_token, session_oidc_access_token.id_token) + self.assertEqual(self.test_refresh_token, session_oidc_access_token.refresh_token) + self.assertNotEqual(old_refresh_token, session_oidc_access_token.refresh_token) expected_expiration_time = datetime.now() + timedelta(seconds=self.test_expires_in) - expiration_timedelta = expected_expiration_time - session_keycloak_access_token.expiration_time + expiration_timedelta = expected_expiration_time - session_oidc_access_token.expiration_time self.assertTrue(expiration_timedelta.total_seconds() < 1) - self.assertNotEqual(old_expiration_time, session_keycloak_access_token.expiration_time) + self.assertNotEqual(old_expiration_time, session_oidc_access_token.expiration_time) expected_refresh_expiration_time = datetime.now() + timedelta(seconds=self.test_refresh_expires_in) - refresh_expiration_timedelta = expected_refresh_expiration_time - session_keycloak_access_token.refresh_expiration_time + refresh_expiration_timedelta = expected_refresh_expiration_time - session_oidc_access_token.refresh_expiration_time self.assertTrue(refresh_expiration_timedelta.total_seconds() < 1) - self.assertNotEqual(old_refresh_expiration_time, session_keycloak_access_token.refresh_expiration_time) - self.assertEqual(self._raw_token, session_keycloak_access_token.raw_token) - self.assertNotEqual(old_raw_token, session_keycloak_access_token.raw_token) + self.assertNotEqual(old_refresh_expiration_time, session_oidc_access_token.refresh_expiration_time) + self.assertEqual(self._raw_token, session_oidc_access_token.raw_token) + self.assertNotEqual(old_raw_token, session_oidc_access_token.raw_token) self.assertTrue(self.trans.sa_session.flush_called) From a3d78c6ebf6507731eac4a40507dc4f140b40683 Mon Sep 17 00:00:00 2001 From: Marcus Christie Date: Fri, 8 Feb 2019 21:47:33 -0500 Subject: [PATCH 0544/1016] Fixing py3 compatibility of oidc_authnz and tests --- lib/galaxy/authnz/oidc_authnz.py | 5 +++-- test/unit/authnz/test_oidc_authnz.py | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/authnz/oidc_authnz.py b/lib/galaxy/authnz/oidc_authnz.py index 60f0d43007e..f35a7e9251f 100644 --- a/lib/galaxy/authnz/oidc_authnz.py +++ b/lib/galaxy/authnz/oidc_authnz.py @@ -9,6 +9,7 @@ import requests from oauthlib.common import generate_nonce from requests_oauthlib import OAuth2Session +from galaxy import util from galaxy.model import OIDCAccessToken, User from ..authnz import IdentityProvider @@ -132,7 +133,7 @@ class OIDCAuthnz(IdentityProvider): def _get_userinfo(self, oauth2_session): userinfo_endpoint = self.config['userinfo_endpoint'] return oauth2_session.get(userinfo_endpoint).json() - + def _get_claim_mappings(self): userinfo_claim_mappings = { 'username': DEFAULT_CLAIM_USERNAME, @@ -155,7 +156,7 @@ class OIDCAuthnz(IdentityProvider): return user def _hash_nonce(self, nonce): - return hashlib.sha256(nonce).hexdigest() + return hashlib.sha256(util.smart_str(nonce)).hexdigest() def _validate_nonce(self, trans, nonce_hash): nonce_cookie = trans.get_cookie(name=NONCE_COOKIE_NAME) diff --git a/test/unit/authnz/test_oidc_authnz.py b/test/unit/authnz/test_oidc_authnz.py index 1c456964272..d68cf0fbd82 100644 --- a/test/unit/authnz/test_oidc_authnz.py +++ b/test/unit/authnz/test_oidc_authnz.py @@ -39,7 +39,7 @@ class OIDCAuthnzTestCase(unittest.TestCase): }) self.setupMocks() self.test_state = "abc123" - self.test_nonce = "4662892146306485421546981092" + self.test_nonce = b"4662892146306485421546981092" self.test_nonce_hash = hashlib.sha256(self.test_nonce).hexdigest() self.test_code = "test-code" self.test_username = "test-username" @@ -53,7 +53,7 @@ class OIDCAuthnzTestCase(unittest.TestCase): self.test_user_id = str(uuid.uuid4()) self.test_alt_user_id = str(uuid.uuid4()) self.trans.request.url = "https://localhost:8000/authnz/google/oidc/callback?state={test_state}&code={test_code}".format(test_state=self.test_state, test_code=self.test_code) - + def setupMocks(self): self.mock_fetch_token(self.oidc_authnz) self.mock_get_userinfo(self.oidc_authnz) @@ -61,7 +61,7 @@ class OIDCAuthnzTestCase(unittest.TestCase): @property def test_id_token(self): - return jwt.encode({'nonce': self.test_nonce_hash}, key=None, algorithm=None) + return jwt.encode({'nonce': self.test_nonce_hash}, key=None, algorithm=None).decode() def mock_create_oauth2_session(self, oidc_authnz): orig_create_oauth2_session = oidc_authnz._create_oauth2_session @@ -311,7 +311,7 @@ class OIDCAuthnzTestCase(unittest.TestCase): self.assertEqual(self._raw_token, added_oidc_access_token.raw_token) self.assertEqual(self.oidc_authnz.config['provider'], added_oidc_access_token.provider) self.assertTrue(self.trans.sa_session.flush_called) - + def test_callback_galaxy_user_created_with_alt_claim_mapping(self): self.oidc_authnz = oidc_authnz.OIDCAuthnz('Google', { 'VERIFY_SSL': True From 290537d5c31b6c9e50bf0b80c939a6f7216b7d17 Mon Sep 17 00:00:00 2001 From: Marcus Christie Date: Mon, 11 Feb 2019 12:30:59 -0500 Subject: [PATCH 0545/1016] Associate external id with currently logged in Galaxy user --- lib/galaxy/authnz/oidc_authnz.py | 7 ++++++- test/unit/authnz/test_oidc_authnz.py | 31 ++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/authnz/oidc_authnz.py b/lib/galaxy/authnz/oidc_authnz.py index f35a7e9251f..b12893c5c8a 100644 --- a/lib/galaxy/authnz/oidc_authnz.py +++ b/lib/galaxy/authnz/oidc_authnz.py @@ -89,7 +89,9 @@ class OIDCAuthnz(IdentityProvider): # Create or update oidc_access_token record oidc_access_token = self._get_oidc_access_token(trans.sa_session, user_id, self.config['provider']) if oidc_access_token is None: - user = self._create_user(trans.sa_session, username, email) + user = self._get_current_user(trans) + if not user: + user = self._create_user(trans.sa_session, username, email) oidc_access_token = OIDCAccessToken(user=user, external_user_id=user_id, provider=self.config['provider'], @@ -148,6 +150,9 @@ class OIDCAuthnz(IdentityProvider): return sa_session.query(OIDCAccessToken).filter_by( external_user_id=user_id, provider=provider).one_or_none() + def _get_current_user(self, trans): + return trans.user if trans.user else None + def _create_user(self, sa_session, username, email): user = User(email=email, username=username) user.set_random_password() diff --git a/test/unit/authnz/test_oidc_authnz.py b/test/unit/authnz/test_oidc_authnz.py index d68cf0fbd82..59704ac42ef 100644 --- a/test/unit/authnz/test_oidc_authnz.py +++ b/test/unit/authnz/test_oidc_authnz.py @@ -167,6 +167,7 @@ class OIDCAuthnzTestCase(unittest.TestCase): cookies_args = {} request = Request() sa_session = Session() + user = None def set_cookie(self, value, name=None, **kwargs): self.cookies[name] = value @@ -312,6 +313,36 @@ class OIDCAuthnzTestCase(unittest.TestCase): self.assertEqual(self.oidc_authnz.config['provider'], added_oidc_access_token.provider) self.assertTrue(self.trans.sa_session.flush_called) + def test_callback_galaxy_user_not_created_when_user_logged_in_and_no_oidc_access_token_exists(self): + """ + Galaxy user is already logged in and trying to associate external + identity with their Galaxy user account. No new user should be created. + """ + self.trans.set_cookie(value=self.test_state, name=oidc_authnz.STATE_COOKIE_NAME) + self.trans.set_cookie(value=self.test_nonce, name=oidc_authnz.NONCE_COOKIE_NAME) + self.trans.user = User() + + self.assertIsNone( + self.trans.sa_session.query(OIDCAccessToken) + .filter_by(external_user_id=self.test_user_id, + provider=self.oidc_authnz.config['provider']) + .one_or_none() + ) + self.assertEqual(0, len(self.trans.sa_session.items)) + login_redirect_url, user = self.oidc_authnz.callback( + state_token="xxx", + authz_code=self.test_code, trans=self.trans, + login_redirect_url="http://localhost:8000/") + self.assertTrue(self._fetch_token_called) + self.assertTrue(self._get_userinfo_called) + self.assertEqual(1, len(self.trans.sa_session.items), "Session has new OIDCAccessToken") + # Verify added_oidc_access_token + added_oidc_access_token = self.trans.sa_session.items[0] + self.assertIsInstance(added_oidc_access_token, OIDCAccessToken) + self.assertIs(user, added_oidc_access_token.user) + self.assertIs(user, self.trans.user) + self.assertTrue(self.trans.sa_session.flush_called) + def test_callback_galaxy_user_created_with_alt_claim_mapping(self): self.oidc_authnz = oidc_authnz.OIDCAuthnz('Google', { 'VERIFY_SSL': True From 72d6f502d8e08fbc9a379291c6d51c5275950d31 Mon Sep 17 00:00:00 2001 From: Marcus Christie Date: Mon, 11 Feb 2019 15:19:09 -0500 Subject: [PATCH 0546/1016] Implemented `disconnect()` for OIDCAuthnz --- lib/galaxy/authnz/oidc_authnz.py | 17 +++++-- test/unit/authnz/test_oidc_authnz.py | 73 +++++++++++++++++++++++++++- 2 files changed, 86 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/authnz/oidc_authnz.py b/lib/galaxy/authnz/oidc_authnz.py index b12893c5c8a..1e1bf6f138c 100644 --- a/lib/galaxy/authnz/oidc_authnz.py +++ b/lib/galaxy/authnz/oidc_authnz.py @@ -112,9 +112,20 @@ class OIDCAuthnz(IdentityProvider): trans.sa_session.flush() return login_redirect_url, oidc_access_token.user - def disconnect(self, provider, trans, disconnect_redirect_url=None, association_id=None): - # TODO: implement? - pass + def disconnect(self, provider, trans, disconnect_redirect_url=None): + try: + user = trans.user + # Find OIDCAccessToken record for this provider (should only be one) + provider_tokens = [token for token in user.oidc_auth if token.provider == self.config["provider"]] + if len(provider_tokens) == 0: + raise Exception("User is not associated with provider {}".format(self.config["provider"])) + if len(provider_tokens) > 1: + raise Exception("User is associated more than once with provider {}".format(self.config["provider"])) + trans.sa_session.delete(provider_tokens[0]) + trans.sa_session.flush() + return True, "", disconnect_redirect_url + except Exception as e: + return False, "Failed to disconnect provider {}: {}".format(provider, str(e)), None def _create_oauth2_session(self, state=None): client_id = self.config['client_id'] diff --git a/test/unit/authnz/test_oidc_authnz.py b/test/unit/authnz/test_oidc_authnz.py index 59704ac42ef..e065eb8f4db 100644 --- a/test/unit/authnz/test_oidc_authnz.py +++ b/test/unit/authnz/test_oidc_authnz.py @@ -1,4 +1,3 @@ - import hashlib import unittest import uuid @@ -152,10 +151,14 @@ class OIDCAuthnzTestCase(unittest.TestCase): items = [] flush_called = False _query = Query() + deleted = [] def add(self, item): self.items.append(item) + def delete(self, item): + self.deleted.append(item) + def flush(self): self.flush_called = True @@ -432,3 +435,71 @@ class OIDCAuthnzTestCase(unittest.TestCase): self.assertEqual(self._raw_token, session_oidc_access_token.raw_token) self.assertNotEqual(old_raw_token, session_oidc_access_token.raw_token) self.assertTrue(self.trans.sa_session.flush_called) + + def test_disconnect(self): + oidc_access_token = OIDCAccessToken( + user=User(email=self.test_email, username=self.test_username), + external_user_id=self.test_user_id, + provider=self.oidc_authnz.config['provider'], + access_token=self.test_access_token, + id_token=self.test_id_token, + refresh_token=self.test_refresh_token, + expiration_time=datetime.now() + timedelta(seconds=self.test_refresh_expires_in), + refresh_expiration_time=datetime.now() + timedelta(seconds=self.test_refresh_expires_in), + raw_token="{}", + ) + self.trans.user = oidc_access_token.user + self.trans.user.oidc_auth = [oidc_access_token] + + success, message, redirect_uri = self.oidc_authnz.disconnect("Google", self.trans, "/") + + self.assertEqual(1, len(self.trans.sa_session.deleted)) + deleted_token = self.trans.sa_session.deleted[0] + self.assertIs(oidc_access_token, deleted_token) + self.assertTrue(self.trans.sa_session.flush_called) + self.assertTrue(success) + self.assertEqual("", message) + self.assertEqual("/", redirect_uri) + + def test_disconnect_when_no_associated_provider(self): + self.trans.user = User() + success, message, redirect_uri = self.oidc_authnz.disconnect("Google", self.trans, "/") + self.assertEqual(0, len(self.trans.sa_session.deleted)) + self.assertFalse(self.trans.sa_session.flush_called) + self.assertFalse(success) + self.assertNotEqual("", message) + self.assertIsNone(redirect_uri) + + def test_disconnect_when_more_than_one_associated_token_for_provider(self): + self.trans.user = User(email=self.test_email, username=self.test_username) + oidc_access_token1 = OIDCAccessToken( + user=self.trans.user, + external_user_id=self.test_user_id + "1", + provider=self.oidc_authnz.config['provider'], + access_token=self.test_access_token, + id_token=self.test_id_token, + refresh_token=self.test_refresh_token, + expiration_time=datetime.now() + timedelta(seconds=self.test_refresh_expires_in), + refresh_expiration_time=datetime.now() + timedelta(seconds=self.test_refresh_expires_in), + raw_token="{}", + ) + oidc_access_token2 = OIDCAccessToken( + user=self.trans.user, + external_user_id=self.test_user_id + "2", + provider=self.oidc_authnz.config['provider'], + access_token=self.test_access_token, + id_token=self.test_id_token, + refresh_token=self.test_refresh_token, + expiration_time=datetime.now() + timedelta(seconds=self.test_refresh_expires_in), + refresh_expiration_time=datetime.now() + timedelta(seconds=self.test_refresh_expires_in), + raw_token="{}", + ) + self.trans.user.oidc_auth = [oidc_access_token1, oidc_access_token2] + + success, message, redirect_uri = self.oidc_authnz.disconnect("Google", self.trans, "/") + + self.assertEqual(0, len(self.trans.sa_session.deleted)) + self.assertFalse(self.trans.sa_session.flush_called) + self.assertFalse(success) + self.assertNotEqual("", message) + self.assertIsNone(redirect_uri) From d76d1d1429dbc617ff6c77120c3791a4e0a9059a Mon Sep 17 00:00:00 2001 From: Marcus Christie Date: Tue, 12 Feb 2019 12:44:35 -0500 Subject: [PATCH 0547/1016] Fixes for Google OIDC Only include scopes in authorization request. When fetching a token, Google will return different scopes than what are requested and oauthlib raises a warning exception. Check if a refresh_token exists in response instead of assuming it will be there. --- lib/galaxy/authnz/oidc_authnz.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/authnz/oidc_authnz.py b/lib/galaxy/authnz/oidc_authnz.py index 1e1bf6f138c..1af044c1925 100644 --- a/lib/galaxy/authnz/oidc_authnz.py +++ b/lib/galaxy/authnz/oidc_authnz.py @@ -45,7 +45,7 @@ class OIDCAuthnz(IdentityProvider): def authenticate(self, trans): base_authorize_url = self.config['authorization_endpoint'] - oauth2_session = self._create_oauth2_session() + oauth2_session = self._create_oauth2_session(scope=('openid', 'email', 'profile')) nonce = generate_nonce() nonce_hash = self._hash_nonce(nonce) extra_params = {"nonce": nonce_hash} @@ -67,9 +67,9 @@ class OIDCAuthnz(IdentityProvider): log.debug("token={}".format(json.dumps(token, indent=True))) access_token = token['access_token'] id_token = token['id_token'] - refresh_token = token['refresh_token'] + refresh_token = token['refresh_token'] if 'refresh_token' in token else None expiration_time = datetime.now() + timedelta(seconds=token['expires_in']) - refresh_expiration_time = datetime.now() + timedelta(seconds=token['refresh_expires_in']) + refresh_expiration_time = (datetime.now() + timedelta(seconds=token['refresh_expires_in'])) if 'refresh_expires_in' in token else None # Get nonce from token['id_token'] and validate. 'nonce' in the # id_token is a hash of the nonce stored in the NONCE_COOKIE_NAME @@ -127,11 +127,11 @@ class OIDCAuthnz(IdentityProvider): except Exception as e: return False, "Failed to disconnect provider {}: {}".format(provider, str(e)), None - def _create_oauth2_session(self, state=None): + def _create_oauth2_session(self, state=None, scope=None): client_id = self.config['client_id'] redirect_uri = self.config['redirect_uri'] return OAuth2Session(client_id, - scope=('openid', 'email', 'profile'), + scope=scope, redirect_uri=redirect_uri, state=state) From e7b65a3b1fa863e70f88c6ca7e1f80e90eeb155c Mon Sep 17 00:00:00 2001 From: Marcus Christie Date: Wed, 13 Feb 2019 15:57:57 -0500 Subject: [PATCH 0548/1016] Automatically set env var when redirect_uri is http://localhost --- lib/galaxy/authnz/oidc_authnz.py | 6 ++++++ test/unit/authnz/test_oidc_authnz.py | 27 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/lib/galaxy/authnz/oidc_authnz.py b/lib/galaxy/authnz/oidc_authnz.py index 1af044c1925..c010e6820a3 100644 --- a/lib/galaxy/authnz/oidc_authnz.py +++ b/lib/galaxy/authnz/oidc_authnz.py @@ -2,6 +2,7 @@ import hashlib import json import logging +import os from datetime import datetime, timedelta import jwt @@ -130,6 +131,11 @@ class OIDCAuthnz(IdentityProvider): def _create_oauth2_session(self, state=None, scope=None): client_id = self.config['client_id'] redirect_uri = self.config['redirect_uri'] + if (redirect_uri.startswith('http://localhost') + and os.environ.get("OAUTHLIB_INSECURE_TRANSPORT", None) != "1"): + log.warn("Setting OAUTHLIB_INSECURE_TRANSPORT to '1' to " + "allow plain HTTP (non-SSL) callback") + os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = "1" return OAuth2Session(client_id, scope=scope, redirect_uri=redirect_uri, diff --git a/test/unit/authnz/test_oidc_authnz.py b/test/unit/authnz/test_oidc_authnz.py index e065eb8f4db..40a2d4a04a7 100644 --- a/test/unit/authnz/test_oidc_authnz.py +++ b/test/unit/authnz/test_oidc_authnz.py @@ -1,4 +1,5 @@ import hashlib +import os import unittest import uuid from datetime import datetime, timedelta @@ -239,6 +240,32 @@ class OIDCAuthnzTestCase(unittest.TestCase): param1_value = parse_qs(parsed.query)['param1'][0] self.assertEqual(param1_value, 'value1') + def test_authenticate_sets_env_var_when_localhost_redirect(self): + """Verify that OAUTHLIB_INSECURE_TRANSPORT var is set with localhost redirect.""" + self.oidc_authnz = oidc_authnz.OIDCAuthnz('Google', { + 'VERIFY_SSL': True + }, { + 'client_id': 'test-client-id', + 'client_secret': 'test-client-secret', + 'redirect_uri': 'http://localhost/auth/callback', + 'well_known_oidc_config_uri': 'https://test-well-known-oidc-config-uri', + 'userinfo_claim_mappings': { + 'email': 'alt_email', + 'username': 'alt_username', + 'id': 'alt_id' + } + }) + self.setupMocks() + self.assertIsNone(os.environ.get('OAUTHLIB_INSECURE_TRANSPORT', None)) + self.oidc_authnz.authenticate(self.trans) + self.assertEqual("1", os.environ['OAUTHLIB_INSECURE_TRANSPORT']) + + def test_authenticate_does_not_set_env_var_when_https_redirect(self): + self.assertTrue(self.oidc_authnz.config['redirect_uri'].startswith("https:")) + self.assertIsNone(os.environ.get('OAUTHLIB_INSECURE_TRANSPORT', None)) + self.oidc_authnz.authenticate(self.trans) + self.assertIsNone(os.environ.get('OAUTHLIB_INSECURE_TRANSPORT', None)) + def test_callback_verify_with_state_cookie(self): """Verify that state from cookie is passed to OAuth2Session constructor.""" self.trans.set_cookie(value=self.test_state, name=oidc_authnz.STATE_COOKIE_NAME) From 21b188f47fe05d8a848acb6c8d24a3d44dad943d Mon Sep 17 00:00:00 2001 From: Marcus Christie Date: Tue, 19 Feb 2019 15:54:43 -0500 Subject: [PATCH 0549/1016] Rename OIDCAccessToken -> OIDCToken --- lib/galaxy/authnz/oidc_authnz.py | 48 ++++---- lib/galaxy/model/__init__.py | 2 +- lib/galaxy/model/mapping.py | 8 +- .../0147_add_oidc_access_token_table.py | 14 +-- test/unit/authnz/test_oidc_authnz.py | 110 +++++++++--------- 5 files changed, 91 insertions(+), 91 deletions(-) diff --git a/lib/galaxy/authnz/oidc_authnz.py b/lib/galaxy/authnz/oidc_authnz.py index c010e6820a3..af5e74f3a8e 100644 --- a/lib/galaxy/authnz/oidc_authnz.py +++ b/lib/galaxy/authnz/oidc_authnz.py @@ -11,7 +11,7 @@ from oauthlib.common import generate_nonce from requests_oauthlib import OAuth2Session from galaxy import util -from galaxy.model import OIDCAccessToken, User +from galaxy.model import OIDCToken, User from ..authnz import IdentityProvider log = logging.getLogger(__name__) @@ -87,36 +87,36 @@ class OIDCAuthnz(IdentityProvider): email = userinfo[claim_mappings['email']] user_id = userinfo[claim_mappings['id']] - # Create or update oidc_access_token record - oidc_access_token = self._get_oidc_access_token(trans.sa_session, user_id, self.config['provider']) - if oidc_access_token is None: + # Create or update oidc_token record + oidc_token = self._get_oidc_token(trans.sa_session, user_id, self.config['provider']) + if oidc_token is None: user = self._get_current_user(trans) if not user: user = self._create_user(trans.sa_session, username, email) - oidc_access_token = OIDCAccessToken(user=user, - external_user_id=user_id, - provider=self.config['provider'], - access_token=access_token, - id_token=id_token, - refresh_token=refresh_token, - expiration_time=expiration_time, - refresh_expiration_time=refresh_expiration_time, - raw_token=token) + oidc_token = OIDCToken(user=user, + external_user_id=user_id, + provider=self.config['provider'], + access_token=access_token, + id_token=id_token, + refresh_token=refresh_token, + expiration_time=expiration_time, + refresh_expiration_time=refresh_expiration_time, + raw_token=token) else: - oidc_access_token.access_token = access_token - oidc_access_token.id_token = id_token - oidc_access_token.refresh_token = refresh_token - oidc_access_token.expiration_time = expiration_time - oidc_access_token.refresh_expiration_time = refresh_expiration_time - oidc_access_token.raw_token = token - trans.sa_session.add(oidc_access_token) + oidc_token.access_token = access_token + oidc_token.id_token = id_token + oidc_token.refresh_token = refresh_token + oidc_token.expiration_time = expiration_time + oidc_token.refresh_expiration_time = refresh_expiration_time + oidc_token.raw_token = token + trans.sa_session.add(oidc_token) trans.sa_session.flush() - return login_redirect_url, oidc_access_token.user + return login_redirect_url, oidc_token.user def disconnect(self, provider, trans, disconnect_redirect_url=None): try: user = trans.user - # Find OIDCAccessToken record for this provider (should only be one) + # Find OIDCToken record for this provider (should only be one) provider_tokens = [token for token in user.oidc_auth if token.provider == self.config["provider"]] if len(provider_tokens) == 0: raise Exception("User is not associated with provider {}".format(self.config["provider"])) @@ -163,8 +163,8 @@ class OIDCAuthnz(IdentityProvider): userinfo_claim_mappings.update(self.config['userinfo_claim_mappings']) return userinfo_claim_mappings - def _get_oidc_access_token(self, sa_session, user_id, provider): - return sa_session.query(OIDCAccessToken).filter_by( + def _get_oidc_token(self, sa_session, user_id, provider): + return sa_session.query(OIDCToken).filter_by( external_user_id=user_id, provider=provider).one_or_none() def _get_current_user(self, trans): diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index f4d79684319..b1acc930f5e 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -5099,7 +5099,7 @@ class UserAuthnzToken(UserMixin, RepresentById): return instance -class OIDCAccessToken(RepresentById): +class OIDCToken(RepresentById): def __init__(self, user, external_user_id, provider, access_token, id_token, refresh_token, expiration_time, refresh_expiration_time, raw_token): self.id = None diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index e131113c077..18ee71f1ab2 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -123,8 +123,8 @@ model.UserAuthnzToken.table = Table( Column('lifetime', Integer), Column('assoc_type', VARCHAR(64))) -model.OIDCAccessToken.table = Table( - "oidc_access_token", metadata, +model.OIDCToken.table = Table( + "oidc_token", metadata, Column('id', Integer, primary_key=True), Column('user_id', Integer, ForeignKey("galaxy_user.id")), Column('external_user_id', String(64)), @@ -1527,9 +1527,9 @@ mapper(model.UserAuthnzToken, model.UserAuthnzToken.table, properties=dict( backref='social_auth') )) -mapper(model.OIDCAccessToken, model.OIDCAccessToken.table, properties=dict( +mapper(model.OIDCToken, model.OIDCToken.table, properties=dict( user=relation(model.User, - primaryjoin=(model.OIDCAccessToken.table.c.user_id == model.User.table.c.id), + primaryjoin=(model.OIDCToken.table.c.user_id == model.User.table.c.id), backref='oidc_auth') )) diff --git a/lib/galaxy/model/migrate/versions/0147_add_oidc_access_token_table.py b/lib/galaxy/model/migrate/versions/0147_add_oidc_access_token_table.py index a448033f449..7c795115aa8 100644 --- a/lib/galaxy/model/migrate/versions/0147_add_oidc_access_token_table.py +++ b/lib/galaxy/model/migrate/versions/0147_add_oidc_access_token_table.py @@ -1,5 +1,5 @@ """ -Migration for adding oidc_access_token table. +Migration for adding oidc_token table. """ from __future__ import print_function @@ -13,8 +13,8 @@ from galaxy.model.custom_types import JSONType log = logging.getLogger(__name__) metadata = MetaData() -OIDCAccessToken_table = Table( - "oidc_access_token", metadata, +OIDCToken_table = Table( + "oidc_token", metadata, Column('id', Integer, primary_key=True), Column('user_id', Integer, ForeignKey("galaxy_user.id")), Column('external_user_id', String(64)), @@ -36,9 +36,9 @@ def upgrade(migrate_engine): metadata.reflect() try: - OIDCAccessToken_table.create() + OIDCToken_table.create() except Exception: - log.exception("Failed to create oidc_access_token table") + log.exception("Failed to create oidc_token table") def downgrade(migrate_engine): @@ -46,6 +46,6 @@ def downgrade(migrate_engine): metadata.reflect() try: - OIDCAccessToken_table.drop() + OIDCToken_table.drop() except Exception: - log.exception("Failed to drop oidc_access_token table") + log.exception("Failed to drop oidc_token table") diff --git a/test/unit/authnz/test_oidc_authnz.py b/test/unit/authnz/test_oidc_authnz.py index 40a2d4a04a7..03d20800c12 100644 --- a/test/unit/authnz/test_oidc_authnz.py +++ b/test/unit/authnz/test_oidc_authnz.py @@ -9,7 +9,7 @@ import requests from six.moves.urllib.parse import parse_qs, urlparse from galaxy.authnz import oidc_authnz -from galaxy.model import OIDCAccessToken, User +from galaxy.model import OIDCToken, User class OIDCAuthnzTestCase(unittest.TestCase): @@ -138,13 +138,13 @@ class OIDCAuthnzTestCase(unittest.TestCase): class Query: external_user_id = None provider = None - oidc_access_token = None + oidc_token = None def filter_by(self, external_user_id=None, provider=None): self.external_user_id = external_user_id self.provider = provider - if self.oidc_access_token: - return QueryResult([self.oidc_access_token]) + if self.oidc_token: + return QueryResult([self.oidc_token]) else: return QueryResult() @@ -303,12 +303,12 @@ class OIDCAuthnzTestCase(unittest.TestCase): self.assertTrue(self._fetch_token_called) self.assertFalse(self._get_userinfo_called) - def test_callback_galaxy_user_created_when_no_oidc_access_token_exists(self): + def test_callback_galaxy_user_created_when_no_oidc_token_exists(self): self.trans.set_cookie(value=self.test_state, name=oidc_authnz.STATE_COOKIE_NAME) self.trans.set_cookie(value=self.test_nonce, name=oidc_authnz.NONCE_COOKIE_NAME) self.assertIsNone( - self.trans.sa_session.query(OIDCAccessToken) + self.trans.sa_session.query(OIDCToken) .filter_by(external_user_id=self.test_user_id, provider=self.oidc_authnz.config['provider']) .one_or_none() @@ -320,30 +320,30 @@ class OIDCAuthnzTestCase(unittest.TestCase): login_redirect_url="http://localhost:8000/") self.assertTrue(self._fetch_token_called) self.assertTrue(self._get_userinfo_called) - self.assertEqual(2, len(self.trans.sa_session.items), "Session has new User and new OIDCAccessToken") + self.assertEqual(2, len(self.trans.sa_session.items), "Session has new User and new OIDCToken") added_user = self.trans.sa_session.items[0] self.assertIsInstance(added_user, User) self.assertEqual(self.test_username, added_user.username) self.assertEqual(self.test_email, added_user.email) self.assertIsNotNone(added_user.password) - # Verify added_oidc_access_token - added_oidc_access_token = self.trans.sa_session.items[1] - self.assertIsInstance(added_oidc_access_token, OIDCAccessToken) - self.assertIs(user, added_oidc_access_token.user) - self.assertEqual(self.test_access_token, added_oidc_access_token.access_token) - self.assertEqual(self.test_id_token, added_oidc_access_token.id_token) - self.assertEqual(self.test_refresh_token, added_oidc_access_token.refresh_token) + # Verify added_oidc_token + added_oidc_token = self.trans.sa_session.items[1] + self.assertIsInstance(added_oidc_token, OIDCToken) + self.assertIs(user, added_oidc_token.user) + self.assertEqual(self.test_access_token, added_oidc_token.access_token) + self.assertEqual(self.test_id_token, added_oidc_token.id_token) + self.assertEqual(self.test_refresh_token, added_oidc_token.refresh_token) expected_expiration_time = datetime.now() + timedelta(seconds=self.test_expires_in) - expiration_timedelta = expected_expiration_time - added_oidc_access_token.expiration_time + expiration_timedelta = expected_expiration_time - added_oidc_token.expiration_time self.assertTrue(expiration_timedelta.total_seconds() < 1) expected_refresh_expiration_time = datetime.now() + timedelta(seconds=self.test_refresh_expires_in) - refresh_expiration_timedelta = expected_refresh_expiration_time - added_oidc_access_token.refresh_expiration_time + refresh_expiration_timedelta = expected_refresh_expiration_time - added_oidc_token.refresh_expiration_time self.assertTrue(refresh_expiration_timedelta.total_seconds() < 1) - self.assertEqual(self._raw_token, added_oidc_access_token.raw_token) - self.assertEqual(self.oidc_authnz.config['provider'], added_oidc_access_token.provider) + self.assertEqual(self._raw_token, added_oidc_token.raw_token) + self.assertEqual(self.oidc_authnz.config['provider'], added_oidc_token.provider) self.assertTrue(self.trans.sa_session.flush_called) - def test_callback_galaxy_user_not_created_when_user_logged_in_and_no_oidc_access_token_exists(self): + def test_callback_galaxy_user_not_created_when_user_logged_in_and_no_oidc_token_exists(self): """ Galaxy user is already logged in and trying to associate external identity with their Galaxy user account. No new user should be created. @@ -353,7 +353,7 @@ class OIDCAuthnzTestCase(unittest.TestCase): self.trans.user = User() self.assertIsNone( - self.trans.sa_session.query(OIDCAccessToken) + self.trans.sa_session.query(OIDCToken) .filter_by(external_user_id=self.test_user_id, provider=self.oidc_authnz.config['provider']) .one_or_none() @@ -365,11 +365,11 @@ class OIDCAuthnzTestCase(unittest.TestCase): login_redirect_url="http://localhost:8000/") self.assertTrue(self._fetch_token_called) self.assertTrue(self._get_userinfo_called) - self.assertEqual(1, len(self.trans.sa_session.items), "Session has new OIDCAccessToken") - # Verify added_oidc_access_token - added_oidc_access_token = self.trans.sa_session.items[0] - self.assertIsInstance(added_oidc_access_token, OIDCAccessToken) - self.assertIs(user, added_oidc_access_token.user) + self.assertEqual(1, len(self.trans.sa_session.items), "Session has new OIDCToken") + # Verify added_oidc_token + added_oidc_token = self.trans.sa_session.items[0] + self.assertIsInstance(added_oidc_token, OIDCToken) + self.assertIs(user, added_oidc_token.user) self.assertIs(user, self.trans.user) self.assertTrue(self.trans.sa_session.flush_called) @@ -398,10 +398,10 @@ class OIDCAuthnzTestCase(unittest.TestCase): self.assertEqual(self.test_alt_username, user.username) self.assertEqual(self.test_alt_email, user.email) self.assertEqual(2, len(self.trans.sa_session.items)) - added_oidc_access_token = self.trans.sa_session.items[1] - self.assertEqual(self.test_alt_user_id, added_oidc_access_token.external_user_id) + added_oidc_token = self.trans.sa_session.items[1] + self.assertEqual(self.test_alt_user_id, added_oidc_token.external_user_id) - def test_callback_galaxy_user_not_created_when_oidc_access_token_exists(self): + def test_callback_galaxy_user_not_created_when_oidc_token_exists(self): self.trans.set_cookie(value=self.test_state, name=oidc_authnz.STATE_COOKIE_NAME) self.trans.set_cookie(value=self.test_nonce, name=oidc_authnz.NONCE_COOKIE_NAME) old_access_token = "old-access-token" @@ -410,7 +410,7 @@ class OIDCAuthnzTestCase(unittest.TestCase): old_expiration_time = datetime.now() - timedelta(days=1) old_refresh_expiration_time = datetime.now() - timedelta(hours=3) old_raw_token = "{}" - existing_oidc_access_token = OIDCAccessToken( + existing_oidc_token = OIDCToken( user=User(email=self.test_email, username=self.test_username), external_user_id=self.test_user_id, provider=self.oidc_authnz.config['provider'], @@ -422,10 +422,10 @@ class OIDCAuthnzTestCase(unittest.TestCase): raw_token=old_raw_token, ) - self.trans.sa_session._query.oidc_access_token = existing_oidc_access_token + self.trans.sa_session._query.oidc_token = existing_oidc_token self.assertIsNotNone( - self.trans.sa_session.query(OIDCAccessToken) + self.trans.sa_session.query(OIDCToken) .filter_by(external_user_id=self.test_user_id, provider=self.oidc_authnz.config['provider']) .one_or_none() @@ -440,31 +440,31 @@ class OIDCAuthnzTestCase(unittest.TestCase): # Make sure query was called with correct parameters self.assertEqual(self.test_user_id, self.trans.sa_session._query.external_user_id) self.assertEqual(self.oidc_authnz.config['provider'], self.trans.sa_session._query.provider) - self.assertEqual(1, len(self.trans.sa_session.items), "Session has updated OIDCAccessToken") - session_oidc_access_token = self.trans.sa_session.items[0] - self.assertIsInstance(session_oidc_access_token, OIDCAccessToken) - self.assertIs(existing_oidc_access_token, session_oidc_access_token, "existing OIDCAccessToken should be updated") - # Verify both that existing oidc_access_token has the correct values and different values than before - self.assertEqual(self.test_access_token, session_oidc_access_token.access_token) - self.assertNotEqual(old_access_token, session_oidc_access_token.access_token) - self.assertEqual(self.test_id_token, session_oidc_access_token.id_token) - self.assertNotEqual(old_id_token, session_oidc_access_token.id_token) - self.assertEqual(self.test_refresh_token, session_oidc_access_token.refresh_token) - self.assertNotEqual(old_refresh_token, session_oidc_access_token.refresh_token) + self.assertEqual(1, len(self.trans.sa_session.items), "Session has updated OIDCToken") + session_oidc_token = self.trans.sa_session.items[0] + self.assertIsInstance(session_oidc_token, OIDCToken) + self.assertIs(existing_oidc_token, session_oidc_token, "existing OIDCToken should be updated") + # Verify both that existing OIDCToken has the correct values and different values than before + self.assertEqual(self.test_access_token, session_oidc_token.access_token) + self.assertNotEqual(old_access_token, session_oidc_token.access_token) + self.assertEqual(self.test_id_token, session_oidc_token.id_token) + self.assertNotEqual(old_id_token, session_oidc_token.id_token) + self.assertEqual(self.test_refresh_token, session_oidc_token.refresh_token) + self.assertNotEqual(old_refresh_token, session_oidc_token.refresh_token) expected_expiration_time = datetime.now() + timedelta(seconds=self.test_expires_in) - expiration_timedelta = expected_expiration_time - session_oidc_access_token.expiration_time + expiration_timedelta = expected_expiration_time - session_oidc_token.expiration_time self.assertTrue(expiration_timedelta.total_seconds() < 1) - self.assertNotEqual(old_expiration_time, session_oidc_access_token.expiration_time) + self.assertNotEqual(old_expiration_time, session_oidc_token.expiration_time) expected_refresh_expiration_time = datetime.now() + timedelta(seconds=self.test_refresh_expires_in) - refresh_expiration_timedelta = expected_refresh_expiration_time - session_oidc_access_token.refresh_expiration_time + refresh_expiration_timedelta = expected_refresh_expiration_time - session_oidc_token.refresh_expiration_time self.assertTrue(refresh_expiration_timedelta.total_seconds() < 1) - self.assertNotEqual(old_refresh_expiration_time, session_oidc_access_token.refresh_expiration_time) - self.assertEqual(self._raw_token, session_oidc_access_token.raw_token) - self.assertNotEqual(old_raw_token, session_oidc_access_token.raw_token) + self.assertNotEqual(old_refresh_expiration_time, session_oidc_token.refresh_expiration_time) + self.assertEqual(self._raw_token, session_oidc_token.raw_token) + self.assertNotEqual(old_raw_token, session_oidc_token.raw_token) self.assertTrue(self.trans.sa_session.flush_called) def test_disconnect(self): - oidc_access_token = OIDCAccessToken( + oidc_token = OIDCToken( user=User(email=self.test_email, username=self.test_username), external_user_id=self.test_user_id, provider=self.oidc_authnz.config['provider'], @@ -475,14 +475,14 @@ class OIDCAuthnzTestCase(unittest.TestCase): refresh_expiration_time=datetime.now() + timedelta(seconds=self.test_refresh_expires_in), raw_token="{}", ) - self.trans.user = oidc_access_token.user - self.trans.user.oidc_auth = [oidc_access_token] + self.trans.user = oidc_token.user + self.trans.user.oidc_auth = [oidc_token] success, message, redirect_uri = self.oidc_authnz.disconnect("Google", self.trans, "/") self.assertEqual(1, len(self.trans.sa_session.deleted)) deleted_token = self.trans.sa_session.deleted[0] - self.assertIs(oidc_access_token, deleted_token) + self.assertIs(oidc_token, deleted_token) self.assertTrue(self.trans.sa_session.flush_called) self.assertTrue(success) self.assertEqual("", message) @@ -499,7 +499,7 @@ class OIDCAuthnzTestCase(unittest.TestCase): def test_disconnect_when_more_than_one_associated_token_for_provider(self): self.trans.user = User(email=self.test_email, username=self.test_username) - oidc_access_token1 = OIDCAccessToken( + oidc_token1 = OIDCToken( user=self.trans.user, external_user_id=self.test_user_id + "1", provider=self.oidc_authnz.config['provider'], @@ -510,7 +510,7 @@ class OIDCAuthnzTestCase(unittest.TestCase): refresh_expiration_time=datetime.now() + timedelta(seconds=self.test_refresh_expires_in), raw_token="{}", ) - oidc_access_token2 = OIDCAccessToken( + oidc_token2 = OIDCToken( user=self.trans.user, external_user_id=self.test_user_id + "2", provider=self.oidc_authnz.config['provider'], @@ -521,7 +521,7 @@ class OIDCAuthnzTestCase(unittest.TestCase): refresh_expiration_time=datetime.now() + timedelta(seconds=self.test_refresh_expires_in), raw_token="{}", ) - self.trans.user.oidc_auth = [oidc_access_token1, oidc_access_token2] + self.trans.user.oidc_auth = [oidc_token1, oidc_token2] success, message, redirect_uri = self.oidc_authnz.disconnect("Google", self.trans, "/") From 28bc901cc42b5d8756a3b89400faa9389302394f Mon Sep 17 00:00:00 2001 From: Marcus Christie Date: Mon, 11 Mar 2019 10:45:47 -0400 Subject: [PATCH 0550/1016] Rename from generic "OIDC" to "Custos" --- .../{oidc_authnz.py => custos_authnz.py} | 40 +-- lib/galaxy/authnz/managers.py | 5 +- lib/galaxy/model/__init__.py | 2 +- lib/galaxy/model/mapping.py | 10 +- ... => 0147_add_custos_authnz_token_table.py} | 14 +- ...t_oidc_authnz.py => test_custos_authnz.py} | 258 +++++++++--------- 6 files changed, 165 insertions(+), 164 deletions(-) rename lib/galaxy/authnz/{oidc_authnz.py => custos_authnz.py} (86%) rename lib/galaxy/model/migrate/versions/{0147_add_oidc_access_token_table.py => 0147_add_custos_authnz_token_table.py} (77%) rename test/unit/authnz/{test_oidc_authnz.py => test_custos_authnz.py} (63%) diff --git a/lib/galaxy/authnz/oidc_authnz.py b/lib/galaxy/authnz/custos_authnz.py similarity index 86% rename from lib/galaxy/authnz/oidc_authnz.py rename to lib/galaxy/authnz/custos_authnz.py index af5e74f3a8e..d2f38e2562a 100644 --- a/lib/galaxy/authnz/oidc_authnz.py +++ b/lib/galaxy/authnz/custos_authnz.py @@ -11,18 +11,18 @@ from oauthlib.common import generate_nonce from requests_oauthlib import OAuth2Session from galaxy import util -from galaxy.model import OIDCToken, User +from galaxy.model import CustosAuthnzToken, User from ..authnz import IdentityProvider log = logging.getLogger(__name__) -STATE_COOKIE_NAME = 'oidc-state' -NONCE_COOKIE_NAME = 'oidc-nonce' +STATE_COOKIE_NAME = 'custos-state' +NONCE_COOKIE_NAME = 'custos-nonce' DEFAULT_CLAIM_USERNAME = 'preferred_username' DEFAULT_CLAIM_EMAIL = 'email' DEFAULT_CLAIM_ID = 'sub' -class OIDCAuthnz(IdentityProvider): +class CustosAuthnz(IdentityProvider): def __init__(self, provider, oidc_config, oidc_backend_config): self.config = {'provider': provider.lower()} self.config['verify_ssl'] = oidc_config['VERIFY_SSL'] @@ -87,13 +87,13 @@ class OIDCAuthnz(IdentityProvider): email = userinfo[claim_mappings['email']] user_id = userinfo[claim_mappings['id']] - # Create or update oidc_token record - oidc_token = self._get_oidc_token(trans.sa_session, user_id, self.config['provider']) - if oidc_token is None: + # Create or update custos_authnz_token record + custos_authnz_token = self._get_custos_authnz_token(trans.sa_session, user_id, self.config['provider']) + if custos_authnz_token is None: user = self._get_current_user(trans) if not user: user = self._create_user(trans.sa_session, username, email) - oidc_token = OIDCToken(user=user, + custos_authnz_token = CustosAuthnzToken(user=user, external_user_id=user_id, provider=self.config['provider'], access_token=access_token, @@ -103,21 +103,21 @@ class OIDCAuthnz(IdentityProvider): refresh_expiration_time=refresh_expiration_time, raw_token=token) else: - oidc_token.access_token = access_token - oidc_token.id_token = id_token - oidc_token.refresh_token = refresh_token - oidc_token.expiration_time = expiration_time - oidc_token.refresh_expiration_time = refresh_expiration_time - oidc_token.raw_token = token - trans.sa_session.add(oidc_token) + custos_authnz_token.access_token = access_token + custos_authnz_token.id_token = id_token + custos_authnz_token.refresh_token = refresh_token + custos_authnz_token.expiration_time = expiration_time + custos_authnz_token.refresh_expiration_time = refresh_expiration_time + custos_authnz_token.raw_token = token + trans.sa_session.add(custos_authnz_token) trans.sa_session.flush() - return login_redirect_url, oidc_token.user + return login_redirect_url, custos_authnz_token.user def disconnect(self, provider, trans, disconnect_redirect_url=None): try: user = trans.user - # Find OIDCToken record for this provider (should only be one) - provider_tokens = [token for token in user.oidc_auth if token.provider == self.config["provider"]] + # Find CustosAuthnzToken record for this provider (should only be one) + provider_tokens = [token for token in user.custos_auth if token.provider == self.config["provider"]] if len(provider_tokens) == 0: raise Exception("User is not associated with provider {}".format(self.config["provider"])) if len(provider_tokens) > 1: @@ -163,8 +163,8 @@ class OIDCAuthnz(IdentityProvider): userinfo_claim_mappings.update(self.config['userinfo_claim_mappings']) return userinfo_claim_mappings - def _get_oidc_token(self, sa_session, user_id, provider): - return sa_session.query(OIDCToken).filter_by( + def _get_custos_authnz_token(self, sa_session, user_id, provider): + return sa_session.query(CustosAuthnzToken).filter_by( external_user_id=user_id, provider=provider).one_or_none() def _get_current_user(self, trans): diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index a09484170da..0714057c0d4 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -17,7 +17,7 @@ from cloudauthz.exceptions import ( from galaxy import exceptions from galaxy import model -from .oidc_authnz import OIDCAuthnz +from .custos_authnz import CustosAuthnz from .psa_authnz import ( BACKENDS_NAME, on_the_fly_config, @@ -110,6 +110,7 @@ class AuthnzManager(object): rtv['prompt'] = config_xml.find('prompt').text return rtv + # TODO: rename def _parse_generic_oidc_backend_config(self, config_xml): rtv = { 'client_id': config_xml.find('client_id').text, @@ -162,7 +163,7 @@ class AuthnzManager(object): if implementation == 'psa': return PSAAuthnz elif implementation == 'oidc': - return OIDCAuthnz + return CustosAuthnz else: return None diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index b1acc930f5e..0b1ed7818a5 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -5099,7 +5099,7 @@ class UserAuthnzToken(UserMixin, RepresentById): return instance -class OIDCToken(RepresentById): +class CustosAuthnzToken(RepresentById): def __init__(self, user, external_user_id, provider, access_token, id_token, refresh_token, expiration_time, refresh_expiration_time, raw_token): self.id = None diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 18ee71f1ab2..96ac7028cb5 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -123,8 +123,8 @@ model.UserAuthnzToken.table = Table( Column('lifetime', Integer), Column('assoc_type', VARCHAR(64))) -model.OIDCToken.table = Table( - "oidc_token", metadata, +model.CustosAuthnzToken.table = Table( + "custos_authnz_token", metadata, Column('id', Integer, primary_key=True), Column('user_id', Integer, ForeignKey("galaxy_user.id")), Column('external_user_id', String(64)), @@ -1527,10 +1527,10 @@ mapper(model.UserAuthnzToken, model.UserAuthnzToken.table, properties=dict( backref='social_auth') )) -mapper(model.OIDCToken, model.OIDCToken.table, properties=dict( +mapper(model.CustosAuthnzToken, model.CustosAuthnzToken.table, properties=dict( user=relation(model.User, - primaryjoin=(model.OIDCToken.table.c.user_id == model.User.table.c.id), - backref='oidc_auth') + primaryjoin=(model.CustosAuthnzToken.table.c.user_id == model.User.table.c.id), + backref='custos_auth') )) mapper(model.CloudAuthz, model.CloudAuthz.table, properties=dict( diff --git a/lib/galaxy/model/migrate/versions/0147_add_oidc_access_token_table.py b/lib/galaxy/model/migrate/versions/0147_add_custos_authnz_token_table.py similarity index 77% rename from lib/galaxy/model/migrate/versions/0147_add_oidc_access_token_table.py rename to lib/galaxy/model/migrate/versions/0147_add_custos_authnz_token_table.py index 7c795115aa8..3006fa22e2b 100644 --- a/lib/galaxy/model/migrate/versions/0147_add_oidc_access_token_table.py +++ b/lib/galaxy/model/migrate/versions/0147_add_custos_authnz_token_table.py @@ -1,5 +1,5 @@ """ -Migration for adding oidc_token table. +Migration for adding custos_authnz_token table. """ from __future__ import print_function @@ -13,8 +13,8 @@ from galaxy.model.custom_types import JSONType log = logging.getLogger(__name__) metadata = MetaData() -OIDCToken_table = Table( - "oidc_token", metadata, +CustosAuthnzToken_table = Table( + "custos_authnz_token", metadata, Column('id', Integer, primary_key=True), Column('user_id', Integer, ForeignKey("galaxy_user.id")), Column('external_user_id', String(64)), @@ -36,9 +36,9 @@ def upgrade(migrate_engine): metadata.reflect() try: - OIDCToken_table.create() + CustosAuthnzToken_table.create() except Exception: - log.exception("Failed to create oidc_token table") + log.exception("Failed to create custos_authnz_token table") def downgrade(migrate_engine): @@ -46,6 +46,6 @@ def downgrade(migrate_engine): metadata.reflect() try: - OIDCToken_table.drop() + CustosAuthnzToken_table.drop() except Exception: - log.exception("Failed to drop oidc_token table") + log.exception("Failed to drop custos_authnz_token table") diff --git a/test/unit/authnz/test_oidc_authnz.py b/test/unit/authnz/test_custos_authnz.py similarity index 63% rename from test/unit/authnz/test_oidc_authnz.py rename to test/unit/authnz/test_custos_authnz.py index 03d20800c12..0fa98283049 100644 --- a/test/unit/authnz/test_oidc_authnz.py +++ b/test/unit/authnz/test_custos_authnz.py @@ -8,11 +8,11 @@ import jwt import requests from six.moves.urllib.parse import parse_qs, urlparse -from galaxy.authnz import oidc_authnz -from galaxy.model import OIDCToken, User +from galaxy.authnz import custos_authnz +from galaxy.model import CustosAuthnzToken, User -class OIDCAuthnzTestCase(unittest.TestCase): +class CustosAuthnzTestCase(unittest.TestCase): _create_oauth2_session_called = False _fetch_token_called = False @@ -26,7 +26,7 @@ class OIDCAuthnzTestCase(unittest.TestCase): "token_endpoint": "https://test-token-endpoint", "userinfo_endpoint": "https://test-userinfo-endpoint" }) - self.oidc_authnz = oidc_authnz.OIDCAuthnz('Google', { + self.custos_authnz = custos_authnz.CustosAuthnz('Custos', { 'VERIFY_SSL': True }, { 'client_id': 'test-client-id', @@ -52,27 +52,27 @@ class OIDCAuthnzTestCase(unittest.TestCase): self.test_refresh_expires_in = 1800 self.test_user_id = str(uuid.uuid4()) self.test_alt_user_id = str(uuid.uuid4()) - self.trans.request.url = "https://localhost:8000/authnz/google/oidc/callback?state={test_state}&code={test_code}".format(test_state=self.test_state, test_code=self.test_code) + self.trans.request.url = "https://localhost:8000/authnz/custos/oidc/callback?state={test_state}&code={test_code}".format(test_state=self.test_state, test_code=self.test_code) def setupMocks(self): - self.mock_fetch_token(self.oidc_authnz) - self.mock_get_userinfo(self.oidc_authnz) + self.mock_fetch_token(self.custos_authnz) + self.mock_get_userinfo(self.custos_authnz) self.trans = self.mockTrans() @property def test_id_token(self): return jwt.encode({'nonce': self.test_nonce_hash}, key=None, algorithm=None).decode() - def mock_create_oauth2_session(self, oidc_authnz): - orig_create_oauth2_session = oidc_authnz._create_oauth2_session + def mock_create_oauth2_session(self, custos_authnz): + orig_create_oauth2_session = custos_authnz._create_oauth2_session def create_oauth2_session(state=None): self._create_oauth2_session_called = True assert state == self.test_state return orig_create_oauth2_session(state) - oidc_authnz._create_oauth2_session = create_oauth2_session + custos_authnz._create_oauth2_session = create_oauth2_session - def mock_fetch_token(self, oidc_authnz): + def mock_fetch_token(self, custos_authnz): def fetch_token(oauth2_session, trans): self._fetch_token_called = True self._raw_token = { @@ -83,9 +83,9 @@ class OIDCAuthnzTestCase(unittest.TestCase): "refresh_expires_in": self.test_refresh_expires_in } return self._raw_token - oidc_authnz._fetch_token = fetch_token + custos_authnz._fetch_token = fetch_token - def mock_get_userinfo(self, oidc_authnz): + def mock_get_userinfo(self, custos_authnz): def get_userinfo(oauth2_session): self._get_userinfo_called = True return { @@ -96,7 +96,7 @@ class OIDCAuthnzTestCase(unittest.TestCase): "alt_email": self.test_alt_email, "alt_id": self.test_alt_user_id } - oidc_authnz._get_userinfo = get_userinfo + custos_authnz._get_userinfo = get_userinfo def mockRequest(self, url, resp): def get(x): @@ -138,13 +138,13 @@ class OIDCAuthnzTestCase(unittest.TestCase): class Query: external_user_id = None provider = None - oidc_token = None + custos_authnz_token = None def filter_by(self, external_user_id=None, provider=None): self.external_user_id = external_user_id self.provider = provider - if self.oidc_token: - return QueryResult([self.oidc_token]) + if self.custos_authnz_token: + return QueryResult([self.custos_authnz_token]) else: return QueryResult() @@ -186,19 +186,19 @@ class OIDCAuthnzTestCase(unittest.TestCase): requests.get = self.orig_requests_get def test_parse_config(self): - self.assertTrue(self.oidc_authnz.config['verify_ssl']) - self.assertEqual(self.oidc_authnz.config['client_id'], 'test-client-id') - self.assertEqual(self.oidc_authnz.config['client_secret'], 'test-client-secret') - self.assertEqual(self.oidc_authnz.config['redirect_uri'], 'https://test-redirect-uri') - self.assertEqual(self.oidc_authnz.config['well_known_oidc_config_uri'], 'https://test-well-known-oidc-config-uri') - self.assertEqual(self.oidc_authnz.config['authorization_endpoint'], 'https://test-auth-endpoint') - self.assertEqual(self.oidc_authnz.config['token_endpoint'], 'https://test-token-endpoint') - self.assertEqual(self.oidc_authnz.config['userinfo_endpoint'], 'https://test-userinfo-endpoint') - self.assertIn('param1', self.oidc_authnz.config['extra_params']) - self.assertEqual(self.oidc_authnz.config['extra_params']['param1'], 'value1') + self.assertTrue(self.custos_authnz.config['verify_ssl']) + self.assertEqual(self.custos_authnz.config['client_id'], 'test-client-id') + self.assertEqual(self.custos_authnz.config['client_secret'], 'test-client-secret') + self.assertEqual(self.custos_authnz.config['redirect_uri'], 'https://test-redirect-uri') + self.assertEqual(self.custos_authnz.config['well_known_oidc_config_uri'], 'https://test-well-known-oidc-config-uri') + self.assertEqual(self.custos_authnz.config['authorization_endpoint'], 'https://test-auth-endpoint') + self.assertEqual(self.custos_authnz.config['token_endpoint'], 'https://test-token-endpoint') + self.assertEqual(self.custos_authnz.config['userinfo_endpoint'], 'https://test-userinfo-endpoint') + self.assertIn('param1', self.custos_authnz.config['extra_params']) + self.assertEqual(self.custos_authnz.config['extra_params']['param1'], 'value1') def test_parse_config_without_well_known_config(self): - self.oidc_authnz = oidc_authnz.OIDCAuthnz('Google', { + self.custos_authnz = custos_authnz.CustosAuthnz('Custos', { 'VERIFY_SSL': True }, { 'client_id': 'test-client-id2', @@ -208,41 +208,41 @@ class OIDCAuthnzTestCase(unittest.TestCase): "token_endpoint": "https://test-token-endpoint2", "userinfo_endpoint": "https://test-userinfo-endpoint2" }) - self.assertTrue(self.oidc_authnz.config['verify_ssl']) - self.assertEqual(self.oidc_authnz.config['client_id'], 'test-client-id2') - self.assertEqual(self.oidc_authnz.config['client_secret'], 'test-client-secret2') - self.assertEqual(self.oidc_authnz.config['redirect_uri'], 'https://test-redirect-uri2') - self.assertNotIn('well_known_oidc_config_uri', self.oidc_authnz.config) - self.assertEqual(self.oidc_authnz.config['authorization_endpoint'], 'https://test-auth-endpoint2') - self.assertEqual(self.oidc_authnz.config['token_endpoint'], 'https://test-token-endpoint2') - self.assertEqual(self.oidc_authnz.config['userinfo_endpoint'], 'https://test-userinfo-endpoint2') + self.assertTrue(self.custos_authnz.config['verify_ssl']) + self.assertEqual(self.custos_authnz.config['client_id'], 'test-client-id2') + self.assertEqual(self.custos_authnz.config['client_secret'], 'test-client-secret2') + self.assertEqual(self.custos_authnz.config['redirect_uri'], 'https://test-redirect-uri2') + self.assertNotIn('well_known_oidc_config_uri', self.custos_authnz.config) + self.assertEqual(self.custos_authnz.config['authorization_endpoint'], 'https://test-auth-endpoint2') + self.assertEqual(self.custos_authnz.config['token_endpoint'], 'https://test-token-endpoint2') + self.assertEqual(self.custos_authnz.config['userinfo_endpoint'], 'https://test-userinfo-endpoint2') def test_authenticate_set_state_cookie(self): """Verify that authenticate() sets a state cookie.""" - authorization_url = self.oidc_authnz.authenticate(self.trans) + authorization_url = self.custos_authnz.authenticate(self.trans) parsed = urlparse(authorization_url) state = parse_qs(parsed.query)['state'][0] - self.assertEqual(state, self.trans.cookies[oidc_authnz.STATE_COOKIE_NAME]) + self.assertEqual(state, self.trans.cookies[custos_authnz.STATE_COOKIE_NAME]) def test_authenticate_set_nonce_cookie(self): """Verify that authenticate() sets a nonce cookie.""" - authorization_url = self.oidc_authnz.authenticate(self.trans) + authorization_url = self.custos_authnz.authenticate(self.trans) parsed = urlparse(authorization_url) hashed_nonce_in_url = parse_qs(parsed.query)['nonce'][0] - nonce_in_cookie = self.trans.cookies[oidc_authnz.NONCE_COOKIE_NAME] - hashed_nonce = self.oidc_authnz._hash_nonce(nonce_in_cookie) + nonce_in_cookie = self.trans.cookies[custos_authnz.NONCE_COOKIE_NAME] + hashed_nonce = self.custos_authnz._hash_nonce(nonce_in_cookie) self.assertEqual(hashed_nonce, hashed_nonce_in_url) def test_authenticate_adds_extra_params(self): """Verify that authenticate() adds configured extra params.""" - authorization_url = self.oidc_authnz.authenticate(self.trans) + authorization_url = self.custos_authnz.authenticate(self.trans) parsed = urlparse(authorization_url) param1_value = parse_qs(parsed.query)['param1'][0] self.assertEqual(param1_value, 'value1') def test_authenticate_sets_env_var_when_localhost_redirect(self): """Verify that OAUTHLIB_INSECURE_TRANSPORT var is set with localhost redirect.""" - self.oidc_authnz = oidc_authnz.OIDCAuthnz('Google', { + self.custos_authnz = custos_authnz.CustosAuthnz('Custos', { 'VERIFY_SSL': True }, { 'client_id': 'test-client-id', @@ -257,27 +257,27 @@ class OIDCAuthnzTestCase(unittest.TestCase): }) self.setupMocks() self.assertIsNone(os.environ.get('OAUTHLIB_INSECURE_TRANSPORT', None)) - self.oidc_authnz.authenticate(self.trans) + self.custos_authnz.authenticate(self.trans) self.assertEqual("1", os.environ['OAUTHLIB_INSECURE_TRANSPORT']) def test_authenticate_does_not_set_env_var_when_https_redirect(self): - self.assertTrue(self.oidc_authnz.config['redirect_uri'].startswith("https:")) + self.assertTrue(self.custos_authnz.config['redirect_uri'].startswith("https:")) self.assertIsNone(os.environ.get('OAUTHLIB_INSECURE_TRANSPORT', None)) - self.oidc_authnz.authenticate(self.trans) + self.custos_authnz.authenticate(self.trans) self.assertIsNone(os.environ.get('OAUTHLIB_INSECURE_TRANSPORT', None)) def test_callback_verify_with_state_cookie(self): """Verify that state from cookie is passed to OAuth2Session constructor.""" - self.trans.set_cookie(value=self.test_state, name=oidc_authnz.STATE_COOKIE_NAME) - self.trans.set_cookie(value=self.test_nonce, name=oidc_authnz.NONCE_COOKIE_NAME) + self.trans.set_cookie(value=self.test_state, name=custos_authnz.STATE_COOKIE_NAME) + self.trans.set_cookie(value=self.test_nonce, name=custos_authnz.NONCE_COOKIE_NAME) self.trans.sa_session._query.user = User(email=self.test_email, username=self.test_username) # Mock _create_oauth2_session to make sure it is created with cookie state token - self.mock_create_oauth2_session(self.oidc_authnz) + self.mock_create_oauth2_session(self.custos_authnz) # Intentionally passing a bad state_token to make sure that code under # test uses the state cookie instead when creating the OAuth2Session - login_redirect_url, user = self.oidc_authnz.callback( + login_redirect_url, user = self.custos_authnz.callback( state_token="xxx", authz_code=self.test_code, trans=self.trans, login_redirect_url="http://localhost:8000/") @@ -288,93 +288,93 @@ class OIDCAuthnzTestCase(unittest.TestCase): self.assertIsNotNone(user) def test_callback_nonce_validation_with_bad_nonce(self): - self.trans.set_cookie(value=self.test_state, name=oidc_authnz.STATE_COOKIE_NAME) - self.trans.set_cookie(value=self.test_nonce, name=oidc_authnz.NONCE_COOKIE_NAME) + self.trans.set_cookie(value=self.test_state, name=custos_authnz.STATE_COOKIE_NAME) + self.trans.set_cookie(value=self.test_nonce, name=custos_authnz.NONCE_COOKIE_NAME) self.trans.sa_session._query.user = User(email=self.test_email, username=self.test_username) # Intentionally create a bad nonce self.test_nonce_hash = self.test_nonce_hash + "Z" - # self.oidc_authnz._fetch_token = fetch_token + # self.custos_authnz._fetch_token = fetch_token with self.assertRaises(Exception): - self.oidc_authnz.callback(state_token="xxx", + self.custos_authnz.callback(state_token="xxx", authz_code=self.test_code, trans=self.trans, login_redirect_url="http://localhost:8000/") self.assertTrue(self._fetch_token_called) self.assertFalse(self._get_userinfo_called) - def test_callback_galaxy_user_created_when_no_oidc_token_exists(self): - self.trans.set_cookie(value=self.test_state, name=oidc_authnz.STATE_COOKIE_NAME) - self.trans.set_cookie(value=self.test_nonce, name=oidc_authnz.NONCE_COOKIE_NAME) + def test_callback_galaxy_user_created_when_no_custos_authnz_token_exists(self): + self.trans.set_cookie(value=self.test_state, name=custos_authnz.STATE_COOKIE_NAME) + self.trans.set_cookie(value=self.test_nonce, name=custos_authnz.NONCE_COOKIE_NAME) self.assertIsNone( - self.trans.sa_session.query(OIDCToken) + self.trans.sa_session.query(CustosAuthnzToken) .filter_by(external_user_id=self.test_user_id, - provider=self.oidc_authnz.config['provider']) + provider=self.custos_authnz.config['provider']) .one_or_none() ) self.assertEqual(0, len(self.trans.sa_session.items)) - login_redirect_url, user = self.oidc_authnz.callback( + login_redirect_url, user = self.custos_authnz.callback( state_token="xxx", authz_code=self.test_code, trans=self.trans, login_redirect_url="http://localhost:8000/") self.assertTrue(self._fetch_token_called) self.assertTrue(self._get_userinfo_called) - self.assertEqual(2, len(self.trans.sa_session.items), "Session has new User and new OIDCToken") + self.assertEqual(2, len(self.trans.sa_session.items), "Session has new User and new CustosAuthnzToken") added_user = self.trans.sa_session.items[0] self.assertIsInstance(added_user, User) self.assertEqual(self.test_username, added_user.username) self.assertEqual(self.test_email, added_user.email) self.assertIsNotNone(added_user.password) - # Verify added_oidc_token - added_oidc_token = self.trans.sa_session.items[1] - self.assertIsInstance(added_oidc_token, OIDCToken) - self.assertIs(user, added_oidc_token.user) - self.assertEqual(self.test_access_token, added_oidc_token.access_token) - self.assertEqual(self.test_id_token, added_oidc_token.id_token) - self.assertEqual(self.test_refresh_token, added_oidc_token.refresh_token) + # Verify added_custos_authnz_token + added_custos_authnz_token = self.trans.sa_session.items[1] + self.assertIsInstance(added_custos_authnz_token, CustosAuthnzToken) + self.assertIs(user, added_custos_authnz_token.user) + self.assertEqual(self.test_access_token, added_custos_authnz_token.access_token) + self.assertEqual(self.test_id_token, added_custos_authnz_token.id_token) + self.assertEqual(self.test_refresh_token, added_custos_authnz_token.refresh_token) expected_expiration_time = datetime.now() + timedelta(seconds=self.test_expires_in) - expiration_timedelta = expected_expiration_time - added_oidc_token.expiration_time + expiration_timedelta = expected_expiration_time - added_custos_authnz_token.expiration_time self.assertTrue(expiration_timedelta.total_seconds() < 1) expected_refresh_expiration_time = datetime.now() + timedelta(seconds=self.test_refresh_expires_in) - refresh_expiration_timedelta = expected_refresh_expiration_time - added_oidc_token.refresh_expiration_time + refresh_expiration_timedelta = expected_refresh_expiration_time - added_custos_authnz_token.refresh_expiration_time self.assertTrue(refresh_expiration_timedelta.total_seconds() < 1) - self.assertEqual(self._raw_token, added_oidc_token.raw_token) - self.assertEqual(self.oidc_authnz.config['provider'], added_oidc_token.provider) + self.assertEqual(self._raw_token, added_custos_authnz_token.raw_token) + self.assertEqual(self.custos_authnz.config['provider'], added_custos_authnz_token.provider) self.assertTrue(self.trans.sa_session.flush_called) - def test_callback_galaxy_user_not_created_when_user_logged_in_and_no_oidc_token_exists(self): + def test_callback_galaxy_user_not_created_when_user_logged_in_and_no_custos_authnz_token_exists(self): """ Galaxy user is already logged in and trying to associate external identity with their Galaxy user account. No new user should be created. """ - self.trans.set_cookie(value=self.test_state, name=oidc_authnz.STATE_COOKIE_NAME) - self.trans.set_cookie(value=self.test_nonce, name=oidc_authnz.NONCE_COOKIE_NAME) + self.trans.set_cookie(value=self.test_state, name=custos_authnz.STATE_COOKIE_NAME) + self.trans.set_cookie(value=self.test_nonce, name=custos_authnz.NONCE_COOKIE_NAME) self.trans.user = User() self.assertIsNone( - self.trans.sa_session.query(OIDCToken) + self.trans.sa_session.query(CustosAuthnzToken) .filter_by(external_user_id=self.test_user_id, - provider=self.oidc_authnz.config['provider']) + provider=self.custos_authnz.config['provider']) .one_or_none() ) self.assertEqual(0, len(self.trans.sa_session.items)) - login_redirect_url, user = self.oidc_authnz.callback( + login_redirect_url, user = self.custos_authnz.callback( state_token="xxx", authz_code=self.test_code, trans=self.trans, login_redirect_url="http://localhost:8000/") self.assertTrue(self._fetch_token_called) self.assertTrue(self._get_userinfo_called) - self.assertEqual(1, len(self.trans.sa_session.items), "Session has new OIDCToken") - # Verify added_oidc_token - added_oidc_token = self.trans.sa_session.items[0] - self.assertIsInstance(added_oidc_token, OIDCToken) - self.assertIs(user, added_oidc_token.user) + self.assertEqual(1, len(self.trans.sa_session.items), "Session has new CustosAuthnzToken") + # Verify added_custos_authnz_token + added_custos_authnz_token = self.trans.sa_session.items[0] + self.assertIsInstance(added_custos_authnz_token, CustosAuthnzToken) + self.assertIs(user, added_custos_authnz_token.user) self.assertIs(user, self.trans.user) self.assertTrue(self.trans.sa_session.flush_called) def test_callback_galaxy_user_created_with_alt_claim_mapping(self): - self.oidc_authnz = oidc_authnz.OIDCAuthnz('Google', { + self.custos_authnz = custos_authnz.CustosAuthnz('Custos', { 'VERIFY_SSL': True }, { 'client_id': 'test-client-id', @@ -389,31 +389,31 @@ class OIDCAuthnzTestCase(unittest.TestCase): }) self.setupMocks() - self.trans.set_cookie(value=self.test_state, name=oidc_authnz.STATE_COOKIE_NAME) - self.trans.set_cookie(value=self.test_nonce, name=oidc_authnz.NONCE_COOKIE_NAME) - login_redirect_url, user = self.oidc_authnz.callback( + self.trans.set_cookie(value=self.test_state, name=custos_authnz.STATE_COOKIE_NAME) + self.trans.set_cookie(value=self.test_nonce, name=custos_authnz.NONCE_COOKIE_NAME) + login_redirect_url, user = self.custos_authnz.callback( state_token="xxx", authz_code=self.test_code, trans=self.trans, login_redirect_url="http://localhost:8000/") self.assertEqual(self.test_alt_username, user.username) self.assertEqual(self.test_alt_email, user.email) self.assertEqual(2, len(self.trans.sa_session.items)) - added_oidc_token = self.trans.sa_session.items[1] - self.assertEqual(self.test_alt_user_id, added_oidc_token.external_user_id) + added_custos_authnz_token = self.trans.sa_session.items[1] + self.assertEqual(self.test_alt_user_id, added_custos_authnz_token.external_user_id) - def test_callback_galaxy_user_not_created_when_oidc_token_exists(self): - self.trans.set_cookie(value=self.test_state, name=oidc_authnz.STATE_COOKIE_NAME) - self.trans.set_cookie(value=self.test_nonce, name=oidc_authnz.NONCE_COOKIE_NAME) + def test_callback_galaxy_user_not_created_when_custos_authnz_token_exists(self): + self.trans.set_cookie(value=self.test_state, name=custos_authnz.STATE_COOKIE_NAME) + self.trans.set_cookie(value=self.test_nonce, name=custos_authnz.NONCE_COOKIE_NAME) old_access_token = "old-access-token" old_id_token = "old-id-token" old_refresh_token = "old-refresh-token" old_expiration_time = datetime.now() - timedelta(days=1) old_refresh_expiration_time = datetime.now() - timedelta(hours=3) old_raw_token = "{}" - existing_oidc_token = OIDCToken( + existing_custos_authnz_token = CustosAuthnzToken( user=User(email=self.test_email, username=self.test_username), external_user_id=self.test_user_id, - provider=self.oidc_authnz.config['provider'], + provider=self.custos_authnz.config['provider'], access_token=old_access_token, id_token=old_id_token, refresh_token=old_refresh_token, @@ -422,16 +422,16 @@ class OIDCAuthnzTestCase(unittest.TestCase): raw_token=old_raw_token, ) - self.trans.sa_session._query.oidc_token = existing_oidc_token + self.trans.sa_session._query.custos_authnz_token = existing_custos_authnz_token self.assertIsNotNone( - self.trans.sa_session.query(OIDCToken) + self.trans.sa_session.query(CustosAuthnzToken) .filter_by(external_user_id=self.test_user_id, - provider=self.oidc_authnz.config['provider']) + provider=self.custos_authnz.config['provider']) .one_or_none() ), self.assertEqual(0, len(self.trans.sa_session.items)) - login_redirect_url, user = self.oidc_authnz.callback( + login_redirect_url, user = self.custos_authnz.callback( state_token="xxx", authz_code=self.test_code, trans=self.trans, login_redirect_url="http://localhost:8000/") @@ -439,35 +439,35 @@ class OIDCAuthnzTestCase(unittest.TestCase): self.assertTrue(self._get_userinfo_called) # Make sure query was called with correct parameters self.assertEqual(self.test_user_id, self.trans.sa_session._query.external_user_id) - self.assertEqual(self.oidc_authnz.config['provider'], self.trans.sa_session._query.provider) - self.assertEqual(1, len(self.trans.sa_session.items), "Session has updated OIDCToken") - session_oidc_token = self.trans.sa_session.items[0] - self.assertIsInstance(session_oidc_token, OIDCToken) - self.assertIs(existing_oidc_token, session_oidc_token, "existing OIDCToken should be updated") - # Verify both that existing OIDCToken has the correct values and different values than before - self.assertEqual(self.test_access_token, session_oidc_token.access_token) - self.assertNotEqual(old_access_token, session_oidc_token.access_token) - self.assertEqual(self.test_id_token, session_oidc_token.id_token) - self.assertNotEqual(old_id_token, session_oidc_token.id_token) - self.assertEqual(self.test_refresh_token, session_oidc_token.refresh_token) - self.assertNotEqual(old_refresh_token, session_oidc_token.refresh_token) + self.assertEqual(self.custos_authnz.config['provider'], self.trans.sa_session._query.provider) + self.assertEqual(1, len(self.trans.sa_session.items), "Session has updated CustosAuthnzToken") + session_custos_authnz_token = self.trans.sa_session.items[0] + self.assertIsInstance(session_custos_authnz_token, CustosAuthnzToken) + self.assertIs(existing_custos_authnz_token, session_custos_authnz_token, "existing CustosAuthnzToken should be updated") + # Verify both that existing CustosAuthnzToken has the correct values and different values than before + self.assertEqual(self.test_access_token, session_custos_authnz_token.access_token) + self.assertNotEqual(old_access_token, session_custos_authnz_token.access_token) + self.assertEqual(self.test_id_token, session_custos_authnz_token.id_token) + self.assertNotEqual(old_id_token, session_custos_authnz_token.id_token) + self.assertEqual(self.test_refresh_token, session_custos_authnz_token.refresh_token) + self.assertNotEqual(old_refresh_token, session_custos_authnz_token.refresh_token) expected_expiration_time = datetime.now() + timedelta(seconds=self.test_expires_in) - expiration_timedelta = expected_expiration_time - session_oidc_token.expiration_time + expiration_timedelta = expected_expiration_time - session_custos_authnz_token.expiration_time self.assertTrue(expiration_timedelta.total_seconds() < 1) - self.assertNotEqual(old_expiration_time, session_oidc_token.expiration_time) + self.assertNotEqual(old_expiration_time, session_custos_authnz_token.expiration_time) expected_refresh_expiration_time = datetime.now() + timedelta(seconds=self.test_refresh_expires_in) - refresh_expiration_timedelta = expected_refresh_expiration_time - session_oidc_token.refresh_expiration_time + refresh_expiration_timedelta = expected_refresh_expiration_time - session_custos_authnz_token.refresh_expiration_time self.assertTrue(refresh_expiration_timedelta.total_seconds() < 1) - self.assertNotEqual(old_refresh_expiration_time, session_oidc_token.refresh_expiration_time) - self.assertEqual(self._raw_token, session_oidc_token.raw_token) - self.assertNotEqual(old_raw_token, session_oidc_token.raw_token) + self.assertNotEqual(old_refresh_expiration_time, session_custos_authnz_token.refresh_expiration_time) + self.assertEqual(self._raw_token, session_custos_authnz_token.raw_token) + self.assertNotEqual(old_raw_token, session_custos_authnz_token.raw_token) self.assertTrue(self.trans.sa_session.flush_called) def test_disconnect(self): - oidc_token = OIDCToken( + custos_authnz_token = CustosAuthnzToken( user=User(email=self.test_email, username=self.test_username), external_user_id=self.test_user_id, - provider=self.oidc_authnz.config['provider'], + provider=self.custos_authnz.config['provider'], access_token=self.test_access_token, id_token=self.test_id_token, refresh_token=self.test_refresh_token, @@ -475,14 +475,14 @@ class OIDCAuthnzTestCase(unittest.TestCase): refresh_expiration_time=datetime.now() + timedelta(seconds=self.test_refresh_expires_in), raw_token="{}", ) - self.trans.user = oidc_token.user - self.trans.user.oidc_auth = [oidc_token] + self.trans.user = custos_authnz_token.user + self.trans.user.custos_auth = [custos_authnz_token] - success, message, redirect_uri = self.oidc_authnz.disconnect("Google", self.trans, "/") + success, message, redirect_uri = self.custos_authnz.disconnect("Custos", self.trans, "/") self.assertEqual(1, len(self.trans.sa_session.deleted)) deleted_token = self.trans.sa_session.deleted[0] - self.assertIs(oidc_token, deleted_token) + self.assertIs(custos_authnz_token, deleted_token) self.assertTrue(self.trans.sa_session.flush_called) self.assertTrue(success) self.assertEqual("", message) @@ -490,7 +490,7 @@ class OIDCAuthnzTestCase(unittest.TestCase): def test_disconnect_when_no_associated_provider(self): self.trans.user = User() - success, message, redirect_uri = self.oidc_authnz.disconnect("Google", self.trans, "/") + success, message, redirect_uri = self.custos_authnz.disconnect("Custos", self.trans, "/") self.assertEqual(0, len(self.trans.sa_session.deleted)) self.assertFalse(self.trans.sa_session.flush_called) self.assertFalse(success) @@ -499,10 +499,10 @@ class OIDCAuthnzTestCase(unittest.TestCase): def test_disconnect_when_more_than_one_associated_token_for_provider(self): self.trans.user = User(email=self.test_email, username=self.test_username) - oidc_token1 = OIDCToken( + custos_authnz_token1 = CustosAuthnzToken( user=self.trans.user, external_user_id=self.test_user_id + "1", - provider=self.oidc_authnz.config['provider'], + provider=self.custos_authnz.config['provider'], access_token=self.test_access_token, id_token=self.test_id_token, refresh_token=self.test_refresh_token, @@ -510,10 +510,10 @@ class OIDCAuthnzTestCase(unittest.TestCase): refresh_expiration_time=datetime.now() + timedelta(seconds=self.test_refresh_expires_in), raw_token="{}", ) - oidc_token2 = OIDCToken( + custos_authnz_token2 = CustosAuthnzToken( user=self.trans.user, external_user_id=self.test_user_id + "2", - provider=self.oidc_authnz.config['provider'], + provider=self.custos_authnz.config['provider'], access_token=self.test_access_token, id_token=self.test_id_token, refresh_token=self.test_refresh_token, @@ -521,9 +521,9 @@ class OIDCAuthnzTestCase(unittest.TestCase): refresh_expiration_time=datetime.now() + timedelta(seconds=self.test_refresh_expires_in), raw_token="{}", ) - self.trans.user.oidc_auth = [oidc_token1, oidc_token2] + self.trans.user.custos_auth = [custos_authnz_token1, custos_authnz_token2] - success, message, redirect_uri = self.oidc_authnz.disconnect("Google", self.trans, "/") + success, message, redirect_uri = self.custos_authnz.disconnect("Custos", self.trans, "/") self.assertEqual(0, len(self.trans.sa_session.deleted)) self.assertFalse(self.trans.sa_session.flush_called) From 12b51ca029deb89fc61b753e8f08c99667951353 Mon Sep 17 00:00:00 2001 From: Marcus Christie Date: Mon, 11 Mar 2019 11:39:33 -0400 Subject: [PATCH 0551/1016] Simplified configuration based on provider name and realm --- lib/galaxy/authnz/custos_authnz.py | 54 +++++++++++--------- test/unit/authnz/test_custos_authnz.py | 70 +++----------------------- 2 files changed, 35 insertions(+), 89 deletions(-) diff --git a/lib/galaxy/authnz/custos_authnz.py b/lib/galaxy/authnz/custos_authnz.py index d2f38e2562a..8fc0b06401a 100644 --- a/lib/galaxy/authnz/custos_authnz.py +++ b/lib/galaxy/authnz/custos_authnz.py @@ -17,9 +17,6 @@ from ..authnz import IdentityProvider log = logging.getLogger(__name__) STATE_COOKIE_NAME = 'custos-state' NONCE_COOKIE_NAME = 'custos-nonce' -DEFAULT_CLAIM_USERNAME = 'preferred_username' -DEFAULT_CLAIM_EMAIL = 'email' -DEFAULT_CLAIM_ID = 'sub' class CustosAuthnz(IdentityProvider): @@ -29,7 +26,7 @@ class CustosAuthnz(IdentityProvider): self.config['client_id'] = oidc_backend_config['client_id'] self.config['client_secret'] = oidc_backend_config['client_secret'] self.config['redirect_uri'] = oidc_backend_config['redirect_uri'] - # Either get OIDC config from well-known config URI or directly + # Either get OIDC config from well-known config URI or lookup known urls based on provider name and realm if 'well_known_oidc_config_uri' in oidc_backend_config: self.config['well_known_oidc_config_uri'] = oidc_backend_config['well_known_oidc_config_uri'] well_known_oidc_config = self._load_well_known_oidc_config( @@ -38,11 +35,8 @@ class CustosAuthnz(IdentityProvider): self.config['token_endpoint'] = well_known_oidc_config['token_endpoint'] self.config['userinfo_endpoint'] = well_known_oidc_config['userinfo_endpoint'] else: - self.config['authorization_endpoint'] = oidc_backend_config['authorization_endpoint'] - self.config['token_endpoint'] = oidc_backend_config['token_endpoint'] - self.config['userinfo_endpoint'] = oidc_backend_config['userinfo_endpoint'] - self.config['extra_params'] = oidc_backend_config.get('extra_params', None) - self.config['userinfo_claim_mappings'] = oidc_backend_config.get('userinfo_claim_mappings', None) + realm = oidc_backend_config['realm'] + self._load_config_for_provider_and_realm(self.config['provider'], realm) def authenticate(self, trans): base_authorize_url = self.config['authorization_endpoint'] @@ -50,7 +44,7 @@ class CustosAuthnz(IdentityProvider): nonce = generate_nonce() nonce_hash = self._hash_nonce(nonce) extra_params = {"nonce": nonce_hash} - if self.config['extra_params']: + if "extra_params" in self.config: extra_params.update(self.config['extra_params']) authorization_url, state = oauth2_session.authorization_url( base_authorize_url, **extra_params) @@ -82,10 +76,9 @@ class CustosAuthnz(IdentityProvider): # Get userinfo and lookup/create Galaxy user record userinfo = self._get_userinfo(oauth2_session) log.debug("userinfo={}".format(json.dumps(userinfo, indent=True))) - claim_mappings = self._get_claim_mappings() - username = userinfo[claim_mappings['username']] - email = userinfo[claim_mappings['email']] - user_id = userinfo[claim_mappings['id']] + username = userinfo['preferred_username'] + email = userinfo['email'] + user_id = userinfo['sub'] # Create or update custos_authnz_token record custos_authnz_token = self._get_custos_authnz_token(trans.sa_session, user_id, self.config['provider']) @@ -153,16 +146,6 @@ class CustosAuthnz(IdentityProvider): userinfo_endpoint = self.config['userinfo_endpoint'] return oauth2_session.get(userinfo_endpoint).json() - def _get_claim_mappings(self): - userinfo_claim_mappings = { - 'username': DEFAULT_CLAIM_USERNAME, - 'email': DEFAULT_CLAIM_EMAIL, - 'id': DEFAULT_CLAIM_ID - } - if self.config['userinfo_claim_mappings']: - userinfo_claim_mappings.update(self.config['userinfo_claim_mappings']) - return userinfo_claim_mappings - def _get_custos_authnz_token(self, sa_session, user_id, provider): return sa_session.query(CustosAuthnzToken).filter_by( external_user_id=user_id, provider=provider).one_or_none() @@ -188,5 +171,26 @@ class CustosAuthnz(IdentityProvider): if nonce_hash != nonce_cookie_hash: raise Exception("Nonce mismatch!") + def _load_config_for_provider_and_realm(self, provider, realm): + self.config['well_known_oidc_config_uri'] = self._get_well_known_uri_for_provider_and_realm(provider, realm) + well_known_oidc_config = self._load_well_known_oidc_config(self.config['well_known_oidc_config_uri']) + self.config['authorization_endpoint'] = well_known_oidc_config['authorization_endpoint'] + self.config['token_endpoint'] = well_known_oidc_config['token_endpoint'] + self.config['userinfo_endpoint'] = well_known_oidc_config['userinfo_endpoint'] + self.config['extra_params'] = {'kc_idp_hint': 'cilogon'} + + def _get_well_known_uri_for_provider_and_realm(self, provider, realm): + # TODO: Look up these URLs from a Python library + if provider == 'custos': + return "https://iam.scigap.org/auth/realms/{}/.well-known/openid-configuration".format(realm) + elif provider == 'custos-dev': + return "https://iamdev.scigap.org/auth/realms/{}/.well-known/openid-configuration".format(realm) + else: + raise Exception("Unknown Custos provider name: {}".format(provider)) + def _load_well_known_oidc_config(self, well_known_uri): - return requests.get(well_known_uri).json() + try: + return requests.get(well_known_uri).json() + except Exception: + log.error("Failed to load well-known OIDC config URI: {}".format(well_known_uri)) + raise diff --git a/test/unit/authnz/test_custos_authnz.py b/test/unit/authnz/test_custos_authnz.py index 0fa98283049..64f4d3786cf 100644 --- a/test/unit/authnz/test_custos_authnz.py +++ b/test/unit/authnz/test_custos_authnz.py @@ -21,7 +21,7 @@ class CustosAuthnzTestCase(unittest.TestCase): def setUp(self): self.orig_requests_get = requests.get - requests.get = self.mockRequest("https://test-well-known-oidc-config-uri", { + requests.get = self.mockRequest("https://iam.scigap.org/auth/realms/test-realm/.well-known/openid-configuration", { "authorization_endpoint": "https://test-auth-endpoint", "token_endpoint": "https://test-token-endpoint", "userinfo_endpoint": "https://test-userinfo-endpoint" @@ -32,10 +32,7 @@ class CustosAuthnzTestCase(unittest.TestCase): 'client_id': 'test-client-id', 'client_secret': 'test-client-secret', 'redirect_uri': 'https://test-redirect-uri', - 'well_known_oidc_config_uri': 'https://test-well-known-oidc-config-uri', - 'extra_params': { - 'param1': 'value1' - } + 'realm': 'test-realm' }) self.setupMocks() self.test_state = "abc123" @@ -190,32 +187,10 @@ class CustosAuthnzTestCase(unittest.TestCase): self.assertEqual(self.custos_authnz.config['client_id'], 'test-client-id') self.assertEqual(self.custos_authnz.config['client_secret'], 'test-client-secret') self.assertEqual(self.custos_authnz.config['redirect_uri'], 'https://test-redirect-uri') - self.assertEqual(self.custos_authnz.config['well_known_oidc_config_uri'], 'https://test-well-known-oidc-config-uri') + self.assertEqual(self.custos_authnz.config['well_known_oidc_config_uri'], 'https://iam.scigap.org/auth/realms/test-realm/.well-known/openid-configuration') self.assertEqual(self.custos_authnz.config['authorization_endpoint'], 'https://test-auth-endpoint') self.assertEqual(self.custos_authnz.config['token_endpoint'], 'https://test-token-endpoint') self.assertEqual(self.custos_authnz.config['userinfo_endpoint'], 'https://test-userinfo-endpoint') - self.assertIn('param1', self.custos_authnz.config['extra_params']) - self.assertEqual(self.custos_authnz.config['extra_params']['param1'], 'value1') - - def test_parse_config_without_well_known_config(self): - self.custos_authnz = custos_authnz.CustosAuthnz('Custos', { - 'VERIFY_SSL': True - }, { - 'client_id': 'test-client-id2', - 'client_secret': 'test-client-secret2', - 'redirect_uri': 'https://test-redirect-uri2', - "authorization_endpoint": "https://test-auth-endpoint2", - "token_endpoint": "https://test-token-endpoint2", - "userinfo_endpoint": "https://test-userinfo-endpoint2" - }) - self.assertTrue(self.custos_authnz.config['verify_ssl']) - self.assertEqual(self.custos_authnz.config['client_id'], 'test-client-id2') - self.assertEqual(self.custos_authnz.config['client_secret'], 'test-client-secret2') - self.assertEqual(self.custos_authnz.config['redirect_uri'], 'https://test-redirect-uri2') - self.assertNotIn('well_known_oidc_config_uri', self.custos_authnz.config) - self.assertEqual(self.custos_authnz.config['authorization_endpoint'], 'https://test-auth-endpoint2') - self.assertEqual(self.custos_authnz.config['token_endpoint'], 'https://test-token-endpoint2') - self.assertEqual(self.custos_authnz.config['userinfo_endpoint'], 'https://test-userinfo-endpoint2') def test_authenticate_set_state_cookie(self): """Verify that authenticate() sets a state cookie.""" @@ -237,8 +212,8 @@ class CustosAuthnzTestCase(unittest.TestCase): """Verify that authenticate() adds configured extra params.""" authorization_url = self.custos_authnz.authenticate(self.trans) parsed = urlparse(authorization_url) - param1_value = parse_qs(parsed.query)['param1'][0] - self.assertEqual(param1_value, 'value1') + param1_value = parse_qs(parsed.query)['kc_idp_hint'][0] + self.assertEqual(param1_value, 'cilogon') def test_authenticate_sets_env_var_when_localhost_redirect(self): """Verify that OAUTHLIB_INSECURE_TRANSPORT var is set with localhost redirect.""" @@ -248,12 +223,7 @@ class CustosAuthnzTestCase(unittest.TestCase): 'client_id': 'test-client-id', 'client_secret': 'test-client-secret', 'redirect_uri': 'http://localhost/auth/callback', - 'well_known_oidc_config_uri': 'https://test-well-known-oidc-config-uri', - 'userinfo_claim_mappings': { - 'email': 'alt_email', - 'username': 'alt_username', - 'id': 'alt_id' - } + 'realm': 'test-realm' }) self.setupMocks() self.assertIsNone(os.environ.get('OAUTHLIB_INSECURE_TRANSPORT', None)) @@ -373,34 +343,6 @@ class CustosAuthnzTestCase(unittest.TestCase): self.assertIs(user, self.trans.user) self.assertTrue(self.trans.sa_session.flush_called) - def test_callback_galaxy_user_created_with_alt_claim_mapping(self): - self.custos_authnz = custos_authnz.CustosAuthnz('Custos', { - 'VERIFY_SSL': True - }, { - 'client_id': 'test-client-id', - 'client_secret': 'test-client-secret', - 'redirect_uri': 'https://test-redirect-uri', - 'well_known_oidc_config_uri': 'https://test-well-known-oidc-config-uri', - 'userinfo_claim_mappings': { - 'email': 'alt_email', - 'username': 'alt_username', - 'id': 'alt_id' - } - }) - self.setupMocks() - - self.trans.set_cookie(value=self.test_state, name=custos_authnz.STATE_COOKIE_NAME) - self.trans.set_cookie(value=self.test_nonce, name=custos_authnz.NONCE_COOKIE_NAME) - login_redirect_url, user = self.custos_authnz.callback( - state_token="xxx", - authz_code=self.test_code, trans=self.trans, - login_redirect_url="http://localhost:8000/") - self.assertEqual(self.test_alt_username, user.username) - self.assertEqual(self.test_alt_email, user.email) - self.assertEqual(2, len(self.trans.sa_session.items)) - added_custos_authnz_token = self.trans.sa_session.items[1] - self.assertEqual(self.test_alt_user_id, added_custos_authnz_token.external_user_id) - def test_callback_galaxy_user_not_created_when_custos_authnz_token_exists(self): self.trans.set_cookie(value=self.test_state, name=custos_authnz.STATE_COOKIE_NAME) self.trans.set_cookie(value=self.test_nonce, name=custos_authnz.NONCE_COOKIE_NAME) From 06acc7a211afd58a20ead2d881b17b81cd5ac8e5 Mon Sep 17 00:00:00 2001 From: Marcus Christie Date: Wed, 13 Mar 2019 12:33:16 -0400 Subject: [PATCH 0552/1016] Rolling back addition of `implementation` param --- config/oidc_backends_config.xml.sample | 32 ++------- lib/galaxy/authnz/custos_authnz.py | 1 + lib/galaxy/authnz/managers.py | 71 ++++++++----------- lib/galaxy/webapps/galaxy/buildapp.py | 9 +-- .../webapps/galaxy/controllers/authnz.py | 8 +-- 5 files changed, 41 insertions(+), 80 deletions(-) diff --git a/config/oidc_backends_config.xml.sample b/config/oidc_backends_config.xml.sample index 486f7b59990..82ca68f2508 100644 --- a/config/oidc_backends_config.xml.sample +++ b/config/oidc_backends_config.xml.sample @@ -1,6 +1,6 @@ - + ... ... http://localhost:8080/authnz/google/callback @@ -32,33 +32,13 @@ consent - + ... ... - http://localhost:8000/authnz/google/oidc/callback + http://localhost:8000/authnz/custos/callback - - https://.../.well-known/openid-configuration - - - - - - - - - - - - - - - + ... + + diff --git a/lib/galaxy/authnz/custos_authnz.py b/lib/galaxy/authnz/custos_authnz.py index 8fc0b06401a..ca95475a915 100644 --- a/lib/galaxy/authnz/custos_authnz.py +++ b/lib/galaxy/authnz/custos_authnz.py @@ -177,6 +177,7 @@ class CustosAuthnz(IdentityProvider): self.config['authorization_endpoint'] = well_known_oidc_config['authorization_endpoint'] self.config['token_endpoint'] = well_known_oidc_config['token_endpoint'] self.config['userinfo_endpoint'] = well_known_oidc_config['userinfo_endpoint'] + # TODO: should we get this from library too? Allow override? self.config['extra_params'] = {'kc_idp_hint': 'cilogon'} def _get_well_known_uri_for_provider_and_realm(self, provider, realm): diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 0714057c0d4..714f6295e1a 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -74,6 +74,7 @@ class AuthnzManager(object): def _parse_oidc_backends_config(self, config_file): self.oidc_backends_config = {} + self.oidc_backends_implementation = {} try: tree = ET.parse(config_file) root = tree.getroot() @@ -89,11 +90,12 @@ class AuthnzManager(object): log.error("Could not find a node attribute 'name'; skipping the node '{}'.".format(child.tag)) continue idp = child.get('name').lower() - implementation = child.get('implementation', 'psa').lower() if idp in BACKENDS_NAME: - self.oidc_backends_config[idp, implementation] = self._parse_idp_config(child) - elif implementation == 'keycloak': - self.oidc_backends_config[idp, implementation] = self._parse_keycloak_config(child) + self.oidc_backends_config[idp] = self._parse_idp_config(child) + self.oidc_backends_implementation[idp] = 'psa' + elif idp == 'custos': + self.oidc_backends_config[idp] = self._parse_custos_config(child) + self.oidc_backends_implementation[idp] = 'custos' if len(self.oidc_backends_config) == 0: raise ParseError("No valid provider configuration parsed.") except ImportError: @@ -110,48 +112,33 @@ class AuthnzManager(object): rtv['prompt'] = config_xml.find('prompt').text return rtv - # TODO: rename - def _parse_generic_oidc_backend_config(self, config_xml): + def _parse_custos_config(self, config_xml): rtv = { 'client_id': config_xml.find('client_id').text, 'client_secret': config_xml.find('client_secret').text, - 'redirect_uri': config_xml.find('redirect_uri').text} + 'redirect_uri': config_xml.find('redirect_uri').text, + 'realm': config_xml.find('realm').text} if config_xml.find('well_known_oidc_config_uri') is not None: rtv['well_known_oidc_config_uri'] = config_xml.find('well_known_oidc_config_uri').text - else: - rtv['authorization_endpoint'] = config_xml.find('authorization_endpoint').text - rtv['token_endpoint'] = config_xml.find('token_endpoint').text - rtv['userinfo_endpoint'] = config_xml.find('userinfo_endpoint').text - if config_xml.find('extra_params') is not None: - rtv['extra_params'] = {} - for extra_param in config_xml.find('extra_params'): - rtv['extra_params'][extra_param.attrib['name']] = extra_param.attrib['value'] - if config_xml.find('userinfo_claim_mappings') is not None: - rtv['userinfo_claim_mappings'] = {} - for claim_mapping in config_xml.find('userinfo_claim_mappings'): - rtv['userinfo_claim_mappings'][claim_mapping.tag] = claim_mapping.text return rtv - def _unify_provider_implementation_names(self, provider, implementation): - if (provider.lower(), implementation.lower()) in self.oidc_backends_config: - return provider.lower(), implementation.lower() + def _unify_provider_name(self, provider): + if provider.lower() in self.oidc_backends_config: + return provider.lower() for k, v in BACKENDS_NAME.iteritems(): if v == provider: - return k.lower(), 'psa' - return None, None + return k.lower() + return None - def _unify_provider_name(self, provider): - return self._unify_provider_implementation_names(provider, 'psa') - - def _get_authnz_backend(self, provider, implementation): - unified_provider_name, implementation = self._unify_provider_implementation_names(provider, implementation) - if (unified_provider_name, implementation) in self.oidc_backends_config: + def _get_authnz_backend(self, provider): + unified_provider_name = self._unify_provider_name(provider) + if unified_provider_name in self.oidc_backends_config: provider = unified_provider_name - authnz_backend_class = self._get_authnz_backend_class(implementation) + identity_provider_class = self._get_identity_provider_class(self.oidc_backends_implementation[provider]) try: - return True, "", authnz_backend_class(provider_, self.oidc_config, self.oidc_backends_config[provider_, implementation_]) + return True, "", identity_provider_class(unified_provider_name, self.oidc_config, self.oidc_backends_config[unified_provider_name]) except Exception as e: - log.exception('An error occurred when loading {}'.format(authnz_backend_class.__name__)) + log.exception('An error occurred when loading {}'.format(identity_provider_class.__name__)) return False, str(e), None else: msg = 'The requested identity provider, `{}`, is not a recognized/expected provider.'.format(provider) @@ -159,10 +146,10 @@ class AuthnzManager(object): return False, msg, None @staticmethod - def _get_authnz_backend_class(implementation): + def _get_identity_provider_class(implementation): if implementation == 'psa': return PSAAuthnz - elif implementation == 'oidc': + elif implementation == 'custos': return CustosAuthnz else: return None @@ -229,19 +216,17 @@ class AuthnzManager(object): raise exceptions.ItemAccessibilityException(msg) return qres - def authenticate(self, provider, implementation, trans): + def authenticate(self, provider, trans): """ :type provider: string :param provider: set the name of the identity provider to be used for authentication flow. - :param implementation: name of technology used to implement - authentication to this identity provider :type trans: GalaxyWebTransaction :param trans: Galaxy web transaction. :return: an identity provider specific authentication redirect URI. """ try: - success, message, backend = self._get_authnz_backend(provider, implementation) + success, message, backend = self._get_authnz_backend(provider) if success is False: return False, message, None return True, "Redirecting to the `{}` identity provider for authentication".format(provider), backend.authenticate(trans) @@ -250,9 +235,9 @@ class AuthnzManager(object): log.exception(msg) return False, msg, None - def callback(self, provider, implementation, state_token, authz_code, trans, login_redirect_url): + def callback(self, provider, state_token, authz_code, trans, login_redirect_url): try: - success, message, backend = self._get_authnz_backend(provider, implementation) + success, message, backend = self._get_authnz_backend(provider) if success is False: return False, message, (None, None) return True, message, backend.callback(state_token, authz_code, trans, login_redirect_url) @@ -261,9 +246,9 @@ class AuthnzManager(object): log.exception(msg) return False, msg, (None, None) - def disconnect(self, provider, implementation, trans, disconnect_redirect_url=None): + def disconnect(self, provider, trans, disconnect_redirect_url=None): try: - success, message, backend = self._get_authnz_backend(provider, implementation) + success, message, backend = self._get_authnz_backend(provider) if success is False: return False, message, None return backend.disconnect(provider, trans, disconnect_redirect_url) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 3a1d65f58ec..34b27f0de72 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -68,12 +68,9 @@ def app_factory(global_conf, load_app_kwds={}, **kwargs): # Authentication endpoints. webapp.add_route('/authnz/', controller='authnz', action='index', provider=None) - webapp.add_route('/authnz/{provider}/login', controller='authnz', action='login', provider=None, implementation='PSA') - webapp.add_route('/authnz/{provider}/callback', controller='authnz', action='callback', provider=None, implementation='PSA') - webapp.add_route('/authnz/{provider}/disconnect', controller='authnz', action='disconnect', provider=None, implementation='PSA') - webapp.add_route('/authnz/{provider}/{implementation}/login', controller='authnz', action='login', provider=None, implementation=None) - webapp.add_route('/authnz/{provider}/{implementation}/callback', controller='authnz', action='callback', provider=None, implementation=None) - webapp.add_route('/authnz/{provider}/{implementation}/disconnect', controller='authnz', action='disconnect', provider=None, implementation=None) + webapp.add_route('/authnz/{provider}/login', controller='authnz', action='login', provider=None) + webapp.add_route('/authnz/{provider}/callback', controller='authnz', action='callback', provider=None) + webapp.add_route('/authnz/{provider}/disconnect', controller='authnz', action='disconnect', provider=None) # These two routes handle our simple needs at the moment webapp.add_route('/async/{tool_id}/{data_id}/{data_secret}', controller='async', action='index', tool_id=None, data_id=None, data_secret=None) diff --git a/lib/galaxy/webapps/galaxy/controllers/authnz.py b/lib/galaxy/webapps/galaxy/controllers/authnz.py index 16cd179603d..659abc82017 100644 --- a/lib/galaxy/webapps/galaxy/controllers/authnz.py +++ b/lib/galaxy/webapps/galaxy/controllers/authnz.py @@ -38,7 +38,7 @@ class OIDC(JSAppLauncher): return rtv @web.expose - def login(self, trans, provider, implementation): + def login(self, trans, provider): if not trans.app.config.enable_oidc: msg = "Login to Galaxy using third-party identities is not enabled on this Galaxy instance." log.debug(msg) @@ -50,7 +50,7 @@ class OIDC(JSAppLauncher): raise exceptions.AuthenticationFailed(message) @web.expose - def callback(self, trans, provider, implementation, **kwargs): + def callback(self, trans, provider, **kwargs): user = trans.user.username if trans.user is not None else 'anonymous' if not bool(kwargs): log.error("OIDC callback received no data for provider `{}` and user `{}`".format(provider, user)) @@ -66,7 +66,6 @@ class OIDC(JSAppLauncher): 'Please try again, and if the problem persists, contact ' 'the Galaxy instance admin'.format(provider)) success, message, (redirect_url, user) = trans.app.authnz_manager.callback(provider, - implementation, kwargs['state'], kwargs['code'], trans, @@ -83,12 +82,11 @@ class OIDC(JSAppLauncher): @web.expose @web.require_login("authenticate against the selected identity provider") - def disconnect(self, trans, provider, implementation, **kwargs): + def disconnect(self, trans, provider, **kwargs): if trans.user is None: # Only logged in users are allowed here. return success, message, redirect_url = trans.app.authnz_manager.disconnect(provider, - implementation, trans, disconnect_redirect_url=url_for('/')) if success is False: From a0f2289a1a010e20e0c1cc1f549c855e0a5961a8 Mon Sep 17 00:00:00 2001 From: Marcus Christie Date: Thu, 14 Mar 2019 10:19:00 -0400 Subject: [PATCH 0553/1016] Renumber custos_authnz_token migration --- ...uthnz_token_table.py => 0149_add_custos_authnz_token_table.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lib/galaxy/model/migrate/versions/{0147_add_custos_authnz_token_table.py => 0149_add_custos_authnz_token_table.py} (100%) diff --git a/lib/galaxy/model/migrate/versions/0147_add_custos_authnz_token_table.py b/lib/galaxy/model/migrate/versions/0149_add_custos_authnz_token_table.py similarity index 100% rename from lib/galaxy/model/migrate/versions/0147_add_custos_authnz_token_table.py rename to lib/galaxy/model/migrate/versions/0149_add_custos_authnz_token_table.py From ce7b0a63b71f3040e28470c786dde74803b55720 Mon Sep 17 00:00:00 2001 From: Marcus Christie Date: Thu, 14 Mar 2019 10:39:11 -0400 Subject: [PATCH 0554/1016] Allow overriding idphint (defaults to 'cilogon') --- config/oidc_backends_config.xml.sample | 2 ++ lib/galaxy/authnz/custos_authnz.py | 5 +++-- lib/galaxy/authnz/managers.py | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/config/oidc_backends_config.xml.sample b/config/oidc_backends_config.xml.sample index 82ca68f2508..5675014d480 100644 --- a/config/oidc_backends_config.xml.sample +++ b/config/oidc_backends_config.xml.sample @@ -40,5 +40,7 @@ ... + + diff --git a/lib/galaxy/authnz/custos_authnz.py b/lib/galaxy/authnz/custos_authnz.py index ca95475a915..d5d23560a6b 100644 --- a/lib/galaxy/authnz/custos_authnz.py +++ b/lib/galaxy/authnz/custos_authnz.py @@ -26,6 +26,9 @@ class CustosAuthnz(IdentityProvider): self.config['client_id'] = oidc_backend_config['client_id'] self.config['client_secret'] = oidc_backend_config['client_secret'] self.config['redirect_uri'] = oidc_backend_config['redirect_uri'] + self.config['extra_params'] = { + 'kc_idp_hint': oidc_backend_config.get('idphint', 'cilogon') + } # Either get OIDC config from well-known config URI or lookup known urls based on provider name and realm if 'well_known_oidc_config_uri' in oidc_backend_config: self.config['well_known_oidc_config_uri'] = oidc_backend_config['well_known_oidc_config_uri'] @@ -177,8 +180,6 @@ class CustosAuthnz(IdentityProvider): self.config['authorization_endpoint'] = well_known_oidc_config['authorization_endpoint'] self.config['token_endpoint'] = well_known_oidc_config['token_endpoint'] self.config['userinfo_endpoint'] = well_known_oidc_config['userinfo_endpoint'] - # TODO: should we get this from library too? Allow override? - self.config['extra_params'] = {'kc_idp_hint': 'cilogon'} def _get_well_known_uri_for_provider_and_realm(self, provider, realm): # TODO: Look up these URLs from a Python library diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 714f6295e1a..615d218484a 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -120,6 +120,8 @@ class AuthnzManager(object): 'realm': config_xml.find('realm').text} if config_xml.find('well_known_oidc_config_uri') is not None: rtv['well_known_oidc_config_uri'] = config_xml.find('well_known_oidc_config_uri').text + if config_xml.find('idphint') is not None: + rtv['idphint'] = config_xml.find('idphint').text return rtv def _unify_provider_name(self, provider): From b935c720c39399f2477b8ea8fb5e132a86c992c7 Mon Sep 17 00:00:00 2001 From: Marcus Christie Date: Thu, 14 Mar 2019 10:40:48 -0400 Subject: [PATCH 0555/1016] Remove custos-dev provider --- lib/galaxy/authnz/custos_authnz.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/galaxy/authnz/custos_authnz.py b/lib/galaxy/authnz/custos_authnz.py index d5d23560a6b..d95a3dbaa30 100644 --- a/lib/galaxy/authnz/custos_authnz.py +++ b/lib/galaxy/authnz/custos_authnz.py @@ -182,11 +182,9 @@ class CustosAuthnz(IdentityProvider): self.config['userinfo_endpoint'] = well_known_oidc_config['userinfo_endpoint'] def _get_well_known_uri_for_provider_and_realm(self, provider, realm): - # TODO: Look up these URLs from a Python library + # TODO: Look up this URL from a Python library if provider == 'custos': return "https://iam.scigap.org/auth/realms/{}/.well-known/openid-configuration".format(realm) - elif provider == 'custos-dev': - return "https://iamdev.scigap.org/auth/realms/{}/.well-known/openid-configuration".format(realm) else: raise Exception("Unknown Custos provider name: {}".format(provider)) From c211103d4a26852a622f828422fc97e732646d84 Mon Sep 17 00:00:00 2001 From: Marcus Christie Date: Tue, 19 Mar 2019 16:33:20 -0400 Subject: [PATCH 0556/1016] Use InCommon CA cert to verify iam.scigap.org --- lib/galaxy/authnz/custos_authnz.py | 24 +++++-- lib/galaxy/authnz/incommon_rsa_server_ca.pem | 68 ++++++++++++++++++++ 2 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 lib/galaxy/authnz/incommon_rsa_server_ca.pem diff --git a/lib/galaxy/authnz/custos_authnz.py b/lib/galaxy/authnz/custos_authnz.py index d95a3dbaa30..277ce2c491c 100644 --- a/lib/galaxy/authnz/custos_authnz.py +++ b/lib/galaxy/authnz/custos_authnz.py @@ -17,6 +17,8 @@ from ..authnz import IdentityProvider log = logging.getLogger(__name__) STATE_COOKIE_NAME = 'custos-state' NONCE_COOKIE_NAME = 'custos-nonce' +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +CA_CERTFILE = os.path.join(BASE_DIR, "incommon_rsa_server_ca.pem") class CustosAuthnz(IdentityProvider): @@ -132,10 +134,12 @@ class CustosAuthnz(IdentityProvider): log.warn("Setting OAUTHLIB_INSECURE_TRANSPORT to '1' to " "allow plain HTTP (non-SSL) callback") os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = "1" - return OAuth2Session(client_id, + session = OAuth2Session(client_id, scope=scope, redirect_uri=redirect_uri, state=state) + session.verify = self._get_verify_param() + return session def _fetch_token(self, oauth2_session, trans): client_secret = self.config['client_secret'] @@ -143,11 +147,13 @@ class CustosAuthnz(IdentityProvider): return oauth2_session.fetch_token( token_endpoint, client_secret=client_secret, - authorization_response=trans.request.url) + authorization_response=trans.request.url, + verify=self._get_verify_param()) def _get_userinfo(self, oauth2_session): userinfo_endpoint = self.config['userinfo_endpoint'] - return oauth2_session.get(userinfo_endpoint).json() + return oauth2_session.get(userinfo_endpoint, + verify=self._get_verify_param()).json() def _get_custos_authnz_token(self, sa_session, user_id, provider): return sa_session.query(CustosAuthnzToken).filter_by( @@ -184,13 +190,23 @@ class CustosAuthnz(IdentityProvider): def _get_well_known_uri_for_provider_and_realm(self, provider, realm): # TODO: Look up this URL from a Python library if provider == 'custos': + self.config['ca_bundle'] = CA_CERTFILE return "https://iam.scigap.org/auth/realms/{}/.well-known/openid-configuration".format(realm) else: raise Exception("Unknown Custos provider name: {}".format(provider)) def _load_well_known_oidc_config(self, well_known_uri): try: - return requests.get(well_known_uri).json() + return requests.get(well_known_uri, + verify=self._get_verify_param()).json() except Exception: log.error("Failed to load well-known OIDC config URI: {}".format(well_known_uri)) raise + + def _get_verify_param(self): + """Return 'ca_bundle' if 'verify_ssl' is true and 'ca_bundle' is configured.""" + # in requests_oauthlib, the verify param can either bea boolean or a CA bundle path + if 'ca_bundle' in self.config and self.config['verify_ssl']: + return self.config['ca_bundle'] + else: + return self.config['verify_ssl'] diff --git a/lib/galaxy/authnz/incommon_rsa_server_ca.pem b/lib/galaxy/authnz/incommon_rsa_server_ca.pem new file mode 100644 index 00000000000..63c6bae423f --- /dev/null +++ b/lib/galaxy/authnz/incommon_rsa_server_ca.pem @@ -0,0 +1,68 @@ +-----BEGIN CERTIFICATE----- +MIIF+TCCA+GgAwIBAgIQRyDQ+oVGGn4XoWQCkYRjdDANBgkqhkiG9w0BAQwFADCB +iDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0pl +cnNleSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNV +BAMTJVVTRVJUcnVzdCBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTQx +MDA2MDAwMDAwWhcNMjQxMDA1MjM1OTU5WjB2MQswCQYDVQQGEwJVUzELMAkGA1UE +CBMCTUkxEjAQBgNVBAcTCUFubiBBcmJvcjESMBAGA1UEChMJSW50ZXJuZXQyMREw +DwYDVQQLEwhJbkNvbW1vbjEfMB0GA1UEAxMWSW5Db21tb24gUlNBIFNlcnZlciBD +QTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJwb8bsvf2MYFVFRVA+e +xU5NEFj6MJsXKZDmMwysE1N8VJG06thum4ltuzM+j9INpun5uukNDBqeso7JcC7v +HgV9lestjaKpTbOc5/MZNrun8XzmCB5hJ0R6lvSoNNviQsil2zfVtefkQnI/tBPP +iwckRR6MkYNGuQmm/BijBgLsNI0yZpUn6uGX6Ns1oytW61fo8BBZ321wDGZq0GTl +qKOYMa0dYtX6kuOaQ80tNfvZnjNbRX3EhigsZhLI2w8ZMA0/6fDqSl5AB8f2IHpT +eIFken5FahZv9JNYyWL7KSd9oX8hzudPR9aKVuDjZvjs3YncJowZaDuNi+L7RyML +fzcCAwEAAaOCAW4wggFqMB8GA1UdIwQYMBaAFFN5v1qqK0rPVIDh2JvAnfKyA2bL +MB0GA1UdDgQWBBQeBaN3j2yW4luHS6a0hqxxAAznODAOBgNVHQ8BAf8EBAMCAYYw +EgYDVR0TAQH/BAgwBgEB/wIBADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUH +AwIwGwYDVR0gBBQwEjAGBgRVHSAAMAgGBmeBDAECAjBQBgNVHR8ESTBHMEWgQ6BB +hj9odHRwOi8vY3JsLnVzZXJ0cnVzdC5jb20vVVNFUlRydXN0UlNBQ2VydGlmaWNh +dGlvbkF1dGhvcml0eS5jcmwwdgYIKwYBBQUHAQEEajBoMD8GCCsGAQUFBzAChjNo +dHRwOi8vY3J0LnVzZXJ0cnVzdC5jb20vVVNFUlRydXN0UlNBQWRkVHJ1c3RDQS5j +cnQwJQYIKwYBBQUHMAGGGWh0dHA6Ly9vY3NwLnVzZXJ0cnVzdC5jb20wDQYJKoZI +hvcNAQEMBQADggIBAC0RBjjW29dYaK+qOGcXjeIT16MUJNkGE+vrkS/fT2ctyNMU +11ZlUp5uH5gIjppIG8GLWZqjV5vbhvhZQPwZsHURKsISNrqOcooGTie3jVgU0W+0 ++Wj8mN2knCVANt69F2YrA394gbGAdJ5fOrQmL2pIhDY0jqco74fzYefbZ/VS29fR +5jBxu4uj1P+5ZImem4Gbj1e4ZEzVBhmO55GFfBjRidj26h1oFBHZ7heDH1Bjzw72 +hipu47Gkyfr2NEx3KoCGMLCj3Btx7ASn5Ji8FoU+hCazwOU1VX55mKPU1I2250Lo +RCASN18JyfsD5PVldJbtyrmz9gn/TKbRXTr80U2q5JhyvjhLf4lOJo/UzL5WCXED +Smyj4jWG3R7Z8TED9xNNCxGBMXnMete+3PvzdhssvbORDwBZByogQ9xL2LUZFI/i +eoQp0UM/L8zfP527vWjEzuDN5xwxMnhi+vCToh7J159o5ah29mP+aJnvujbXEnGa +nrNxHzu+AGOePV8hwrGGG7hOIcPDQwkuYwzN/xT29iLp/cqf9ZhEtkGcQcIImH3b +oJ8ifsCnSbu0GB9L06Yqh7lcyvKDTEADslIaeSEINxhO2Y1fmcYFX/Fqrrp1WnhH +OjplXuXE0OPa0utaKC25Aplgom88L2Z8mEWcyfoB7zKOfD759AN7JKZWCYwk +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIF3jCCA8agAwIBAgIQAf1tMPyjylGoG7xkDjUDLTANBgkqhkiG9w0BAQwFADCB +iDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0pl +cnNleSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNV +BAMTJVVTRVJUcnVzdCBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAw +MjAxMDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBiDELMAkGA1UEBhMCVVMxEzARBgNV +BAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQKExVU +aGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBSU0EgQ2Vy +dGlmaWNhdGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK +AoICAQCAEmUXNg7D2wiz0KxXDXbtzSfTTK1Qg2HiqiBNCS1kCdzOiZ/MPans9s/B +3PHTsdZ7NygRK0faOca8Ohm0X6a9fZ2jY0K2dvKpOyuR+OJv0OwWIJAJPuLodMkY +tJHUYmTbf6MG8YgYapAiPLz+E/CHFHv25B+O1ORRxhFnRghRy4YUVD+8M/5+bJz/ +Fp0YvVGONaanZshyZ9shZrHUm3gDwFA66Mzw3LyeTP6vBZY1H1dat//O+T23LLb2 +VN3I5xI6Ta5MirdcmrS3ID3KfyI0rn47aGYBROcBTkZTmzNg95S+UzeQc0PzMsNT +79uq/nROacdrjGCT3sTHDN/hMq7MkztReJVni+49Vv4M0GkPGw/zJSZrM233bkf6 +c0Plfg6lZrEpfDKEY1WJxA3Bk1QwGROs0303p+tdOmw1XNtB1xLaqUkL39iAigmT +Yo61Zs8liM2EuLE/pDkP2QKe6xJMlXzzawWpXhaDzLhn4ugTncxbgtNMs+1b/97l +c6wjOy0AvzVVdAlJ2ElYGn+SNuZRkg7zJn0cTRe8yexDJtC/QV9AqURE9JnnV4ee +UB9XVKg+/XRjL7FQZQnmWEIuQxpMtPAlR1n6BB6T1CZGSlCBst6+eLf8ZxXhyVeE +Hg9j1uliutZfVS7qXMYoCAQlObgOK6nyTJccBz8NUvXt7y+CDwIDAQABo0IwQDAd +BgNVHQ4EFgQUU3m/WqorSs9UgOHYm8Cd8rIDZsswDgYDVR0PAQH/BAQDAgEGMA8G +A1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAFzUfA3P9wF9QZllDHPF +Up/L+M+ZBn8b2kMVn54CVVeWFPFSPCeHlCjtHzoBN6J2/FNQwISbxmtOuowhT6KO +VWKR82kV2LyI48SqC/3vqOlLVSoGIG1VeCkZ7l8wXEskEVX/JJpuXior7gtNn3/3 +ATiUFJVDBwn7YKnuHKsSjKCaXqeYalltiz8I+8jRRa8YFWSQEg9zKC7F4iRO/Fjs +8PRF/iKz6y+O0tlFYQXBl2+odnKPi4w2r78NBc5xjeambx9spnFixdjQg3IM8WcR +iQycE0xyNN+81XHfqnHd4blsjDwSXWXavVcStkNr/+XeTWYRUc+ZruwXtuhxkYze +Sf7dNXGiFSeUHM9h4ya7b6NnJSFd5t0dCy5oGzuCr+yDZ4XUmFF0sbmZgIn/f3gZ +XHlKYC6SQK5MNyosycdiyA5d9zZbyuAlJQG03RoHnHcAP9Dc1ew91Pq7P8yF1m9/ +qS3fuQL39ZeatTXaw2ewh0qpKJ4jjv9cJ2vhsE/zB+4ALtRZh8tSQZXq9EfX7mRB +VXyNWQKV3WKdwrnuWih0hKWbt5DHDAff9Yk2dDLWKMGwsAvgnEzDHNb842m1R0aB +L6KCq9NjRHDEjf8tM7qtj3u1cIiuPhnPQCjY/MiQu12ZIvVS5ljFH4gxQ+6IHdfG +jjxDah2nGN59PRbxYvnKkKj9 +-----END CERTIFICATE----- From e192462c4ac50b05d190ff9fb0083f2e77bd0a9d Mon Sep 17 00:00:00 2001 From: guerler Date: Tue, 19 Mar 2019 14:29:48 -0400 Subject: [PATCH 0557/1016] Increase spacing between tool execution button and help text --- .../galaxy/scripts/mvc/tool/tool-form-base.js | 3 +-- client/galaxy/style/scss/base.scss | 18 +++++++++--------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/client/galaxy/scripts/mvc/tool/tool-form-base.js b/client/galaxy/scripts/mvc/tool/tool-form-base.js index 08d79e4f368..646115ff9b6 100644 --- a/client/galaxy/scripts/mvc/tool/tool-form-base.js +++ b/client/galaxy/scripts/mvc/tool/tool-form-base.js @@ -267,8 +267,7 @@ export default FormBase.extend({ /** Templates */ _templateHelp: function(options) { var $tmpl = $("
") - .addClass("form-help") - .addClass("form-text") + .addClass("form-help form-text mt-4") .append(options.help); $tmpl.find("a").attr("target", "_blank"); $tmpl.find("img").each(function() { diff --git a/client/galaxy/style/scss/base.scss b/client/galaxy/style/scss/base.scss index ca467b2c230..738aa4fc9bd 100644 --- a/client/galaxy/style/scss/base.scss +++ b/client/galaxy/style/scss/base.scss @@ -840,9 +840,9 @@ button { .warningmessagelarge, .donemessagelarge, .infomessagelarge, -.ui-form-help .error, -.ui-form-help .warning, -.ui-form-help .note { +.form-help .error, +.form-help .warning, +.form-help .note { @extend .alert; min-height: 36px; padding-left: 52px; @@ -891,9 +891,9 @@ button { .warningmessagesmall, .donemessagesmall, .infomessagesmall, -.ui-form-help .error, -.ui-form-help .warning, -.ui-form-help .note { +.form-help .error, +.form-help .warning, +.form-help .note { @extend .alert; padding: 5px; padding-left: 25px; @@ -907,13 +907,13 @@ button { .errormessage, .errormessagesmall, -.ui-form-help .error { +.form-help .error { @extend .alert-danger; } .warningmessage, .warningmessagesmall, -.ui-form-help .warning { +.form-help .warning { @extend .alert-warning; background-image: url(../../images/warn_small.png); } @@ -926,7 +926,7 @@ button { .infomessage, .infomessagesmall, -.ui-form-help .note { +.form-help .note { @extend .alert-info; background-image: url(../../images/info_small.png); } From 1d240e4ee2e2f9394df3cfdecd32790bab28dbb8 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 7 Feb 2018 08:29:05 -0500 Subject: [PATCH 0558/1016] Library methods for dealing with sandboxed JavaScript expressions. Based on work by Peter Amstutz in cwltool (https://github.com/common-workflow-language/cwltool). --- lib/galaxy/config.py | 1 + lib/galaxy/tools/__init__.py | 11 ++ lib/galaxy/tools/expressions/__init__.py | 12 ++ lib/galaxy/tools/expressions/cwlNodeEngine.js | 46 ++++++ lib/galaxy/tools/expressions/evaluation.py | 37 +++++ lib/galaxy/tools/expressions/sandbox.py | 151 ++++++++++++++++++ lib/galaxy/tools/expressions/util.py | 13 ++ test/unit/tools/test_expression_basics.py | 8 + 8 files changed, 279 insertions(+) create mode 100644 lib/galaxy/tools/expressions/__init__.py create mode 100644 lib/galaxy/tools/expressions/cwlNodeEngine.js create mode 100644 lib/galaxy/tools/expressions/evaluation.py create mode 100644 lib/galaxy/tools/expressions/sandbox.py create mode 100644 lib/galaxy/tools/expressions/util.py create mode 100644 test/unit/tools/test_expression_basics.py diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index e529fddf706..bb1877b2d83 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -313,6 +313,7 @@ class Configuration(object): log.warning("preserve_python_environment set to unknown value [%s], defaulting to legacy_only") preserve_python_environment = "legacy_only" self.preserve_python_environment = preserve_python_environment + self.nodejs_path = kwargs.get("nodejs_path", None) # Older default container cache path, I don't think anyone is using it anymore and it wasn't documented - we # should probably drop the backward compatiblity to save the path check. self.container_image_cache_path = self.resolve_path(kwargs.get("container_image_cache_path", "database/container_images")) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 0e2c3da4abd..19618bf4ca2 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -36,6 +36,7 @@ from galaxy.tools.actions.model_operations import ModelOperationToolAction from galaxy.tools.deps import ( CachedDependencyManager, ) +from galaxy.tools import expressions from galaxy.tools.fetcher import ToolLocationFetcher from galaxy.tools.parameters import ( check_param, @@ -102,6 +103,12 @@ from .provided_metadata import parse_tool_provided_metadata log = logging.getLogger(__name__) +REQUIRES_JS_RUNTIME_MESSAGE = ("The tool [%s] requires a nodejs runtime to execute " + "but node nor nodejs could be found on Galaxy's PATH and " + "no runtime was configured using the nodejs_path option in " + "galaxy.ini.") + + HELP_UNINITIALIZED = threading.Lock() MODEL_TOOLS_PATH = os.path.abspath(os.path.dirname(__file__)) # Tools that require Galaxy's Python environment to be preserved. @@ -747,6 +754,10 @@ class Tool(Dictifiable): module, cls = action mod = __import__(module, globals(), locals(), [cls]) self.tool_action = getattr(mod, cls)() + if getattr(self.tool_action, "requires_js_runtime", False): + if expressions.find_engine(self.app.config) is None: + message = REQUIRES_JS_RUNTIME_MESSAGE % self.tool_id + raise Exception(message) # Tests self.__parse_tests(tool_source) diff --git a/lib/galaxy/tools/expressions/__init__.py b/lib/galaxy/tools/expressions/__init__.py new file mode 100644 index 00000000000..b7519e5f57c --- /dev/null +++ b/lib/galaxy/tools/expressions/__init__.py @@ -0,0 +1,12 @@ +from .evaluation import evaluate +from .sandbox import execjs, interpolate +from .util import jshead, find_engine + + +__all__ = ( + 'evaluate', + 'execjs', + 'find_engine', + 'interpolate', + 'jshead', +) diff --git a/lib/galaxy/tools/expressions/cwlNodeEngine.js b/lib/galaxy/tools/expressions/cwlNodeEngine.js new file mode 100644 index 00000000000..1129584977f --- /dev/null +++ b/lib/galaxy/tools/expressions/cwlNodeEngine.js @@ -0,0 +1,46 @@ +#!/usr/bin/env nodejs + +"use strict"; + +process.stdin.setEncoding('utf8'); + +var incoming = ""; + +process.stdin.on('readable', function() { + var chunk = process.stdin.read(); + if (chunk !== null) { + incoming += chunk; + } +}); + +process.stdin.on('end', function() { + var j = JSON.parse(incoming); + var exp = "" + + if (j.script[0] == "{") { + exp = "{return function()" + j.script + "();}"; + } + else { + exp = "{return " + j.script + ";}"; + } + + var fn = '"use strict";\n'; + + if (j.engineConfig) { + for (var index = 0; index < j.engineConfig.length; ++index) { + fn += j.engineConfig[index] + "\n"; + } + } + + fn += "var $job = " + JSON.stringify(j.job) + ";\n"; + fn += "var $self = " + JSON.stringify(j.context) + ";\n" + + fn += "var $runtime = " + JSON.stringify(j.runtime) + ";\n" + fn += "var $tmpdir = " + JSON.stringify(j.tmpdir) + ";\n" + fn += "var $outdir = " + JSON.stringify(j.outdir) + ";\n" + + + fn += "(function()" + exp + ")()"; + + process.stdout.write(JSON.stringify(require("vm").runInNewContext(fn, {}))); +}); diff --git a/lib/galaxy/tools/expressions/evaluation.py b/lib/galaxy/tools/expressions/evaluation.py new file mode 100644 index 00000000000..89becc1f862 --- /dev/null +++ b/lib/galaxy/tools/expressions/evaluation.py @@ -0,0 +1,37 @@ +import json +import os +import subprocess + +from .util import find_engine + +FILE_DIRECTORY = os.path.normpath(os.path.dirname(os.path.join(__file__))) +NODE_ENGINE = os.path.join(FILE_DIRECTORY, "cwlNodeEngine.js") + + +def evaluate(config, input): + application = find_engine(config) + + default_context = { + "engineConfig": [], + "job": {}, + "context": None, + "outdir": None, + "tmpdir": None, + } + + new_input = default_context + new_input.update(input) + + sp = subprocess.Popen([application, NODE_ENGINE], + shell=False, + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + + (stdoutdata, stderrdata) = sp.communicate(json.dumps(new_input) + "\n\n") + if sp.returncode != 0: + args = (json.dumps(new_input, indent=4), stdoutdata, stderrdata) + message = "Expression engine returned non-zero exit code on evaluation of\n%s%s%s" % args + raise Exception(message) + + return json.loads(stdoutdata) diff --git a/lib/galaxy/tools/expressions/sandbox.py b/lib/galaxy/tools/expressions/sandbox.py new file mode 100644 index 00000000000..a3646e100d4 --- /dev/null +++ b/lib/galaxy/tools/expressions/sandbox.py @@ -0,0 +1,151 @@ +import subprocess +import json +import threading + +from .util import find_engine + + +class JavascriptException(Exception): + pass + + +def execjs(config, js, jslib): + application = find_engine(config) + try: + nodejs = subprocess.Popen([application], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except OSError as e: + if e.errno == 2: + nodejs = subprocess.Popen(["docker", "run", + "--attach=STDIN", "--attach=STDOUT", "--attach=STDERR", + "--interactive", + "--rm", + "commonworkflowlanguage/nodejs-engine", "nodejs"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + else: + raise + + fn = "\"use strict\";%s\n(function()%s)()" % (jslib, js if isinstance(js, basestring) and len(js) > 1 and js[0] == '{' else ("{return (%s);}" % js)) + script = "console.log(JSON.stringify(require(\"vm\").runInNewContext(%s, {})));\n" % json.dumps(fn) + + def term(): + try: + nodejs.terminate() + except OSError: + pass + + # Time out after 5 seconds + tm = threading.Timer(5, term) + tm.start() + + stdoutdata, stderrdata = nodejs.communicate(script) + tm.cancel() + + if nodejs.returncode != 0: + raise JavascriptException("Returncode was: %s\nscript was: %s\nstdout was: '%s'\nstderr was: '%s'\n" % (nodejs.returncode, script, stdoutdata, stderrdata)) + else: + return json.loads(stdoutdata) + + +class SubstitutionError(Exception): + pass + + +DEFAULT = 0 +DOLLAR = 1 +PAREN = 2 +BRACE = 3 +SINGLE_QUOTE = 4 +DOUBLE_QUOTE = 5 +BACKSLASH = 6 + + +def scanner(scan): + + i = 0 + stack = [DEFAULT] + start = 0 + while i < len(scan): + state = stack[-1] + c = scan[i] + + if state == DEFAULT: + if c == '$': + stack.append(DOLLAR) + elif c == '\\': + stack.append(BACKSLASH) + elif state == BACKSLASH: + stack.pop() + if stack[-1] == DEFAULT: + return [i - 1, i + 1] + elif state == DOLLAR: + if c == '(': + start = i - 1 + stack.append(PAREN) + elif c == '{': + start = i - 1 + stack.append(BRACE) + elif state == PAREN: + if c == '(': + stack.append(PAREN) + elif c == ')': + stack.pop() + if stack[-1] == DOLLAR: + return [start, i + 1] + elif c == "'": + stack.append(SINGLE_QUOTE) + elif c == '"': + stack.append(DOUBLE_QUOTE) + elif state == BRACE: + if c == '{': + stack.append(BRACE) + elif c == '}': + stack.pop() + if stack[-1] == DOLLAR: + return [start, i + 1] + elif c == "'": + stack.append(SINGLE_QUOTE) + elif c == '"': + stack.append(DOUBLE_QUOTE) + elif state == SINGLE_QUOTE: + if c == "'": + stack.pop() + elif c == '\\': + stack.append(BACKSLASH) + elif state == DOUBLE_QUOTE: + if c == '"': + stack.pop() + elif c == '\\': + stack.append(BACKSLASH) + i += 1 + + if len(stack) > 1: + raise SubstitutionError("Substitution error, unfinished block starting at position {}: {}".format(start, scan[start:])) + else: + return None + + +def interpolate(scan, jslib): + scan = scan.strip() + parts = [] + w = scanner(scan) + while w: + parts.append(scan[0:w[0]]) + + if scan[w[0]] == '$': + e = execjs(scan[w[0] + 1:w[1]], jslib) + if w[0] == 0 and w[1] == len(scan): + return e + leaf = json.dumps(e, sort_keys=True) + if leaf[0] == '"': + leaf = leaf[1:-1] + parts.append(leaf) + elif scan[w[0]] == '\\': + e = scan[w[1] - 1] + parts.append(e) + + scan = scan[w[1]:] + w = scanner(scan) + parts.append(scan) + return ''.join(parts) diff --git a/lib/galaxy/tools/expressions/util.py b/lib/galaxy/tools/expressions/util.py new file mode 100644 index 00000000000..b7ea1bf090a --- /dev/null +++ b/lib/galaxy/tools/expressions/util.py @@ -0,0 +1,13 @@ +import json +from galaxy.tools.deps.commands import which + + +def find_engine(config): + nodejs_path = getattr(config, "nodejs_path", None) + if nodejs_path is None: + nodejs_path = which("nodejs") or which("node") or None + return nodejs_path + + +def jshead(engine_config, root_vars): + return "\n".join(engine_config + ["var %s = %s;" % (k, json.dumps(v)) for k, v in root_vars.items()]) diff --git a/test/unit/tools/test_expression_basics.py b/test/unit/tools/test_expression_basics.py new file mode 100644 index 00000000000..51c7293ac3f --- /dev/null +++ b/test/unit/tools/test_expression_basics.py @@ -0,0 +1,8 @@ +from galaxy.tools.expressions import evaluate + + +def test_evaluate(): + input = { + "script": "{return 5;}" + } + assert evaluate(None, input) == 5 From f5c93e868b7be73f7ced5b7bba8b35b7e41728cc Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 19 Apr 2018 16:08:30 -0400 Subject: [PATCH 0559/1016] Implement expression tools and non-data tool outputs. PR #6925 introduced a GUI for connecting non-data (e.g. integer, boolean, color, etc..) workflow input parameter to tool input parameters (the backend for this was originally added in #1306). That ideally was just the beginning of work toward using such values in structured ways in workflows. This PR extends tool output handling to allow producing of non-data parameters. These can serve as a source for non-data values in workflows the same work workflow input parameters can. To make such values more easy to produce, this PR also introduces Galaxy expression tools - mirroring functionality regularly used in CWL. These small JavaScript-based tools that consume inputs just like a regular Galaxy tool but that produce dictionary of non-data values. I think these expressions will be maximally useful when paired with format 2 workflows once we allow users to load arbitrary tools (I make the case more in full here https://github.com/galaxyproject/galaxy/pull/7545#issuecomment-473894424), but I outline some potential uses there as well. Because there is always a checklist in my PR descriptions: - Tool definition language and plumbing and datatype for expressing expressions as jobs. - Allow connecting expression tools to parameters in workflows, will delay evaluation of workflow so calculated value - Example test expression tools for testing and demonstration. --- config/datatypes_conf.xml.sample | 1 + lib/galaxy/datatypes/text.py | 24 +++ lib/galaxy/tools/__init__.py | 69 ++++++- lib/galaxy/tools/evaluation.py | 2 +- lib/galaxy/tools/expressions/__init__.py | 8 + lib/galaxy/tools/expressions/script.py | 15 ++ lib/galaxy/tools/parameters/wrapped_json.py | 25 ++- lib/galaxy/tools/parser/interface.py | 5 + lib/galaxy/tools/parser/output_objects.py | 26 ++- lib/galaxy/tools/parser/xml.py | 49 ++++- lib/galaxy/tools/parser/yaml.py | 3 + lib/galaxy/tools/xsd/galaxy.xsd | 194 ++++++++++++------ lib/galaxy/workflow/run.py | 12 ++ lib/galaxy_ext/expressions/__init__.py | 0 lib/galaxy_ext/expressions/handle_job.py | 49 +++++ test/api/test_tools.py | 42 ++++ test/api/test_workflows.py | 23 ++- .../functional/tools/expression_forty_two.xml | 15 ++ .../tools/expression_log_line_count.xml | 14 ++ .../functional/tools/expression_parse_int.xml | 14 ++ test/functional/tools/samples_tool_conf.xml | 4 +- test/unit/jobs/test_expression_run.py | 48 +++++ test/unit/tools/test_parsing.py | 49 +++++ 23 files changed, 623 insertions(+), 68 deletions(-) create mode 100644 lib/galaxy/tools/expressions/script.py create mode 100644 lib/galaxy_ext/expressions/__init__.py create mode 100644 lib/galaxy_ext/expressions/handle_job.py create mode 100644 test/functional/tools/expression_forty_two.xml create mode 100644 test/functional/tools/expression_log_line_count.xml create mode 100644 test/functional/tools/expression_parse_int.xml create mode 100644 test/unit/jobs/test_expression_run.py diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 76fdb0c7ee5..12f4fae5d2f 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -414,6 +414,7 @@ + diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index c2e71444cf1..c563c2d318c 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -106,6 +106,30 @@ class Json(Text): return "JSON file (%s)" % (nice_size(dataset.get_size())) +class ExpressionJson(Json): + """ Represents the non-data input or output to a tool or workflow. + """ + file_ext = "json" + MetadataElement(name="json_type", default=None, desc="JavaScript or JSON type of expression", readonly=True, visible=True, no_value=None) + + def set_meta(self, dataset, **kwd): + """ + """ + json_type = "null" + with open(dataset.file_name) as f: + obj = json.load(f) + if isinstance(obj, int): + json_type = "int" + elif isinstance(obj, float): + json_type = "float" + elif isinstance(obj, list): + json_type = "list" + elif isinstance(obj, dict): + json_type = "object" + + dataset.metadata.json_type = json_type + + @build_sniff_from_prefix class Ipynb(Json): file_ext = "ipynb" diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 19618bf4ca2..f63f33a94b1 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -63,6 +63,7 @@ from galaxy.tools.parameters.dataset_matcher import ( from galaxy.tools.parameters.grouping import Conditional, ConditionalWhen, Repeat, Section, UploadDataset from galaxy.tools.parameters.input_translation import ToolInputTranslator from galaxy.tools.parameters.meta import expand_meta_parameters +from galaxy.tools.parameters.wrapped_json import json_wrap from galaxy.tools.parser import ( get_tool_source, ToolOutputCollectionPart @@ -2188,6 +2189,72 @@ class OutputParameterJSONTool(Tool): out.close() +class ExpressionTool(Tool): + requires_js_runtime = True + tool_type = 'expression' + EXPRESSION_INPUTS_NAME = "_expression_inputs_.json" + + def parse_command(self, tool_source): + self.command = "cd ../; %s" % expressions.EXPRESSION_SCRIPT_CALL + self.interpreter = None + self._expression = tool_source.parse_expression().strip() + + def parse_outputs(self, tool_source): + # Setup self.outputs and self.output_collections + super(ExpressionTool, self).parse_outputs(tool_source) + + # Validate these outputs for expression tools. + if len(self.output_collections) != 0: + message = "Expression tools may not declare output collections at this time." + raise Exception(message) + for output in self.outputs.values(): + if not hasattr(output, "from_expression"): + message = "Expression tools may not declare output datasets at this time." + raise Exception(message) + + def exec_before_job(self, app, inp_data, out_data, param_dict=None): + super(ExpressionTool, self).exec_before_job(app, inp_data, out_data, param_dict=param_dict) + local_working_directory = param_dict["__local_working_directory__"] + expression_inputs_path = os.path.join(local_working_directory, ExpressionTool.EXPRESSION_INPUTS_NAME) + + outputs = [] + for i, (out_name, data) in enumerate(out_data.iteritems()): + output_def = self.outputs[out_name] + wrapped_data = param_dict.get(out_name) + file_name = str(wrapped_data) + + outputs.append(dict( + name=out_name, + from_expression=output_def.from_expression, + path=file_name, + )) + + if param_dict is None: + raise Exception("Internal error - param_dict is empty.") + + job = {} + json_wrap(self.inputs, param_dict, job, handle_files='OBJECT') + expression_inputs = { + 'job': job, + 'script': self._expression, + 'outputs': outputs, + } + expressions.write_evalute_script(os.path.join(local_working_directory)) + with open(expression_inputs_path, "w") as f: + json.dump(expression_inputs, f) + + def parse_environment_variables(self, tool_source): + """ Setup environment variable for inputs file. + """ + environmnt_variables_raw = super(ExpressionTool, self).parse_environment_variables(tool_source) + expression_script_inputs = dict( + name="GALAXY_EXPRESSION_INPUTS", + template=ExpressionTool.EXPRESSION_INPUTS_NAME, + ) + environmnt_variables_raw.append(expression_script_inputs) + return environmnt_variables_raw + + class DataSourceTool(OutputParameterJSONTool): """ Alternate implementation of Tool for data_source tools -- those that @@ -2934,7 +3001,7 @@ class FilterFromFileTool(DatabaseOperationTool): # Populate tool_type to ToolClass mappings tool_types = {} -for tool_class in [Tool, SetMetadataTool, OutputParameterJSONTool, +for tool_class in [Tool, SetMetadataTool, OutputParameterJSONTool, ExpressionTool, DataManagerTool, DataSourceTool, AsyncDataSourceTool, UnzipCollectionTool, ZipCollectionTool, MergeCollectionTool, RelabelFromFileTool, FilterFromFileTool, BuildListCollectionTool, ExtractDatasetCollectionTool, diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index b244364ac6e..9e56df5e56c 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -380,7 +380,7 @@ class ToolEvaluator(object): param_dict['__tool_directory__'] = self.compute_environment.tool_directory() param_dict['__get_data_table_entry__'] = get_data_table_entry - + param_dict['__local_working_directory__'] = self.local_working_directory # We add access to app here, this allows access to app.config, etc param_dict['__app__'] = RawObjectWrapper(self.app) # More convienent access to app.config.new_file_path; we don't need to diff --git a/lib/galaxy/tools/expressions/__init__.py b/lib/galaxy/tools/expressions/__init__.py index b7519e5f57c..b731044825a 100644 --- a/lib/galaxy/tools/expressions/__init__.py +++ b/lib/galaxy/tools/expressions/__init__.py @@ -1,12 +1,20 @@ from .evaluation import evaluate from .sandbox import execjs, interpolate from .util import jshead, find_engine +from .script import ( + write_evalute_script, + EXPRESSION_SCRIPT_CALL, + EXPRESSION_SCRIPT_NAME, +) __all__ = ( 'evaluate', 'execjs', + 'EXPRESSION_SCRIPT_CALL', + 'EXPRESSION_SCRIPT_NAME', 'find_engine', 'interpolate', 'jshead', + 'write_evalute_script', ) diff --git a/lib/galaxy/tools/expressions/script.py b/lib/galaxy/tools/expressions/script.py new file mode 100644 index 00000000000..0cb18ae7464 --- /dev/null +++ b/lib/galaxy/tools/expressions/script.py @@ -0,0 +1,15 @@ +import os + +EXPRESSION_SCRIPT_NAME = "_evaluate_expression_.py" +EXPRESSION_SCRIPT_CALL = "python %s" % EXPRESSION_SCRIPT_NAME + + +def write_evalute_script(in_directory): + """ Responsible for writing the script that evaluates expressions + in Galaxy jobs. + """ + script = os.path.join(in_directory, EXPRESSION_SCRIPT_NAME) + with open(script, "w") as f: + f.write('from galaxy_ext.expressions.handle_job import run; run()') + + return script diff --git a/lib/galaxy/tools/parameters/wrapped_json.py b/lib/galaxy/tools/parameters/wrapped_json.py index c045c9595ef..69549943143 100644 --- a/lib/galaxy/tools/parameters/wrapped_json.py +++ b/lib/galaxy/tools/parameters/wrapped_json.py @@ -62,8 +62,14 @@ def _json_wrap_input(input, value, handle_files="skip"): json_value = _data_input_to_path(value) elif handle_files == "skip": return SKIP_INPUT - else: - raise NotImplementedError() + elif handle_files == "OBJECT": + if value: + if isinstance(value, list): + value = value[0] + return _hda_to_object(value) + else: + return None + raise NotImplementedError() elif input_type == "data_collection": if handle_files == "skip": return SKIP_INPUT @@ -88,6 +94,21 @@ def _json_wrap_input(input, value, handle_files="skip"): return json_value +def _hda_to_object(hda): + hda_dict = hda.to_dict() + metadata_dict = {} + + for key, value in hda_dict.items(): + if key.startswith("metadata_"): + metadata_dict[key[len("metadata_"):]] = value + + return { + 'file_ext': hda_dict['file_ext'], + 'name': hda_dict['name'], + 'metadata': metadata_dict, + } + + def _cast_if_not_none(value, cast_to, empty_to_none=False): # log.debug("value [%s], type[%s]" % (value, type(value))) if value is None or (empty_to_none and str(value) == ''): diff --git a/lib/galaxy/tools/parser/interface.py b/lib/galaxy/tools/parser/interface.py index 2cd31a1aa6f..bdc979dccfc 100644 --- a/lib/galaxy/tools/parser/interface.py +++ b/lib/galaxy/tools/parser/interface.py @@ -88,6 +88,11 @@ class ToolSource(object): """ Return string contianing command to run. """ + def parse_expression(self): + """ Return string contianing command to run. + """ + return None + @abstractmethod def parse_environment_variables(self): """ Return environment variable templates to expose. diff --git a/lib/galaxy/tools/parser/output_objects.py b/lib/galaxy/tools/parser/output_objects.py index bfe51934ff9..805335a8358 100644 --- a/lib/galaxy/tools/parser/output_objects.py +++ b/lib/galaxy/tools/parser/output_objects.py @@ -24,12 +24,13 @@ class ToolOutput(ToolOutputBase): (format, metadata_source, parent) """ - dict_collection_visible_keys = ['name', 'format', 'label', 'hidden'] + dict_collection_visible_keys = ['name', 'format', 'label', 'hidden', 'output_type'] def __init__(self, name, format=None, format_source=None, metadata_source=None, parent=None, label=None, filters=None, actions=None, hidden=False, implicit=False): super(ToolOutput, self).__init__(name, label=label, filters=filters, hidden=hidden) + self.output_type = "data" self.format = format self.format_source = format_source self.metadata_source = metadata_source @@ -70,6 +71,27 @@ class ToolOutput(ToolOutputBase): return as_dict +class ToolExpressionOutput(ToolOutputBase): + dict_collection_visible_keys = ('name', 'format', 'label', 'hidden', 'output_type') + + def __init__(self, name, output_type, from_expression, + label=None, filters=None, actions=None, hidden=False): + super(ToolExpressionOutput, self).__init__(name, label=label, filters=filters, hidden=hidden) + self.output_type = output_type # JSON type... + self.from_expression = from_expression + self.format = "expression.json" # galaxy.datatypes.text.ExpressionJson.file_ext + + self.format_source = None + self.metadata_source = None + self.parent = None + self.actions = actions + + # Initialize default values + self.change_format = [] + self.implicit = False + self.from_work_dir = None + + class ToolOutputCollection(ToolOutputBase): """ Represents a HistoryDatasetCollectionAssociation of output datasets produced @@ -85,6 +107,7 @@ class ToolOutputCollection(ToolOutputBase): """ + dict_collection_visible_keys = ('name', 'format', 'label', 'hidden', 'output_type') dict_collection_visible_keys = ['name', 'default_format', 'label', 'hidden', 'inherit_format', 'inherit_metadata'] @@ -102,6 +125,7 @@ class ToolOutputCollection(ToolOutputBase): inherit_metadata=False ): super(ToolOutputCollection, self).__init__(name, label=label, filters=filters, hidden=hidden) + self.output_type = "collection" self.collection = True self.default_format = default_format self.structure = structure diff --git a/lib/galaxy/tools/parser/xml.py b/lib/galaxy/tools/parser/xml.py index 997eb760406..4d67278c26f 100644 --- a/lib/galaxy/tools/parser/xml.py +++ b/lib/galaxy/tools/parser/xml.py @@ -22,6 +22,7 @@ from .interface import ( from .output_actions import ToolOutputActionGroup from .output_collection_def import dataset_collector_descriptions_from_elem from .output_objects import ( + ToolExpressionOutput, ToolOutput, ToolOutputCollection, ToolOutputCollectionStructure @@ -111,6 +112,12 @@ class XmlToolSource(ToolSource): command_el = self._command_el return ((command_el is not None) and command_el.text) or None + def parse_expression(self): + """ Return string contianing command to run. + """ + expression_el = self.root.find("expression") + return ((expression_el is not None) and expression_el.text) or None + def parse_environment_variables(self): environment_variables_el = self.root.find("environment_variables") if environment_variables_el is None: @@ -256,7 +263,12 @@ class XmlToolSource(ToolSource): for _ in out_elem.findall("data"): _parse(_) - for collection_elem in out_elem.findall("collection"): + def _parse_expression(output_elem, **kwds): + output_def = self._parse_expression_output(output_elem, tool, **kwds) + data_dict[output_def.name] = output_def + return output_def + + def _parse_collection(collection_elem): name = collection_elem.get("name") label = xml_text(collection_elem, "label") default_format = collection_elem.get("format", "data") @@ -312,6 +324,24 @@ class XmlToolSource(ToolSource): output_collection.outputs[output_name] = data output_collections[name] = output_collection + for out_child in out_elem.getchildren(): + if out_child.tag == "data": + _parse(out_child) + elif out_child.tag == "collection": + _parse_collection(out_child) + elif out_child.tag == "output": + output_type = out_child.get("type") + if output_type == "data": + _parse(out_child) + elif output_type == "collection": + out_child.attrib["type"] = out_child.get("collection_type") + out_child.attrib["type_source"] = out_child.get("collection_type_source") + _parse_collection(out_child) + else: + _parse_expression(out_child) + else: + log.warn("Unknown output tag encountered [%s]" % out_child.tag) + for output_def in data_dict.values(): outputs[output_def.name] = output_def return outputs, output_collections @@ -323,6 +353,7 @@ class XmlToolSource(ToolSource): default_format="data", default_format_source=None, default_metadata_source="", + expression_type=None, ): output = ToolOutput(data_elem.get("name")) output_format = data_elem.get("format", default_format) @@ -347,6 +378,22 @@ class XmlToolSource(ToolSource): output.dataset_collector_descriptions = dataset_collector_descriptions_from_elem(data_elem, legacy=self.legacy_defaults) return output + def _parse_expression_output(self, output_elem, tool, **kwds): + output_type = output_elem.get("type") + from_expression = output_elem.get("from") + output = ToolExpressionOutput( + output_elem.get("name"), + output_type, + from_expression, + ) + output.path = output_elem.get("value") + output.label = xml_text(output_elem, "label") + + output.hidden = string_as_bool(output_elem.get("hidden", "")) + output.actions = ToolOutputActionGroup(output, output_elem.find('actions')) + output.dataset_collector_descriptions = [] + return output + def parse_stdio(self): """ parse error handling from command and stdio tag diff --git a/lib/galaxy/tools/parser/yaml.py b/lib/galaxy/tools/parser/yaml.py index 63e642e5648..52ae9d4cb07 100644 --- a/lib/galaxy/tools/parser/yaml.py +++ b/lib/galaxy/tools/parser/yaml.py @@ -54,6 +54,9 @@ class YamlToolSource(ToolSource): def parse_command(self): return self.root_dict.get("command") + def parse_expression(self): + return self.root_dict.get("expression") + def parse_environment_variables(self): return [] diff --git a/lib/galaxy/tools/xsd/galaxy.xsd b/lib/galaxy/tools/xsd/galaxy.xsd index 17e3191df80..a298cc13dd1 100644 --- a/lib/galaxy/tools/xsd/galaxy.xsd +++ b/lib/galaxy/tools/xsd/galaxy.xsd @@ -67,7 +67,8 @@ the tool menu immediately following the hyperlink for the tool (based on the - + + @@ -2781,6 +2782,33 @@ Prior to Galaxy release 19.01 the stdio block has only been used for non-legacy + + + + + + + + + Type of expression defined by this expression block. The only current valid option is emca5.1 - which will evaluate the expression in a sandbox using node. The option still must be specified to allow a different default in the future. + + + + + + + + + + + + + + @@ -3684,6 +3712,7 @@ The default is ``galaxy.json``. + @@ -3725,6 +3754,71 @@ pipes or periods (e.g. ``.``).]]> + + + + + + + + + The short name for the output datatype. +The valid values for format can be found in +[/config/datatypes_conf.xml.sample](https://github.com/galaxyproject/galaxy/blob/dev/config/datatypes_conf.xml.sample) +(e.g. ``format="pdf"`` or ``format="fastqsanger"``). + + + + + Sets the source of element identifier to the specified input. +This only applies to collections that are mapped over a non-collection input and that have equivalent structures. If this references input elements in conditionals, this value should be qualified (e.g. ``cond|input`` instead of ``input`` if ``input`` is in a conditional with ``name="cond"``). + + + + + This copies the metadata information +from the tool's input dataset. This is particularly useful for interval data +types where the order of the columns is not set. + + + + + Relative path to a file produced by the +tool in its working directory. Output's contents are set to this file's +contents. + + + + + Boolean indicating whether to hide +dataset in the history view. (Default is ``false``.) + + + + + + + + This is the name of input collection or +dataset to derive "structure" of the output from (output element count and +identifiers). For instance, if the referenced input has three ordered items with +identifiers ``sample1``, ``sample2``, and ``sample3``. If this references input +elements in conditionals, this value should be qualified (e.g. ``cond|input`` instead +of ``input`` if ``input`` is in a conditional with ``name="cond"``). + + + + + If ``structured_like`` is set, inherit +format of outputs from format of corresponding input. + + + + - - - - - - - - The short name for the output datatype. -The valid values for format can be found in -[/config/datatypes_conf.xml.sample](https://github.com/galaxyproject/galaxy/blob/dev/config/datatypes_conf.xml.sample) -(e.g. ``format="pdf"`` or ``format="fastqsanger"``). - - - - - Sets the source of element identifier to the specified input. -This only applies to collections that are mapped over a non-collection input and that have equivalent structures. If this references input elements in conditionals, this value should be qualified (e.g. ``cond|input`` instead of ``input`` if ``input`` is in a conditional with ``name="cond"``). - - - - - This copies the metadata information -from the tool's input dataset. This is particularly useful for interval data -types where the order of the columns is not set. - - - - - Relative path to a file produced by the -tool in its working directory. Output's contents are set to this file's -contents. - - - - - Boolean indicating whether to hide -dataset in the history view. (Default is ``false``.) - - + @@ -3887,6 +3939,7 @@ Creating collections in tools is covered in-depth in + Collection type for output (e.g. ``paired``, ``list``, or ``list:list``). @@ -3898,24 +3951,44 @@ Creating collections in tools is covered in-depth in derive collection's type (e.g. ``collection_type``) from. - - - This is the name of input collection or -dataset to derive "structure" of the output from (output element count and -identifiers). For instance, if the referenced input has three ordered items with -identifiers ``sample1``, ``sample2``, and ``sample3``. If this references input -elements in conditionals, this value should be qualified (e.g. ``cond|input`` instead -of ``input`` if ``input`` is in a conditional with ``name="cond"``). - - - - - If ``structured_like`` is set, inherit -format of outputs from format of corresponding input. - - + + + + + + + + + + + + + Output type. This could be older more established Galaxy types (e.g. data and collection) - in which case the semantics of this largely reflect the corresponding ``data`` and ``collection`` tags. This could also be newer non-data types such as ``integer`` or ``boolean``. + + + + + In expression tools, use this to specify a dictionary value to populate this output from. The semantics may change for other expression types in the future. + + + + + Collection type for output (e.g. ``paired``, ``list``, or ``list:list``). + + + + + This is the name of input collection to +derive collection's type (e.g. ``collection_type``) from. + + + + + diff --git a/lib/galaxy/workflow/run.py b/lib/galaxy/workflow/run.py index b75bee7ecac..961a673e9f4 100644 --- a/lib/galaxy/workflow/run.py +++ b/lib/galaxy/workflow/run.py @@ -1,3 +1,4 @@ +import json import logging import uuid @@ -367,6 +368,17 @@ class WorkflowProgress(object): delayed_why = "dependent collection [%s] not yet populated with datasets" % replacement.id raise modules.DelayedWorkflowEvaluation(why=delayed_why) + + is_hda = isinstance(replacement, model.HistoryDatasetAssociation) + if not is_data and is_hda: + if replacement.is_ok: + with open(replacement.file_name, 'r') as f: + replacement = json.load(f) + elif replacement.is_pending: + raise modules.DelayedWorkflowEvaluation() + else: + raise modules.CancelWorkflowEvaluation() + return replacement def get_replacement_workflow_output(self, workflow_output): diff --git a/lib/galaxy_ext/expressions/__init__.py b/lib/galaxy_ext/expressions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/galaxy_ext/expressions/handle_job.py b/lib/galaxy_ext/expressions/handle_job.py new file mode 100644 index 00000000000..4cfc3b52dc4 --- /dev/null +++ b/lib/galaxy_ext/expressions/handle_job.py @@ -0,0 +1,49 @@ +""" +Execute an external process to evaluate expressions for Galaxy jobs. + +Galaxy should be importable on sys.path . +""" + +import json +import logging +import os +import sys + +# insert *this* galaxy before all others on sys.path +sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))) + +# ensure supported version +assert sys.version_info[:2] >= (2, 7) and sys.version_info[:2] <= (2, 7), 'Python version must be 2.7, this is: %s' % sys.version + +logging.basicConfig() +log = logging.getLogger(__name__) + +from galaxy.tools.expressions import evaluate + +try: + from cwltool import expression +except ImportError: + expression = None + + +def run(environment_path=None): + if expression is None: + raise Exception("Python library cwltool must available to evaluate expressions.") + + if environment_path is None: + environment_path = os.environ.get("GALAXY_EXPRESSION_INPUTS") + with open(environment_path, "r") as f: + raw_inputs = json.load(f) + + outputs = raw_inputs["outputs"] + inputs = raw_inputs.copy() + del inputs["outputs"] + + result = evaluate(None, inputs) + + for output in outputs: + path = output["path"] + from_expression = "$(" + output["from_expression"] + ")" + output_value = expression.interpolate(from_expression, result) + with open(path, "w") as f: + json.dump(output_value, f) diff --git a/test/api/test_tools.py b/test/api/test_tools.py index d1c1e275a80..d2900e5a8d0 100644 --- a/test/api/test_tools.py +++ b/test/api/test_tools.py @@ -2030,6 +2030,48 @@ class ToolsTestCase(api.ApiTestCase): output_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output) self.assertEqual(output_content.strip(), "123\n456\n456\n0ab") + @skip_without_tool("expression_forty_two") + def test_galaxy_expression_tool_simplest(self): + history_id = self.dataset_populator.new_history() + inputs = { + } + run_response = self._run( + "expression_forty_two", history_id, inputs + ) + self._assert_status_code_is(run_response, 200) + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + output_content = self.dataset_populator.get_history_dataset_content(history_id) + self.assertEqual(output_content, "42") + + @skip_without_tool("expression_parse_int") + def test_galaxy_expression_tool_simple(self): + history_id = self.dataset_populator.new_history() + inputs = { + 'input1': '7', + } + run_response = self._run( + "expression_parse_int", history_id, inputs + ) + self._assert_status_code_is(run_response, 200) + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + output_content = self.dataset_populator.get_history_dataset_content(history_id) + self.assertEqual(output_content, "7") + + @skip_without_tool("expression_log_line_count") + def test_galaxy_expression_metadata(self): + history_id = self.dataset_populator.new_history() + new_dataset1 = self.dataset_populator.new_dataset(history_id, content='1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n11\n12\n13\n14') + inputs = { + 'input1': dataset_to_param(new_dataset1), + } + run_response = self._run( + "expression_log_line_count", history_id, inputs + ) + self._assert_status_code_is(run_response, 200) + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + output_content = self.dataset_populator.get_history_dataset_content(history_id) + self.assertEqual(output_content, "3") + def __build_group_list(self, history_id): response = self.dataset_collection_populator.upload_collection(history_id, "list", elements=[ { diff --git a/test/api/test_workflows.py b/test/api/test_workflows.py index 61f2c558be9..5ab72f3341c 100644 --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -1965,7 +1965,7 @@ text_input: type: raw """, history_id=history_id, wait=True, assert_ok=False) - def test_run_with_text_connection(self): + def test_run_with_text_input_connection(self): with self.dataset_populator.test_history() as history_id: self._run_jobs(""" class: GalaxyWorkflow @@ -1996,6 +1996,27 @@ text_input: content = self.dataset_populator.get_history_dataset_content(history_id) self.assertEqual("chrX\t152691446\t152691471\tCCDS14735.1_cds_0_0_chrX_152691447_f\t0\t+\n", content) + def test_run_with_numeric_input_connection(self): + history_id = self.dataset_populator.new_history() + self._run_jobs(""" +class: GalaxyWorkflow +steps: +- label: forty_two + tool_id: expression_forty_two + state: {} +- label: consume_expression_parameter + tool_id: cheetah_casting + state: + floattest: 3.14 + inttest: + $link: forty_two#out1 +test_data: {} +""", history_id=history_id) + + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + content = self.dataset_populator.get_history_dataset_content(history_id) + self.assertEquals("43\n4.14\n", content) + @skip_without_tool('cat1') def test_workflow_rerun_with_use_cached_job(self): workflow = self.workflow_populator.load_workflow(name="test_for_run") diff --git a/test/functional/tools/expression_forty_two.xml b/test/functional/tools/expression_forty_two.xml new file mode 100644 index 00000000000..c30553f85fa --- /dev/null +++ b/test/functional/tools/expression_forty_two.xml @@ -0,0 +1,15 @@ + + Parse Int + + {return {'output': + 42}; + } + + + + + + + Produces the integer 42. + diff --git a/test/functional/tools/expression_log_line_count.xml b/test/functional/tools/expression_log_line_count.xml new file mode 100644 index 00000000000..26804a97127 --- /dev/null +++ b/test/functional/tools/expression_log_line_count.xml @@ -0,0 +1,14 @@ + + Log Lines + + {return {'output': Math.max(Math.round(Math.log(parseInt($job.input1.metadata.data_lines))), 1)};} + + + + + + + + + diff --git a/test/functional/tools/expression_parse_int.xml b/test/functional/tools/expression_parse_int.xml new file mode 100644 index 00000000000..df2e9b98436 --- /dev/null +++ b/test/functional/tools/expression_parse_int.xml @@ -0,0 +1,14 @@ + + Parse Int + + {return {'output': parseInt($job.input1)};} + + + + + + + + Parse an integer from text. + diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index 2409bd0dd4b..77d4f11b78d 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -147,8 +147,10 @@ + + + - diff --git a/test/unit/jobs/test_expression_run.py b/test/unit/jobs/test_expression_run.py new file mode 100644 index 00000000000..71f07a0b97c --- /dev/null +++ b/test/unit/jobs/test_expression_run.py @@ -0,0 +1,48 @@ +import json +import os +import tempfile +import shutil +import subprocess + +from galaxy.tools import expressions + +THIS_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) +TEST_DIRECTORY = os.path.join(THIS_DIRECTORY, os.path.pardir, os.path.pardir) +ROOT_DIRECTORY = os.path.join(TEST_DIRECTORY, os.path.pardir) +LIB_DIRECTORY = os.path.join(ROOT_DIRECTORY, "lib") + + +def test_run_simple(): + test_directory = tempfile.mkdtemp() + try: + environment_path = os.path.join(test_directory, "env.json") + environment = { + 'job': {'input1': '7'}, + 'outputs': [ + {'name': 'out1', 'from_expression': "output1", 'path': 'moo'} + ], + 'script': "{return {'output1': parseInt($job.input1)};}", + } + with open(environment_path, "w") as f: + json.dump(environment, f) + expressions.write_evalute_script( + test_directory, + ) + new_env = os.environ.copy() + if "PYTHONPATH" in new_env: + new_env['PYTHONPATH'] = "%s:%s" % (LIB_DIRECTORY, new_env["PYTHONPATH"]) + else: + new_env['PYTHONPATH'] = "%s" % (LIB_DIRECTORY) + new_env['GALAXY_EXPRESSION_INPUTS'] = environment_path + p = subprocess.Popen( + args=expressions.EXPRESSION_SCRIPT_CALL, + shell=True, + cwd=test_directory, + env=new_env, + ) + assert p.wait() == 0 + with open(os.path.join(test_directory, 'moo')) as f: + out_content = f.read() + assert out_content == '7', out_content + finally: + shutil.rmtree(test_directory) diff --git a/test/unit/tools/test_parsing.py b/test/unit/tools/test_parsing.py index a87b9734e0d..e442cd5ca21 100644 --- a/test/unit/tools/test_parsing.py +++ b/test/unit/tools/test_parsing.py @@ -92,6 +92,39 @@ tests: compare: sim_size """ +TOOL_EXPRESSION_XML_1 = """ + + Parse Int + + {return {'output': parseInt($job.input1)};} + + + + + + + + Parse an integer from text. + +""" + + +TOOL_EXPRESSION_YAML_1 = """ +class: GalaxyExpressionTool +name: "parse_int" +id: parse_int +version: 1.0.2 +expression: "{return {'output': parseInt($job.input1)};}" +inputs: + - name: input1 + label: Text to parse + type: text +outputs: + out1: + type: integer + from: "#output" +""" + class BaseLoaderTestCase(unittest.TestCase): @@ -119,6 +152,22 @@ class BaseLoaderTestCase(unittest.TestCase): return tool_source +class XmlExpressionLoaderTestCase(BaseLoaderTestCase): + source_file_name = "expression.xml" + source_contents = TOOL_EXPRESSION_XML_1 + + def test_expression(self): + assert self._tool_source.parse_expression().strip() == "{return {'output': parseInt($job.input1)};}" + + def test_tool_type(self): + assert self._tool_source.parse_tool_type() == "expression" + + +class YamlExpressionLoaderTestCase(BaseLoaderTestCase): + source_file_name = "expression.yml" + source_contents = TOOL_EXPRESSION_XML_1 + + class XmlLoaderTestCase(BaseLoaderTestCase): source_file_name = "bwa.xml" source_contents = TOOL_XML_1 From 4659dd8bc63d55e94d13391fc048b7d31aea3981 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 19 Mar 2019 17:21:15 -0400 Subject: [PATCH 0560/1016] cwltool requirements --- .../dependencies/pipfiles/default/Pipfile | 1 + .../default/pinned-dev-requirements.txt | 2 +- .../pipfiles/default/pinned-requirements.txt | 20 ++++++++++++++++++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index b6ad3afea6d..abc4b2e76cd 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -40,6 +40,7 @@ uWSGI = "*" pysam = "==0.15.2" bdbag = "==1.4.1" # 1.5.0 requires Python >=2.7.9 bleach = "*" +cwltool = "==1.0.20180721142728" "bz2file" = {version = "*", markers = "python_version < '3.3'"} ipaddress = {version = "*", markers = "python_version < '3.3'"} isa-rwval = "*" diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt index 0dc921a00ac..fef1dbf7b42 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt @@ -23,7 +23,7 @@ more-itertools==5.0.0 ; python_version <= '2.7' nose==1.3.7 nosehtml==0.4.5 packaging==19.0 -pathlib2==2.3.3 ; python_version < '3.6' +pathlib2==2.3.2 ; python_version < '3' pathtools==0.1.2 pbr==5.1.3 pluggy==0.9.0 diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index a5da11c3af7..17d49415785 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -4,6 +4,7 @@ adal==1.2.1 amqp==2.4.2 appdirs==1.4.3 asn1crypto==0.24.0 +avro==1.8.1 ; python_version < '3' azure-common==1.1.14 azure-cosmosdb-nspkg==2.0.2 azure-cosmosdb-table==1.0.4 @@ -31,6 +32,7 @@ botocore==1.12.115 bunch==1.0.1 bx-python==0.8.2 bz2file==0.98 ; python_version < '3.3' +cachecontrol==0.11.7 cachetools==3.1.0 certifi==2019.3.9 cffi==1.12.2 @@ -42,6 +44,7 @@ cloudbridge==2.0.0 cmd2==0.8.9 contextlib2==0.5.5 ; python_version < '3.5' cryptography==2.6.1 +cwltool==1.0.20180721142728 debtcollector==1.21.0 decorator==4.3.2 deprecated==1.2.5 @@ -75,14 +78,18 @@ jsonpointer==2.0 jsonschema==2.6.0 keystoneauth1==3.13.1 kombu==4.4.0 +lockfile==0.12.2 +lxml==4.3.2 mako==1.0.7 markupsafe==1.1.1 mercurial==3.7.3 ; python_version < '3' +mistune==0.8.4 monotonic==1.5 msgpack==0.6.1 msrest==0.5.5 msrestazure==0.5.0 munch==2.3.2 +mypy-extensions==0.4.1 netaddr==0.7.19 netifaces==0.10.9 networkx==1.11 @@ -107,14 +114,17 @@ parsley==1.3 paste==3.0.8 pastedeploy==2.0.1 pastescript==3.1.0 +pathlib2==2.3.2 ; python_version < '3' pbr==5.1.3 prettytable==0.7.2 +prov==1.5.1 psutil==5.6.1 pulsar-galaxy-lib==0.8.3 pyasn1-modules==0.2.4 pyasn1==0.4.5 pycparser==2.19 pycryptodome==3.7.3 +pyeventsystem==0.1.0 pyinotify==0.9.6 ; sys_platform != 'win32' and sys_platform != 'darwin' and sys_platform != 'sunos5' pyjwt==1.7.1 pykwalify==1.7.0 @@ -136,6 +146,8 @@ python-openid==2.2.5 ; python_version < '3.0' python-swiftclient==3.6.0 pytz==2018.9 pyyaml==5.1 +rdflib-jsonld==0.4.0 +rdflib==4.2.2 repoze.lru==0.7 requests-oauthlib==1.2.0 requests-toolbelt==0.9.1 @@ -144,7 +156,12 @@ requestsexceptions==1.4.0 rfc3986==1.2.0 routes==2.4.1 rsa==4.0 +ruamel.ordereddict==0.4.13 ; platform_python_implementation == 'CPython' and python_version <= '2.7' +ruamel.yaml==0.15.89 s3transfer==0.2.0 +scandir==1.10.0 ; python_version < '3.5' +schema-salad==2.7.20181126142424 +shellescape==3.4.1 simplejson==3.16.0 six==1.11.0 social-auth-core[openidconnect]==3.1.0 @@ -157,7 +174,8 @@ subprocess32==3.5.3 ; python_version < '3.0' svgwrite==1.2.1 tempita==0.5.2 tenacity==4.12.0 -typing==3.6.6 ; python_version < '3.5' +typing-extensions==3.7.2 +typing==3.6.6 ; python_version < '3.6' tzlocal==1.5.1 unicodecsv==0.14.1 ; python_version < '3.0' uritemplate==3.0.0 From 5c80323dcb86bbcc952482584c23e232f93fd57d Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 17 Mar 2019 20:29:23 -0400 Subject: [PATCH 0561/1016] Rollback expression utilities for operation tools. Some extras in here for group and filter tool that never made it out of CWL branch and into Galaxy - clean those up for getting expression tools into Galaxy. --- lib/galaxy/tools/expressions/__init__.py | 6 +- lib/galaxy/tools/expressions/sandbox.py | 151 ----------------------- lib/galaxy/tools/expressions/util.py | 4 - 3 files changed, 1 insertion(+), 160 deletions(-) delete mode 100644 lib/galaxy/tools/expressions/sandbox.py diff --git a/lib/galaxy/tools/expressions/__init__.py b/lib/galaxy/tools/expressions/__init__.py index b731044825a..c44e4437b2a 100644 --- a/lib/galaxy/tools/expressions/__init__.py +++ b/lib/galaxy/tools/expressions/__init__.py @@ -1,6 +1,5 @@ from .evaluation import evaluate -from .sandbox import execjs, interpolate -from .util import jshead, find_engine +from .util import find_engine from .script import ( write_evalute_script, EXPRESSION_SCRIPT_CALL, @@ -10,11 +9,8 @@ from .script import ( __all__ = ( 'evaluate', - 'execjs', 'EXPRESSION_SCRIPT_CALL', 'EXPRESSION_SCRIPT_NAME', 'find_engine', - 'interpolate', - 'jshead', 'write_evalute_script', ) diff --git a/lib/galaxy/tools/expressions/sandbox.py b/lib/galaxy/tools/expressions/sandbox.py deleted file mode 100644 index a3646e100d4..00000000000 --- a/lib/galaxy/tools/expressions/sandbox.py +++ /dev/null @@ -1,151 +0,0 @@ -import subprocess -import json -import threading - -from .util import find_engine - - -class JavascriptException(Exception): - pass - - -def execjs(config, js, jslib): - application = find_engine(config) - try: - nodejs = subprocess.Popen([application], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - except OSError as e: - if e.errno == 2: - nodejs = subprocess.Popen(["docker", "run", - "--attach=STDIN", "--attach=STDOUT", "--attach=STDERR", - "--interactive", - "--rm", - "commonworkflowlanguage/nodejs-engine", "nodejs"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - else: - raise - - fn = "\"use strict\";%s\n(function()%s)()" % (jslib, js if isinstance(js, basestring) and len(js) > 1 and js[0] == '{' else ("{return (%s);}" % js)) - script = "console.log(JSON.stringify(require(\"vm\").runInNewContext(%s, {})));\n" % json.dumps(fn) - - def term(): - try: - nodejs.terminate() - except OSError: - pass - - # Time out after 5 seconds - tm = threading.Timer(5, term) - tm.start() - - stdoutdata, stderrdata = nodejs.communicate(script) - tm.cancel() - - if nodejs.returncode != 0: - raise JavascriptException("Returncode was: %s\nscript was: %s\nstdout was: '%s'\nstderr was: '%s'\n" % (nodejs.returncode, script, stdoutdata, stderrdata)) - else: - return json.loads(stdoutdata) - - -class SubstitutionError(Exception): - pass - - -DEFAULT = 0 -DOLLAR = 1 -PAREN = 2 -BRACE = 3 -SINGLE_QUOTE = 4 -DOUBLE_QUOTE = 5 -BACKSLASH = 6 - - -def scanner(scan): - - i = 0 - stack = [DEFAULT] - start = 0 - while i < len(scan): - state = stack[-1] - c = scan[i] - - if state == DEFAULT: - if c == '$': - stack.append(DOLLAR) - elif c == '\\': - stack.append(BACKSLASH) - elif state == BACKSLASH: - stack.pop() - if stack[-1] == DEFAULT: - return [i - 1, i + 1] - elif state == DOLLAR: - if c == '(': - start = i - 1 - stack.append(PAREN) - elif c == '{': - start = i - 1 - stack.append(BRACE) - elif state == PAREN: - if c == '(': - stack.append(PAREN) - elif c == ')': - stack.pop() - if stack[-1] == DOLLAR: - return [start, i + 1] - elif c == "'": - stack.append(SINGLE_QUOTE) - elif c == '"': - stack.append(DOUBLE_QUOTE) - elif state == BRACE: - if c == '{': - stack.append(BRACE) - elif c == '}': - stack.pop() - if stack[-1] == DOLLAR: - return [start, i + 1] - elif c == "'": - stack.append(SINGLE_QUOTE) - elif c == '"': - stack.append(DOUBLE_QUOTE) - elif state == SINGLE_QUOTE: - if c == "'": - stack.pop() - elif c == '\\': - stack.append(BACKSLASH) - elif state == DOUBLE_QUOTE: - if c == '"': - stack.pop() - elif c == '\\': - stack.append(BACKSLASH) - i += 1 - - if len(stack) > 1: - raise SubstitutionError("Substitution error, unfinished block starting at position {}: {}".format(start, scan[start:])) - else: - return None - - -def interpolate(scan, jslib): - scan = scan.strip() - parts = [] - w = scanner(scan) - while w: - parts.append(scan[0:w[0]]) - - if scan[w[0]] == '$': - e = execjs(scan[w[0] + 1:w[1]], jslib) - if w[0] == 0 and w[1] == len(scan): - return e - leaf = json.dumps(e, sort_keys=True) - if leaf[0] == '"': - leaf = leaf[1:-1] - parts.append(leaf) - elif scan[w[0]] == '\\': - e = scan[w[1] - 1] - parts.append(e) - - scan = scan[w[1]:] - w = scanner(scan) - parts.append(scan) - return ''.join(parts) diff --git a/lib/galaxy/tools/expressions/util.py b/lib/galaxy/tools/expressions/util.py index b7ea1bf090a..83f954a2ba3 100644 --- a/lib/galaxy/tools/expressions/util.py +++ b/lib/galaxy/tools/expressions/util.py @@ -7,7 +7,3 @@ def find_engine(config): if nodejs_path is None: nodejs_path = which("nodejs") or which("node") or None return nodejs_path - - -def jshead(engine_config, root_vars): - return "\n".join(engine_config + ["var %s = %s;" % (k, json.dumps(v)) for k, v in root_vars.items()]) From dc48dffb29dc8fbd50a147d592c4d1afe4322cba Mon Sep 17 00:00:00 2001 From: guerler Date: Tue, 19 Mar 2019 18:09:22 -0400 Subject: [PATCH 0562/1016] Fix spacing for info text in data selectors --- client/galaxy/scripts/mvc/form/form-input.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/galaxy/scripts/mvc/form/form-input.js b/client/galaxy/scripts/mvc/form/form-input.js index aa92a5b84a5..245d19287ed 100644 --- a/client/galaxy/scripts/mvc/form/form-input.js +++ b/client/galaxy/scripts/mvc/form/form-input.js @@ -201,7 +201,7 @@ export default Backbone.View.extend({ .append( $("
") .addClass("ui-form-field") - .append($("").addClass("ui-form-info form-text text-muted")) + .append($("").addClass("ui-form-info form-text text-muted mt-2")) .append($("
").addClass("ui-form-backdrop")) ) .append($("
").addClass("ui-form-preview")); From c2e67e3270408dfe85c512d3f25f8506c16d61e7 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 17 Mar 2019 20:30:40 -0400 Subject: [PATCH 0563/1016] flake imports --- lib/galaxy/tools/__init__.py | 2 +- lib/galaxy/tools/expressions/__init__.py | 4 ++-- lib/galaxy/tools/expressions/util.py | 1 - lib/galaxy_ext/expressions/handle_job.py | 14 +++++--------- test/unit/jobs/test_expression_run.py | 2 +- 5 files changed, 9 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index f63f33a94b1..60d6289f937 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -29,6 +29,7 @@ from galaxy.managers.jobs import JobSearch from galaxy.metadata import get_metadata_compute_strategy from galaxy.model.tags import GalaxyTagHandler from galaxy.queue_worker import send_control_task +from galaxy.tools import expressions from galaxy.tools.actions import DefaultToolAction from galaxy.tools.actions.data_manager import DataManagerToolAction from galaxy.tools.actions.data_source import DataSourceToolAction @@ -36,7 +37,6 @@ from galaxy.tools.actions.model_operations import ModelOperationToolAction from galaxy.tools.deps import ( CachedDependencyManager, ) -from galaxy.tools import expressions from galaxy.tools.fetcher import ToolLocationFetcher from galaxy.tools.parameters import ( check_param, diff --git a/lib/galaxy/tools/expressions/__init__.py b/lib/galaxy/tools/expressions/__init__.py index c44e4437b2a..bf20411a392 100644 --- a/lib/galaxy/tools/expressions/__init__.py +++ b/lib/galaxy/tools/expressions/__init__.py @@ -1,10 +1,10 @@ from .evaluation import evaluate -from .util import find_engine from .script import ( - write_evalute_script, EXPRESSION_SCRIPT_CALL, EXPRESSION_SCRIPT_NAME, + write_evalute_script, ) +from .util import find_engine __all__ = ( diff --git a/lib/galaxy/tools/expressions/util.py b/lib/galaxy/tools/expressions/util.py index 83f954a2ba3..93531186fc9 100644 --- a/lib/galaxy/tools/expressions/util.py +++ b/lib/galaxy/tools/expressions/util.py @@ -1,4 +1,3 @@ -import json from galaxy.tools.deps.commands import which diff --git a/lib/galaxy_ext/expressions/handle_job.py b/lib/galaxy_ext/expressions/handle_job.py index 4cfc3b52dc4..29fc11f83bc 100644 --- a/lib/galaxy_ext/expressions/handle_job.py +++ b/lib/galaxy_ext/expressions/handle_job.py @@ -3,7 +3,6 @@ Execute an external process to evaluate expressions for Galaxy jobs. Galaxy should be importable on sys.path . """ - import json import logging import os @@ -12,19 +11,16 @@ import sys # insert *this* galaxy before all others on sys.path sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))) -# ensure supported version -assert sys.version_info[:2] >= (2, 7) and sys.version_info[:2] <= (2, 7), 'Python version must be 2.7, this is: %s' % sys.version - -logging.basicConfig() -log = logging.getLogger(__name__) - -from galaxy.tools.expressions import evaluate - try: from cwltool import expression except ImportError: expression = None +from galaxy.tools.expressions import evaluate + +logging.basicConfig() +log = logging.getLogger(__name__) + def run(environment_path=None): if expression is None: diff --git a/test/unit/jobs/test_expression_run.py b/test/unit/jobs/test_expression_run.py index 71f07a0b97c..c23455d6e07 100644 --- a/test/unit/jobs/test_expression_run.py +++ b/test/unit/jobs/test_expression_run.py @@ -1,8 +1,8 @@ import json import os -import tempfile import shutil import subprocess +import tempfile from galaxy.tools import expressions From f579066655c7ce3bd7198446f97e3214a08228b2 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 19 Mar 2019 17:16:35 -0400 Subject: [PATCH 0564/1016] Python 3 fixes for galaxy.tools.expressions.evaluation. --- lib/galaxy/tools/expressions/evaluation.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tools/expressions/evaluation.py b/lib/galaxy/tools/expressions/evaluation.py index 89becc1f862..54066d5094b 100644 --- a/lib/galaxy/tools/expressions/evaluation.py +++ b/lib/galaxy/tools/expressions/evaluation.py @@ -27,11 +27,12 @@ def evaluate(config, input): close_fds=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) - - (stdoutdata, stderrdata) = sp.communicate(json.dumps(new_input) + "\n\n") + input_str = json.dumps(new_input) + "\n\n" + input_bytes = input_str.encode("utf-8") + (stdoutdata, stderrdata) = sp.communicate(input_bytes) if sp.returncode != 0: args = (json.dumps(new_input, indent=4), stdoutdata, stderrdata) message = "Expression engine returned non-zero exit code on evaluation of\n%s%s%s" % args raise Exception(message) - return json.loads(stdoutdata) + return json.loads(stdoutdata.decode("utf-8")) From c4332f4e68480c57472949303daacdcceddb698a Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 19 Mar 2019 17:17:29 -0400 Subject: [PATCH 0565/1016] Error message clean up thanks to @mvdbeek. --- lib/galaxy/tools/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 60d6289f937..ff24a451eae 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -105,10 +105,7 @@ from .provided_metadata import parse_tool_provided_metadata log = logging.getLogger(__name__) REQUIRES_JS_RUNTIME_MESSAGE = ("The tool [%s] requires a nodejs runtime to execute " - "but node nor nodejs could be found on Galaxy's PATH and " - "no runtime was configured using the nodejs_path option in " - "galaxy.ini.") - + "but node or nodejs could not be found. Please contact the Galaxy adminstrator") HELP_UNINITIALIZED = threading.Lock() MODEL_TOOLS_PATH = os.path.abspath(os.path.dirname(__file__)) From f427a74dd97f28a93f036a8c1ef963fa18e8618c Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 19 Mar 2019 17:19:52 -0400 Subject: [PATCH 0566/1016] More changes to expression tool commit thanks to @mvdbeek. --- lib/galaxy/tools/parameters/wrapped_json.py | 3 ++- lib/galaxy_ext/expressions/handle_job.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/parameters/wrapped_json.py b/lib/galaxy/tools/parameters/wrapped_json.py index 69549943143..f9c90d4abe1 100644 --- a/lib/galaxy/tools/parameters/wrapped_json.py +++ b/lib/galaxy/tools/parameters/wrapped_json.py @@ -69,7 +69,8 @@ def _json_wrap_input(input, value, handle_files="skip"): return _hda_to_object(value) else: return None - raise NotImplementedError() + else: + raise NotImplementedError() elif input_type == "data_collection": if handle_files == "skip": return SKIP_INPUT diff --git a/lib/galaxy_ext/expressions/handle_job.py b/lib/galaxy_ext/expressions/handle_job.py index 29fc11f83bc..680e16fad20 100644 --- a/lib/galaxy_ext/expressions/handle_job.py +++ b/lib/galaxy_ext/expressions/handle_job.py @@ -24,7 +24,7 @@ log = logging.getLogger(__name__) def run(environment_path=None): if expression is None: - raise Exception("Python library cwltool must available to evaluate expressions.") + raise Exception("Python library cwltool must be available to evaluate expressions.") if environment_path is None: environment_path = os.environ.get("GALAXY_EXPRESSION_INPUTS") From 3c4f0f33687c3651e0d6728fa5036f64caa3ae46 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 20 Mar 2019 00:04:20 +0100 Subject: [PATCH 0567/1016] Update lib/galaxy/datatypes/microarrays.py Co-Authored-By: bensellak --- lib/galaxy/datatypes/microarrays.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/microarrays.py b/lib/galaxy/datatypes/microarrays.py index b70a04c0995..a3ab1d683a0 100644 --- a/lib/galaxy/datatypes/microarrays.py +++ b/lib/galaxy/datatypes/microarrays.py @@ -163,7 +163,7 @@ class Cel(Binary): header_bytes = handle.read(5) if header_bytes.decode("utf8", errors="ignore")[0] == '@': found_cel_4 = True - elif header_bytes == b';\x01\x00\x00': + elif struct.unpack(" Date: Wed, 20 Mar 2019 00:05:38 +0100 Subject: [PATCH 0568/1016] Add files via upload --- lib/galaxy/datatypes/test/affy_v_4.cel | Bin 0 -> 1079 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 lib/galaxy/datatypes/test/affy_v_4.cel diff --git a/lib/galaxy/datatypes/test/affy_v_4.cel b/lib/galaxy/datatypes/test/affy_v_4.cel new file mode 100644 index 0000000000000000000000000000000000000000..9dc05e342d5a71e71a948739973017b6cbff36d1 GIT binary patch literal 1079 zcmZwGF>~556aZjfuN}&ip-VSyop2-@*{ON$t`lR% zj2Rg-X3SRlPx2G`6Z#}Wn%-51#OEjRWciu8VHmHWzuENn@csLhVSE-mYNjG#qSp&3 zsUlK!oNDPXm0>&yEb8FGCxsjFG!$Eb;c4PICidHW!6KDo#D6gFa5!%H z?L3a#ER?xm$2^iV&wUx{Fi0*T)}1D)V4EGYZZewj3#m@ywHv(=q$yIg7E*-gQthX+ z57IQ&X<>pTnVulNiWZrRuv$+1amGgZw<(lsn{*f^Q96fP9l(9Z+h|LORD23Eo_Qk~ zUG-U*_<(*MOMPk#&EL=GdyT{Ad<5#%Sn#aOlPe~qD%D;nJ1mTt2)lu;jnSrACPtX<$vuh1eYJ7|5P&S(AP?JP+e;Y zT57F8E3GwXt@Q+Y(lQ!`Vf<3R-2v@rArR8SAgomf)wKqofz}!5OzRSKskH2OBdZ{{R30 literal 0 HcmV?d00001 From f3c0f429553d90f76afb16ba443e1c6e5f52c245 Mon Sep 17 00:00:00 2001 From: bensellak Date: Wed, 20 Mar 2019 00:08:53 +0100 Subject: [PATCH 0569/1016] Update microarrays.py --- lib/galaxy/datatypes/microarrays.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/microarrays.py b/lib/galaxy/datatypes/microarrays.py index a3ab1d683a0..358bdd82149 100644 --- a/lib/galaxy/datatypes/microarrays.py +++ b/lib/galaxy/datatypes/microarrays.py @@ -149,7 +149,7 @@ class Cel(Binary): >>> fname = get_test_fname('affy_v_3.cel') >>> Cel().sniff(fname) True - >>> fname = get_test_fname('test.cel') + >>> fname = get_test_fname('affy_v_4.cel') >>> Cel().sniff(fname) True >>> fname = get_test_fname('test.gal') From 8a6dc008b090ecc783801a7f80099c3f8b2006c3 Mon Sep 17 00:00:00 2001 From: bensellak Date: Wed, 20 Mar 2019 00:18:56 +0100 Subject: [PATCH 0570/1016] Update microarrays.py --- lib/galaxy/datatypes/microarrays.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/datatypes/microarrays.py b/lib/galaxy/datatypes/microarrays.py index 358bdd82149..581058f6332 100644 --- a/lib/galaxy/datatypes/microarrays.py +++ b/lib/galaxy/datatypes/microarrays.py @@ -1,5 +1,6 @@ import logging +import struct from galaxy.datatypes import data from galaxy.datatypes.binary import Binary from galaxy.datatypes.data import get_file_peek From 75ae51e9f90e5d27dcad34a100d5693ef818ead9 Mon Sep 17 00:00:00 2001 From: bensellak Date: Tue, 19 Mar 2019 23:25:01 +0000 Subject: [PATCH 0571/1016] Update microarrays.py --- lib/galaxy/datatypes/microarrays.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/datatypes/microarrays.py b/lib/galaxy/datatypes/microarrays.py index 581058f6332..eb392973b78 100644 --- a/lib/galaxy/datatypes/microarrays.py +++ b/lib/galaxy/datatypes/microarrays.py @@ -157,11 +157,11 @@ class Cel(Binary): >>> Cel().sniff(fname) False """ - handle = open(filename, 'rb') + with open(filename, 'rb') as handle: + header_bytes = handle.read(5) found_cel_4 = False found_cel_3 = False found_cel_agcc = False - header_bytes = handle.read(5) if header_bytes.decode("utf8", errors="ignore")[0] == '@': found_cel_4 = True elif struct.unpack(" Date: Tue, 19 Mar 2019 23:41:11 +0000 Subject: [PATCH 0572/1016] Update microarrays.py --- lib/galaxy/datatypes/microarrays.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/microarrays.py b/lib/galaxy/datatypes/microarrays.py index eb392973b78..ecf47469516 100644 --- a/lib/galaxy/datatypes/microarrays.py +++ b/lib/galaxy/datatypes/microarrays.py @@ -1,6 +1,6 @@ import logging - import struct + from galaxy.datatypes import data from galaxy.datatypes.binary import Binary from galaxy.datatypes.data import get_file_peek From 75ed37d03da286614b3578c22b17c87ffddd2e8e Mon Sep 17 00:00:00 2001 From: bensellak Date: Wed, 20 Mar 2019 00:32:02 +0000 Subject: [PATCH 0573/1016] Update microarrays.py --- lib/galaxy/datatypes/microarrays.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/microarrays.py b/lib/galaxy/datatypes/microarrays.py index ecf47469516..16b13828ee1 100644 --- a/lib/galaxy/datatypes/microarrays.py +++ b/lib/galaxy/datatypes/microarrays.py @@ -175,7 +175,7 @@ class Cel(Binary): Set metadata for Cel file. """ with open(dataset.file_name, 'rb') as handle: - header_bytes = handle.read(5) + header_bytes = handle.read(5) if header_bytes.decode("utf8", errors="ignore")[0] == '@': dataset.metadata.version = "4" elif header_bytes.decode("utf8", errors="ignore")[0] == ';': From 98738d80aa259c0158310a6bbcabec7311e2afb1 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 20 Mar 2019 10:02:18 +0100 Subject: [PATCH 0574/1016] Raise exception when node is not on PATH --- lib/galaxy/tools/__init__.py | 6 ++++-- lib/galaxy/tools/expressions/util.py | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index ff24a451eae..e5ee2c7dc72 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -753,8 +753,10 @@ class Tool(Dictifiable): mod = __import__(module, globals(), locals(), [cls]) self.tool_action = getattr(mod, cls)() if getattr(self.tool_action, "requires_js_runtime", False): - if expressions.find_engine(self.app.config) is None: - message = REQUIRES_JS_RUNTIME_MESSAGE % self.tool_id + try: + expressions.find_engine(self.app.config) + except Exception: + message = REQUIRES_JS_RUNTIME_MESSAGE % self.tool_id or self.tool_uuid raise Exception(message) # Tests self.__parse_tests(tool_source) diff --git a/lib/galaxy/tools/expressions/util.py b/lib/galaxy/tools/expressions/util.py index 93531186fc9..082ed3c916f 100644 --- a/lib/galaxy/tools/expressions/util.py +++ b/lib/galaxy/tools/expressions/util.py @@ -5,4 +5,6 @@ def find_engine(config): nodejs_path = getattr(config, "nodejs_path", None) if nodejs_path is None: nodejs_path = which("nodejs") or which("node") or None + if nodejs_path is None: + raise Exception("nodejs or node not found on PATH") return nodejs_path From 5e778f4812290cf9c9827b606dcd32fa2f6e2e5a Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 20 Mar 2019 10:02:56 +0100 Subject: [PATCH 0575/1016] Install node as part of common_startup.sh --- scripts/common_startup.sh | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index f224ed910b1..e5dc5d0935c 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -224,14 +224,23 @@ if [ $SKIP_CLIENT_BUILD -eq 0 ]; then fi fi +# Install node if not installed +if [ -n "$VIRTUAL_ENV" ]; then + if ! in_venv "$(command -v node)" || [ "$(node --version)" != "v$NODE_VERSION" ]; then + echo "Installing node into $VIRTUAL_ENV with nodeenv." + nodeenv -n $NODE_VERSION -p + fi +elif [ -n "$CONDA_DEFAULT_ENV" -a -n "$CONDA_EXE" ]; then + if ! in_conda_env "$(command -v node)"; then + echo "Installing node into '$CONDA_DEFAULT_ENV' Conda environment with conda." + $CONDA_EXE install --yes --override-channels --channel conda-forge --channel defaults --name $CONDA_DEFAULT_ENV node=$NODE_VERSION + fi +fi + # Build client if necessary. if [ $SKIP_CLIENT_BUILD -eq 0 ]; then # Ensure dependencies are installed if [ -n "$VIRTUAL_ENV" ]; then - if ! in_venv "$(command -v node)" || [ "$(node --version)" != "v$NODE_VERSION" ]; then - echo "Installing node into $VIRTUAL_ENV with nodeenv." - nodeenv -n $NODE_VERSION -p - fi if ! in_venv "$(command -v yarn)"; then echo "Installing yarn into $VIRTUAL_ENV with npm." npm install --global yarn From 552942fca9f4b6e74d74a1593a28ec274a24bdfe Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 20 Mar 2019 11:12:51 +0100 Subject: [PATCH 0576/1016] Fix enumerating over output There's no iteritems in python 3. We could use six.iteritems which calls iteritems on py2 and items on py3, but I don't expect these dicts to be large enough that there would be any noticeable difference. --- lib/galaxy/tools/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index e5ee2c7dc72..aaccb3604fa 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2217,7 +2217,7 @@ class ExpressionTool(Tool): expression_inputs_path = os.path.join(local_working_directory, ExpressionTool.EXPRESSION_INPUTS_NAME) outputs = [] - for i, (out_name, data) in enumerate(out_data.iteritems()): + for i, (out_name, data) in enumerate(out_data.items()): output_def = self.outputs[out_name] wrapped_data = param_dict.get(out_name) file_name = str(wrapped_data) From 5e8cbcd5b4e39809fe0f83547ecf062d6d01c6eb Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 20 Mar 2019 11:14:56 +0100 Subject: [PATCH 0577/1016] Allow connecting ExpressionTool outputs to inputs of correct type This means we can now connect these tools in the workflow editor. --- lib/galaxy/workflow/modules.py | 4 ++++ test/unit/workflows/test_modules.py | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 06e02a0678b..ba5edbd20ea 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -45,6 +45,7 @@ from galaxy.tools.parameters.basic import ( ) from galaxy.tools.parameters.history_query import HistoryQuery from galaxy.tools.parameters.wrapped import make_dict_copy +from galaxy.tools.parser.output_objects import ToolExpressionOutput from galaxy.util.bunch import Bunch from galaxy.util.json import safe_loads from galaxy.util.odict import odict @@ -937,6 +938,8 @@ class ToolModule(WorkflowModule): if filter_output(tool_output, self.state.inputs): continue extra_kwds = {} + if isinstance(tool_output, ToolExpressionOutput): + extra_kwds['parameter'] = True if tool_output.collection: extra_kwds["collection"] = True collection_type = tool_output.structure.collection_type @@ -973,6 +976,7 @@ class ToolModule(WorkflowModule): dict( name=name, extensions=formats, + type=tool_output.output_type, **extra_kwds ) ) diff --git a/test/unit/workflows/test_modules.py b/test/unit/workflows/test_modules.py index 921596b8d9f..4ad43a127f2 100644 --- a/test/unit/workflows/test_modules.py +++ b/test/unit/workflows/test_modules.py @@ -452,7 +452,8 @@ def __mock_tool( format_source=None, change_format=[], filters=[], - label=None)}, + label=None, + output_type='data')}, params_from_strings=mock.Mock(), check_and_update_param_values=mock.Mock(), to_json=_to_json From c80b80d3cf96b0137e172e460f63326e34ef8591 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 20 Mar 2019 12:25:14 +0100 Subject: [PATCH 0578/1016] Fix test_workflow_resume_with_mapped_over_input test self.dataset_populator._get_contents_request(history_id=history_id).json() returns the dataset collection as the first element. We do need a HDA here though, so we'd just randomly pick a HDA with the HDCA ID. So if that dataset was deleted by chance the test would fail. Ouch. --- test/api/test_workflows.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/api/test_workflows.py b/test/api/test_workflows.py index 5ab72f3341c..e35e72aba2d 100644 --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -791,12 +791,14 @@ test_data: """, history_id=history_id, assert_ok=False, wait=False) self.wait_for_invocation_and_jobs(history_id, job_summary.workflow_id, job_summary.invocation_id, assert_ok=False) history_contents = self.dataset_populator._get_contents_request(history_id=history_id).json() + first_input = history_contents[1] + assert first_input['history_content_type'] == 'dataset' paused_dataset = history_contents[-1] failed_dataset = self.dataset_populator.get_history_dataset_details(history_id, hid=5, assert_ok=False) assert paused_dataset['state'] == 'paused', paused_dataset assert failed_dataset['state'] == 'error', failed_dataset inputs = {"input1": {'values': [{'src': 'hda', - 'id': history_contents[0]['id']}] + 'id': first_input['id']}] }, "failbool": "false", "rerun_remap_job_id": failed_dataset['creating_job']} From a8fcd5809eb76a3781df6e0a0bd2558883240a27 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 20 Mar 2019 12:59:03 -0400 Subject: [PATCH 0579/1016] Move styling used (only) in the single StatelessTags component into the SFC. --- .../components/Tags/StatelessTags.styles.scss | 69 ------------------ .../scripts/components/Tags/StatelessTags.vue | 72 ++++++++++++++++++- 2 files changed, 71 insertions(+), 70 deletions(-) delete mode 100644 client/galaxy/scripts/components/Tags/StatelessTags.styles.scss diff --git a/client/galaxy/scripts/components/Tags/StatelessTags.styles.scss b/client/galaxy/scripts/components/Tags/StatelessTags.styles.scss deleted file mode 100644 index 9f2300bbdb7..00000000000 --- a/client/galaxy/scripts/components/Tags/StatelessTags.styles.scss +++ /dev/null @@ -1,69 +0,0 @@ -// Most styling of the tags should happen in here. - -@import "theme/blue"; -@import "scss/mixins"; - -// Puts a little graphic in place of the text-input -// when the input is not in focus -@mixin newTagHoverButton() { - .vue-tags-input .ti-tags .ti-new-tag-input-wrapper { - input { - background-color: transparent; - } - input:not(:focus) { - background: url("/static/images/fugue/tag--plus.png"); - background-repeat: no-repeat; - color: transparent; - &::placeholder { - color: transparent; - } - } - } -} - -// general style butchering -@mixin matchBootstrapStyling() { - // TODO: actually match the bootstrap button classes in - // here either by importing mixins or just using colors - // from the boostrap .scss files - .vue-tags-input { - @include fill(); - .ti-input { - padding: 0; - border: none; - } - .ti-tag { - border-radius: 4px; - font-size: 0.8rem; - font-weight: 400; - } - &.tag-area { - background-color: transparent; - } - } -} - -// Version of the tags that only allow clicking -// existing tags instead of the full editing UI -@mixin forDisplayOnly() { - .vue-tags-input { - .ti-actions, - .ti-new-tag-input-wrapper { - display: none; - } - } -} - -.tags-display { - - // adds in a graphic in place of the text input - @include newTagHoverButton(); - - // match bootstrap tag styles/colors - @include matchBootstrapStyling(); - - // display-only tags - &.disabled { - @include forDisplayOnly(); - } -} diff --git a/client/galaxy/scripts/components/Tags/StatelessTags.vue b/client/galaxy/scripts/components/Tags/StatelessTags.vue index bb48b15a713..b09394c3eda 100644 --- a/client/galaxy/scripts/components/Tags/StatelessTags.vue +++ b/client/galaxy/scripts/components/Tags/StatelessTags.vue @@ -119,4 +119,74 @@ export default { - + From ee469a929b0647ee3473573a16bc374cd1a6b158 Mon Sep 17 00:00:00 2001 From: guerler Date: Wed, 20 Mar 2019 15:35:52 -0400 Subject: [PATCH 0580/1016] Adjust white space handling in data selectors --- client/galaxy/scripts/mvc/ui/ui-select-content.js | 1 - client/galaxy/style/scss/ui.scss | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/client/galaxy/scripts/mvc/ui/ui-select-content.js b/client/galaxy/scripts/mvc/ui/ui-select-content.js index 581bb2a53e3..c581485474b 100644 --- a/client/galaxy/scripts/mvc/ui/ui-select-content.js +++ b/client/galaxy/scripts/mvc/ui/ui-select-content.js @@ -351,7 +351,6 @@ var View = Backbone.View.extend({ let $left = $("
"); let $right = $("
").addClass("w-100"); this.$el.empty() - .append("
") .addClass("d-flex flex-row") .append($left) .append($right); diff --git a/client/galaxy/style/scss/ui.scss b/client/galaxy/style/scss/ui.scss index 243dee8611d..5d19dd67279 100644 --- a/client/galaxy/style/scss/ui.scss +++ b/client/galaxy/style/scss/ui.scss @@ -421,6 +421,9 @@ $ui-margin-horizontal-large: $margin-v * 2; -webkit-appearance: none; -moz-border-radius: $border-radius-base; line-height: 1.5rem; + .select2-chosen { + white-space: normal; + } .select2-arrow { display: none; } From 23e6bb715f7447167cfe2f6f1c1a177b85881d1f Mon Sep 17 00:00:00 2001 From: guerler Date: Wed, 20 Mar 2019 16:16:55 -0400 Subject: [PATCH 0581/1016] Fix visibility handling in options, right align browser button --- client/galaxy/scripts/mvc/ui/ui-options.js | 14 ++++++++- .../scripts/mvc/ui/ui-select-content.js | 30 +++++++++---------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/client/galaxy/scripts/mvc/ui/ui-options.js b/client/galaxy/scripts/mvc/ui/ui-options.js index 76a49bf6b8d..03fe661287a 100644 --- a/client/galaxy/scripts/mvc/ui/ui-options.js +++ b/client/galaxy/scripts/mvc/ui/ui-options.js @@ -12,6 +12,7 @@ var Base = Backbone.View.extend({ (options && options.model) || new Backbone.Model({ visible: true, + cls: null, data: [], id: Utils.uid(), error_text: "No options available.", @@ -36,6 +37,7 @@ var Base = Backbone.View.extend({ .empty() .removeClass() .addClass("ui-options") + .addClass(this.model.get("cls")) .append((this.$message = $("
").addClass("mt-2"))) .append((this.$menu = $("
").addClass("ui-options-menu"))) .append((this.$options = $(this._template()))); @@ -154,6 +156,16 @@ var Base = Backbone.View.extend({ return this.$(".ui-option").length; }, + /** Shows the options */ + show: function() { + this.model.set("visible", true); + }, + + /** Hides the options */ + hide: function() { + this.model.set("visible", false); + }, + /** Set value to dom */ _setValue: function(new_value) { var self = this; @@ -260,7 +272,7 @@ RadioButton.View = Base.extend({ /** Template for a single option */ _templateOption: function(pair) { - var $el = $("