From ffdf42f9d21b3da3699c58b038810e1ec246da54 Mon Sep 17 00:00:00 2001 From: vahid Date: Sat, 8 Jun 2019 23:46:30 -0700 Subject: [PATCH 01/79] Add CloudAuthz config file to config_schema.yml. --- config/galaxy.yml.sample | 5 +++++ doc/source/admin/galaxy_options.rst | 12 ++++++++++++ lib/galaxy/webapps/galaxy/config_schema.yml | 10 ++++++++++ 3 files changed, 27 insertions(+) diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample index ff5c0b71f46..104f7d31469 100644 --- a/config/galaxy.yml.sample +++ b/config/galaxy.yml.sample @@ -1421,6 +1421,11 @@ galaxy: # Sets the path to OIDC backends configuration file. #oidc_backends_config_file: config/oidc_backends_config.xml + # Sets the path to a file containing setters necessary for + # configuration and/or initialization of cloud authorization + # (cloudauthz) backends (e.g., Google Cloud Platform). + #cloudauthz_backends_config_file: config/cloudauthz_backends_config.xml + # XML config file that allows the use of different authentication # providers (e.g. LDAP) instead or in addition to local authentication # (.sample is used if default does not exist). diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index ce904ed90b8..8b5b3a622e7 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -2923,6 +2923,18 @@ :Type: str +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``cloudauthz_backends_config_file`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Sets the path to a file containing setters necessary for + configuration and/or initialization of cloud authorization + (cloudauthz) backends (e.g., Google Cloud Platform). +:Default: ``config/cloudauthz_backends_config.xml`` +:Type: str + + ~~~~~~~~~~~~~~~~~~~~ ``auth_config_file`` ~~~~~~~~~~~~~~~~~~~~ diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 91b3482b4e1..327a30eda16 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -2171,6 +2171,16 @@ mapping: desc: | Sets the path to OIDC backends configuration file. + cloudauthz_backends_config_file: + type: str + default: config/cloudauthz_backends_config.xml + required: false + desc: | + Sets the path to a file containing setters necessary + for configuration and/or initialization of + cloud authorization (cloudauthz) backends (e.g., + Google Cloud Platform). + auth_config_file: type: str default: config/auth_conf.xml From 2036ac35e8979584918479820ee401e2dbf02910 Mon Sep 17 00:00:00 2001 From: vahid Date: Sun, 16 Jun 2019 16:47:35 -0700 Subject: [PATCH 02/79] Pass cloudauthz backend config to authnz manager and parse it. --- lib/galaxy/app.py | 5 ++++- lib/galaxy/authnz/managers.py | 35 ++++++++++++++++++++++++++++++++++- lib/galaxy/config.py | 2 ++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index e771ce70ba8..1c7136c68d7 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -183,7 +183,10 @@ class UniverseApplication(config.ConfiguresGalaxyMixin): if self.config.enable_oidc: from galaxy.authnz import managers - self.authnz_manager = managers.AuthnzManager(self, self.config.oidc_config, self.config.oidc_backends_config) + self.authnz_manager = managers.AuthnzManager(self, + self.config.oidc_config, + self.config.oidc_backends_config, + self.config.cloudauthz_backends_config) self.sentry_client = None if self.config.sentry_dsn: diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 96fcd9b52e3..f85d3459522 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -31,7 +31,7 @@ log = logging.getLogger(__name__) class AuthnzManager(object): - def __init__(self, app, oidc_config_file, oidc_backends_config_file): + def __init__(self, app, oidc_config_file, oidc_backends_config_file, cloudauthz_backends_config_file): """ :type app: galaxy.app.UniverseApplication :param app: @@ -43,6 +43,7 @@ class AuthnzManager(object): self.app = app self._parse_oidc_config(oidc_config_file) self._parse_oidc_backends_config(oidc_backends_config_file) + self._parse_cloudauthz_backends_config(cloudauthz_backends_config_file) def _parse_oidc_config(self, config_file): self.oidc_config = {} @@ -106,6 +107,38 @@ class AuthnzManager(object): except ParseError as e: raise ParseError("Invalid configuration at `{}`: {} -- unable to continue.".format(config_file, e)) + def _parse_cloudauthz_backends_config(self, config_file): + self.cloudauthz_backends_config = {} + try: + tree = ET.parse(config_file) + root = tree.getroot() + if root.tag != 'Backends': + raise ParseError("The root element in cloudauthz_backends_config.xml file " + "is expected to be `Backends`, found `{}` instead -- unable " + "to continue.".format(root.tag)) + + for child in root: + if child.tag != "provider": + log.error("Expect a node with `provider` tag, found a node with `{}` tag instead; " + "skipping the node.".format(child.tag)) + continue + + if 'name' not in child.attrib: + log.error("Could not find a node attribute 'name'; skipping the node '{}'.".format(child.tag)) + continue + + provider = child.get('name').lower() + if provider == "gcp": + try: + config = {"service_account_credentials": child.find("service_account_credentials").text} + self.cloudauthz_backends_config[provider] = config + except Exception as e: + raise e + except ImportError: + raise + except ParseError as e: + raise ParseError("Invalid configuration at `{}`: {} -- unable to continue.".format(config_file, e)) + def _parse_idp_config(self, config_xml): rtv = { 'client_id': config_xml.find('client_id').text, diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 7a1fd5aa37a..391e2700a59 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -222,6 +222,8 @@ class Configuration(object): self.oidc_config = kwargs.get("oidc_config_file", self.oidc_config_file) self.oidc_backends_config = kwargs.get("oidc_backends_config_file", self.oidc_backends_config_file) self.oidc = [] + # Galaxy cloud authorization (cloudauthz) settings. + self.cloudauthz_backends_config = kwargs.get("cloudauthz_backends_config_file", None) # The value of migrated_tools_config is the file reserved for containing only those tools that have been eliminated from the distribution # and moved to the tool shed. self.integrated_tool_panel_config = resolve_path(kwargs.get('integrated_tool_panel_config', 'integrated_tool_panel.xml'), self.root) From ee1cf3c5c24cbb1b23a8cd2a9f811191da0f0ab6 Mon Sep 17 00:00:00 2001 From: vahid Date: Sun, 16 Jun 2019 16:48:45 -0700 Subject: [PATCH 03/79] Create GCP configuration. --- lib/galaxy/authnz/managers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index f85d3459522..ed424237316 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -213,6 +213,8 @@ class AuthnzManager(object): raise exceptions.AuthenticationFailed( err_msg="An error occurred getting your ID token. {}. If the problem persists, please " "contact Galaxy admin.".format(msg)) + elif cloudauthz.provider == "gcp": + config["server_credentials"] = self.cloudauthz_backends_config.get("gcp").get("service_account_credentials") return config @staticmethod From e0a3d12e7ba778808762b87095ec8d5fe3da5143 Mon Sep 17 00:00:00 2001 From: vahid Date: Sun, 16 Jun 2019 17:22:48 -0700 Subject: [PATCH 04/79] Wrap GCP credentials for cloudbridge. --- lib/galaxy/authnz/managers.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index ed424237316..773eddf0a6d 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -217,6 +217,14 @@ class AuthnzManager(object): config["server_credentials"] = self.cloudauthz_backends_config.get("gcp").get("service_account_credentials") return config + def _extend_cloudauthz_credentials(self, cloudauthz, credentials): + if cloudauthz.provider == "gcp": + credentials = { + "credentials": credentials, + "project_id": cloudauthz.config.get("project_id", None) + } + return credentials + @staticmethod def can_user_assume_authn(trans, authn_id): qres = trans.sa_session.query(model.UserAuthnzToken).get(authn_id) @@ -338,7 +346,8 @@ class AuthnzManager(object): ca = CloudAuthz() log.info("Requesting credentials using CloudAuthz with config id `{}` on be half of user `{}`.".format( cloudauthz.id, user_id)) - return ca.authorize(cloudauthz.provider, config) + credentials = ca.authorize(cloudauthz.provider, config) + return self._extend_cloudauthz_credentials(cloudauthz, credentials) except CloudAuthzBaseException as e: log.info(e) raise exceptions.AuthenticationFailed(e) From 58c73f08953cf9f3661074c956ac336c3f31830d Mon Sep 17 00:00:00 2001 From: vahid Date: Sun, 16 Jun 2019 17:23:33 -0700 Subject: [PATCH 05/79] Capture an exception calling cloudauthz with a not implemented backend. --- lib/galaxy/authnz/managers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 773eddf0a6d..320dabe9083 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -351,6 +351,9 @@ class AuthnzManager(object): except CloudAuthzBaseException as e: log.info(e) raise exceptions.AuthenticationFailed(e) + except NotImplementedError as e: + log.info(e.message) + raise exceptions.RequestParameterInvalidException(e.message) def get_cloud_access_credentials_in_file(self, new_file_path, cloudauthz, sa_session, user_id, request=None): """ From 8ad42b1cfaaa5215c3e888ece7c826c760ced3a2 Mon Sep 17 00:00:00 2001 From: vahid Date: Sun, 16 Jun 2019 17:24:47 -0700 Subject: [PATCH 06/79] configure and create a gcp backend of cloudbridge. --- lib/galaxy/managers/cloud.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index b43798c219e..16f6850ddbc 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -147,6 +147,10 @@ class CloudManager(sharable.SharableModelManager): 'os_project_domain_name': prj_domain_name, 'os_user_domain_name': user_domain_name} connection = CloudProviderFactory().create_provider(ProviderList.OPENSTACK, config) + elif provider == "gcp": + config = {"gcp_credentials_obj": credentials.get("credentials"), + "gcp_service_creds_dict": {"project_id": credentials.get("project_id")}} + connection = CloudProviderFactory().create_provider(ProviderList.GCP, config) else: raise RequestParameterInvalidException("Unrecognized provider '{}'; the following are the supported " "providers: {}.".format(provider, SUPPORTED_PROVIDERS.keys())) From d221a006817799cbed3a37cb1ee23da0095840a8 Mon Sep 17 00:00:00 2001 From: vahid Date: Sun, 16 Jun 2019 17:25:06 -0700 Subject: [PATCH 07/79] Update a comment. --- lib/galaxy/managers/cloud.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index 16f6850ddbc..1550a6b504f 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -344,8 +344,8 @@ class CloudManager(sharable.SharableModelManager): try: object_label = hda.name.replace(" ", "_") args = { - # We encode ID here because it the tool wrapper assumes - # it receives an encoded ID and attempts decoding it. + # We encode ID here because the tool wrapper expects + # an encoded ID and attempts decoding it. "authz_id": trans.security.encode_id(cloudauthz.id), "bucket": bucket_name, "object_label": object_label, From e8700fde5bf46ce6af9faed372a412fe53c6ca81 Mon Sep 17 00:00:00 2001 From: vahid Date: Sun, 30 Jun 2019 10:58:05 -0700 Subject: [PATCH 08/79] Update to CloudAuthz v0.6.0. # - dependencies update; # - authorization flow update; # - configuration update. --- config/galaxy.yml.sample | 5 - doc/source/admin/galaxy_options.rst | 12 --- lib/galaxy/app.py | 3 +- lib/galaxy/authnz/managers.py | 47 +--------- lib/galaxy/config.py | 2 - .../dependencies/pipfiles/default/Pipfile | 2 +- .../default/pinned-dev-requirements.txt | 41 +++++---- .../pipfiles/default/pinned-requirements.txt | 92 ++++++++++--------- .../pipfiles/flake8/pinned-requirements.txt | 2 +- lib/galaxy/managers/cloud.py | 3 +- lib/galaxy/webapps/galaxy/config_schema.yml | 10 -- 11 files changed, 76 insertions(+), 143 deletions(-) diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample index 104f7d31469..ff5c0b71f46 100644 --- a/config/galaxy.yml.sample +++ b/config/galaxy.yml.sample @@ -1421,11 +1421,6 @@ galaxy: # Sets the path to OIDC backends configuration file. #oidc_backends_config_file: config/oidc_backends_config.xml - # Sets the path to a file containing setters necessary for - # configuration and/or initialization of cloud authorization - # (cloudauthz) backends (e.g., Google Cloud Platform). - #cloudauthz_backends_config_file: config/cloudauthz_backends_config.xml - # XML config file that allows the use of different authentication # providers (e.g. LDAP) instead or in addition to local authentication # (.sample is used if default does not exist). diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 8b5b3a622e7..ce904ed90b8 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -2923,18 +2923,6 @@ :Type: str -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``cloudauthz_backends_config_file`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:Description: - Sets the path to a file containing setters necessary for - configuration and/or initialization of cloud authorization - (cloudauthz) backends (e.g., Google Cloud Platform). -:Default: ``config/cloudauthz_backends_config.xml`` -:Type: str - - ~~~~~~~~~~~~~~~~~~~~ ``auth_config_file`` ~~~~~~~~~~~~~~~~~~~~ diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 1c7136c68d7..f0e3c24e2a4 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -185,8 +185,7 @@ class UniverseApplication(config.ConfiguresGalaxyMixin): from galaxy.authnz import managers self.authnz_manager = managers.AuthnzManager(self, self.config.oidc_config, - self.config.oidc_backends_config, - self.config.cloudauthz_backends_config) + self.config.oidc_backends_config) self.sentry_client = None if self.config.sentry_dsn: diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 320dabe9083..b26bb4e856a 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -31,7 +31,7 @@ log = logging.getLogger(__name__) class AuthnzManager(object): - def __init__(self, app, oidc_config_file, oidc_backends_config_file, cloudauthz_backends_config_file): + def __init__(self, app, oidc_config_file, oidc_backends_config_file): """ :type app: galaxy.app.UniverseApplication :param app: @@ -43,7 +43,6 @@ class AuthnzManager(object): self.app = app self._parse_oidc_config(oidc_config_file) self._parse_oidc_backends_config(oidc_backends_config_file) - self._parse_cloudauthz_backends_config(cloudauthz_backends_config_file) def _parse_oidc_config(self, config_file): self.oidc_config = {} @@ -107,38 +106,6 @@ class AuthnzManager(object): except ParseError as e: raise ParseError("Invalid configuration at `{}`: {} -- unable to continue.".format(config_file, e)) - def _parse_cloudauthz_backends_config(self, config_file): - self.cloudauthz_backends_config = {} - try: - tree = ET.parse(config_file) - root = tree.getroot() - if root.tag != 'Backends': - raise ParseError("The root element in cloudauthz_backends_config.xml file " - "is expected to be `Backends`, found `{}` instead -- unable " - "to continue.".format(root.tag)) - - for child in root: - if child.tag != "provider": - log.error("Expect a node with `provider` tag, found a node with `{}` tag instead; " - "skipping the node.".format(child.tag)) - continue - - if 'name' not in child.attrib: - log.error("Could not find a node attribute 'name'; skipping the node '{}'.".format(child.tag)) - continue - - provider = child.get('name').lower() - if provider == "gcp": - try: - config = {"service_account_credentials": child.find("service_account_credentials").text} - self.cloudauthz_backends_config[provider] = config - except Exception as e: - raise e - except ImportError: - raise - except ParseError as e: - raise ParseError("Invalid configuration at `{}`: {} -- unable to continue.".format(config_file, e)) - def _parse_idp_config(self, config_xml): rtv = { 'client_id': config_xml.find('client_id').text, @@ -213,18 +180,8 @@ class AuthnzManager(object): raise exceptions.AuthenticationFailed( err_msg="An error occurred getting your ID token. {}. If the problem persists, please " "contact Galaxy admin.".format(msg)) - elif cloudauthz.provider == "gcp": - config["server_credentials"] = self.cloudauthz_backends_config.get("gcp").get("service_account_credentials") return config - def _extend_cloudauthz_credentials(self, cloudauthz, credentials): - if cloudauthz.provider == "gcp": - credentials = { - "credentials": credentials, - "project_id": cloudauthz.config.get("project_id", None) - } - return credentials - @staticmethod def can_user_assume_authn(trans, authn_id): qres = trans.sa_session.query(model.UserAuthnzToken).get(authn_id) @@ -347,7 +304,7 @@ class AuthnzManager(object): log.info("Requesting credentials using CloudAuthz with config id `{}` on be half of user `{}`.".format( cloudauthz.id, user_id)) credentials = ca.authorize(cloudauthz.provider, config) - return self._extend_cloudauthz_credentials(cloudauthz, credentials) + return credentials except CloudAuthzBaseException as e: log.info(e) raise exceptions.AuthenticationFailed(e) diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 391e2700a59..7a1fd5aa37a 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -222,8 +222,6 @@ class Configuration(object): self.oidc_config = kwargs.get("oidc_config_file", self.oidc_config_file) self.oidc_backends_config = kwargs.get("oidc_backends_config_file", self.oidc_backends_config_file) self.oidc = [] - # Galaxy cloud authorization (cloudauthz) settings. - self.cloudauthz_backends_config = kwargs.get("cloudauthz_backends_config_file", None) # The value of migrated_tools_config is the file reserved for containing only those tools that have been eliminated from the distribution # and moved to the tool shed. self.integrated_tool_panel_config = resolve_path(kwargs.get('integrated_tool_panel_config', 'integrated_tool_panel.xml'), self.root) diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index 6741c925615..07c68e00cf8 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -76,7 +76,7 @@ pyparsing = "*" paramiko = "*" python-genomespaceclient = "<2.0" social_auth_core = {version = "==3.1.0", extras = ['openidconnect']} -cloudauthz = "<=0.2.0" +cloudauthz = "==0.6.0" gxformat2 = "*" [requires] diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt index 282c185ca8d..a9bbd88d4d1 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt @@ -4,22 +4,25 @@ alabaster==0.7.12 argh==0.26.2 atomicwrites==1.3.0 attrs==19.1.0 -babel==2.6.0 -certifi==2019.3.9 +babel==2.7.0 +certifi==2019.6.16 chardet==3.0.4 -commonmark==0.8.1 +commonmark==0.9.0 +configparser==3.7.4 ; python_version < '3.2' +contextlib2==0.5.5 ; python_version < '3.5' docutils==0.14 funcsigs==1.0.2 ; python_version < '3.3' future==0.17.1 gunicorn==19.9.0 idna==2.8 imagesize==1.1.0 +importlib-metadata==0.18 jinja2==2.10.1 -lxml==4.3.3 +lxml==4.3.4 markdown==2.6.11 markupsafe==1.1.1 mirakuru==1.1.0 -mock==2.0.0 +mock==3.0.5 more-itertools==5.0.0 nose==1.3.7 nosehtml==0.4.5 @@ -27,32 +30,34 @@ packaging==19.0 pathlib2==2.3.2 ; python_version < '3' pathtools==0.1.2 pbr==5.1.3 -pluggy==0.9.0 +pluggy==0.12.0 port-for==0.4 -psutil==5.6.1 +psutil==5.6.3 py==1.8.0 pygithub3==0.5.1 ; python_version < '3' -pygments==2.3.1 +pygments==2.4.2 pyparsing==2.4.0 -pytest-html==1.20.0 +pytest-html==1.21.1 pytest-metadata==1.8.0 -pytest-postgresql==1.4.0 +pytest-postgresql==1.4.1 pytest-pythonpath==0.7.3 -pytest==4.4.1 +pytest==4.6.4 pytz==2019.1 -pyyaml==5.1 +pyyaml==5.1.1 recommonmark==0.5.0 -requests==2.21.0 +requests==2.22.0 scandir==1.10.0 ; python_version < '3.5' selenium==3.141.0 six==1.11.0 -snowballstemmer==1.2.1 +snowballstemmer==1.9.0 sphinx-markdown-tables==0.0.9 sphinx-rtd-theme==0.4.3 sphinx==1.8.5 -sphinxcontrib-websupport==1.1.0 -testfixtures==6.7.0 +sphinxcontrib-websupport==1.1.2 +testfixtures==6.10.0 twill==0.9.1 ; python_version < '3' -typing==3.6.6 ; python_version < '3.5' -urllib3==1.24.2 ; python_version == '2.7' +typing==3.7.4 ; python_version < '3.5' +urllib3==1.25.3 ; python_version == '2.7' watchdog==0.9.0 +wcwidth==0.1.7 ; sys_platform != 'win32' +zipp==0.5.1 diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index fd625aa8efb..2989917489d 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -1,9 +1,10 @@ -i https://wheels.galaxyproject.org/simple --extra-index-url https://pypi.python.org/simple adal==1.2.1 -amqp==2.4.2 +amqp==2.5.0 appdirs==1.4.3 asn1crypto==0.24.0 +attrs==19.1.0 avro==1.8.1 ; python_version < '3' azure-common==1.1.14 azure-cosmosdb-nspkg==2.0.2 @@ -16,11 +17,11 @@ azure-mgmt-resource==2.0.0 azure-mgmt-storage==2.0.0 azure-nspkg==3.0.2 azure-storage-blob==1.3.1 -azure-storage-common==1.4.0 +azure-storage-common==1.4.2 azure-storage-nspkg==3.1.0 -babel==2.6.0 +babel==2.7.0 bagit==1.6.4 -bcrypt==3.1.6 +bcrypt==3.1.7 bdbag==1.4.1 beaker==1.10.1 bioblend==0.12.0 @@ -28,21 +29,21 @@ bleach==3.1.0 boltons==19.1.0 boto3==1.9.114 boto==2.49.0 -botocore==1.12.133 -bx-python==0.8.2 +botocore==1.12.180 +bx-python==0.8.4 bz2file==0.98 ; python_version < '3.3' cachecontrol==0.11.7 -cachetools==3.1.0 -certifi==2019.3.9 +cachetools==3.1.1 +certifi==2019.6.16 cffi==1.12.3 chardet==3.0.4 -cheetah3==3.2.2 -cliff==2.14.1 -cloudauthz==0.2.0 +cheetah3==3.2.3 +cliff==2.15.0 +cloudauthz==0.6.0 cloudbridge==2.0.0 cmd2==0.8.9 contextlib2==0.5.5 ; python_version < '3.5' -cryptography==2.6.1 +cryptography==2.7 cwltool==1.0.20180721142728 debtcollector==1.21.0 decorator==4.4.0 @@ -63,9 +64,9 @@ galaxy-sequence-utils==1.1.3 google-api-python-client==1.7.8 google-auth-httplib2==0.0.3 google-auth==1.6.3 -gxformat2==0.8.3 +gxformat2==0.8.4 h5py==2.9.0 -httplib2==0.12.1 +httplib2==0.13.0 idna==2.8 ipaddress==1.0.22 ; python_version < '3.3' isa-rwval==0.10.7 @@ -74,12 +75,12 @@ isodate==0.6.0 jmespath==0.9.4 jsonpatch==1.23 jsonpointer==2.0 -jsonschema==2.6.0 +jsonschema==3.0.1 keystoneauth1==3.14.0 -kombu==4.5.0 +kombu==4.6.3 lockfile==0.12.2 -lxml==4.3.3 -mako==1.0.9 +lxml==4.3.4 +mako==1.0.12 markupsafe==1.1.1 mercurial==3.7.3 ; python_version < '3' mistune==0.8.4 @@ -94,35 +95,35 @@ netifaces==0.10.9 networkx==1.11 nodeenv==1.3.3 nose==1.3.7 -numpy==1.16.2 +numpy==1.16.4 oauth2client==4.1.3 oauthlib==3.0.1 openstacksdk==0.17.0 os-client-config==1.32.0 -os-service-types==1.6.0 -osc-lib==1.12.1 -oslo.config==6.8.1 +os-service-types==1.7.0 +osc-lib==1.13.0 +oslo.config==6.10.0 oslo.context==2.22.1 oslo.i18n==3.23.1 -oslo.log==3.42.3 -oslo.serialization==2.28.2 -oslo.utils==3.40.3 +oslo.log==3.44.0 +oslo.serialization==2.29.1 +oslo.utils==3.41.0 packaging==19.0 -paramiko==2.4.2 +paramiko==2.6.0 parsley==1.3 -paste==3.0.8 +paste==3.0.8 ; python_version == '2.7' pastedeploy==2.0.1 pastescript==3.1.0 pathlib2==2.3.2 ; python_version < '3' -pbr==5.1.3 +pbr==5.3.1 prettytable==0.7.2 prov==1.5.1 -psutil==5.6.1 -pulsar-galaxy-lib==0.11.0 -pyasn1-modules==0.2.4 +psutil==5.6.3 +pulsar-galaxy-lib==0.13.0 +pyasn1-modules==0.2.5 pyasn1==0.4.5 pycparser==2.19 -pycryptodome==3.8.1 +pycryptodome==3.8.2 pyeventsystem==0.1.0 pyinotify==0.9.6 ; sys_platform != 'win32' and sys_platform != 'darwin' and sys_platform != 'sunos5' pyjwt==1.7.1 @@ -131,6 +132,7 @@ pynacl==1.3.0 pyopenssl==19.0.0 pyparsing==2.4.0 pyperclip==1.7.0 +pyrsistent==0.15.2 pysam==0.15.2 pysftp==0.2.9 python-cinderclient==4.0.0 @@ -144,46 +146,46 @@ python-novaclient==11.0.0 python-openid==2.2.5 ; python_version < '3.0' python-swiftclient==3.6.0 pytz==2019.1 -pyyaml==5.1 +pyyaml==5.1.1 rdflib-jsonld==0.4.0 rdflib==4.2.2 repoze.lru==0.7 requests-oauthlib==1.2.0 requests-toolbelt==0.9.1 -requests==2.21.0 +requests==2.22.0 requestsexceptions==1.4.0 -rfc3986==1.2.0 +rfc3986==1.3.2 routes==2.4.1 rsa==4.0 ruamel.ordereddict==0.4.13 ; platform_python_implementation == 'CPython' and python_version <= '2.7' -ruamel.yaml==0.15.92 -s3transfer==0.2.0 +ruamel.yaml==0.15.97 +s3transfer==0.2.1 scandir==1.10.0 ; python_version < '3.5' schema-salad==2.7.20181126142424 shellescape==3.4.1 simplejson==3.16.0 six==1.11.0 -social-auth-core[openidconnect]==3.1.0+gx0 +social-auth-core[openidconnect]==3.1.0 sqlalchemy-migrate==0.12.0 sqlalchemy-utils==0.33.11 sqlalchemy==1.2.18 sqlparse==0.3.0 stevedore==1.30.1 -subprocess32==3.5.3 ; python_version < '3.0' -svgwrite==1.2.1 +subprocess32==3.5.4 ; python_version < '3.0' +svgwrite==1.3.1 tempita==0.5.2 tenacity==4.12.0 -typing-extensions==3.7.2 -typing==3.6.6 ; python_version < '3.5' +typing-extensions==3.7.4 +typing==3.7.4 ; python_version < '3.5' tzlocal==1.5.1 unicodecsv==0.14.1 ; python_version < '3.0' uritemplate==3.0.0 -urllib3==1.24.2 ; python_version == '2.7' +urllib3==1.25.3 ; python_version == '2.7' uwsgi==2.0.18 vine==1.3.0 -warlock==1.3.0 +warlock==1.3.3 wcwidth==0.1.7 ; sys_platform != 'win32' webencodings==0.5.1 webob==1.8.5 whoosh==2.7.4 -wrapt==1.11.1 +wrapt==1.11.2 diff --git a/lib/galaxy/dependencies/pipfiles/flake8/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/flake8/pinned-requirements.txt index b2aeef4d492..d3da4aaf062 100644 --- a/lib/galaxy/dependencies/pipfiles/flake8/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/flake8/pinned-requirements.txt @@ -8,4 +8,4 @@ functools32==3.2.3.post2 ; python_version < '3.2' mccabe==0.6.1 pycodestyle==2.5.0 pyflakes==2.1.1 -typing==3.6.6 ; python_version < '3.5' +typing==3.7.4 ; python_version < '3.5' diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index 1550a6b504f..7f8f272b15e 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -148,8 +148,7 @@ class CloudManager(sharable.SharableModelManager): 'os_user_domain_name': user_domain_name} connection = CloudProviderFactory().create_provider(ProviderList.OPENSTACK, config) elif provider == "gcp": - config = {"gcp_credentials_obj": credentials.get("credentials"), - "gcp_service_creds_dict": {"project_id": credentials.get("project_id")}} + config = {"gcp_service_creds_dict": credentials} connection = CloudProviderFactory().create_provider(ProviderList.GCP, config) else: raise RequestParameterInvalidException("Unrecognized provider '{}'; the following are the supported " diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 327a30eda16..91b3482b4e1 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -2171,16 +2171,6 @@ mapping: desc: | Sets the path to OIDC backends configuration file. - cloudauthz_backends_config_file: - type: str - default: config/cloudauthz_backends_config.xml - required: false - desc: | - Sets the path to a file containing setters necessary - for configuration and/or initialization of - cloud authorization (cloudauthz) backends (e.g., - Google Cloud Platform). - auth_config_file: type: str default: config/auth_conf.xml From f293e5b827e8c7efeb8f4288521c5389ebc4bf70 Mon Sep 17 00:00:00 2001 From: vahid Date: Sun, 30 Jun 2019 11:36:34 -0700 Subject: [PATCH 09/79] Merge remote and resolve conflicts. Fix social-auth-core dependency. --- .../pipfiles/default/pinned-dev-requirements.txt | 8 ++++---- .../pipfiles/default/pinned-requirements.txt | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt index c4e39239c69..79986eca757 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt @@ -38,11 +38,11 @@ pygithub3==0.5.1 ; python_version < '3' pygments==2.4.2 pyparsing==2.4.0 pytest-cov==2.7.1 -pytest-html==1.20.0 +pytest-html==1.21.1 pytest-metadata==1.8.0 pytest-postgresql==1.4.1 pytest-pythonpath==0.7.3 -pytest==4.6.3 +pytest==4.6.4 pytz==2019.1 pyyaml==5.1.1 recommonmark==0.5.0 @@ -55,9 +55,9 @@ sphinx-markdown-tables==0.0.9 sphinx-rtd-theme==0.4.3 sphinx==1.8.5 sphinxcontrib-websupport==1.1.2 -testfixtures==6.9.0 +testfixtures==6.10.0 twill==0.9.1 ; python_version < '3' -typing==3.6.6 ; python_version < '3.5' +typing==3.7.4 ; python_version < '3.5' urllib3==1.25.3 ; python_version == '2.7' watchdog==0.9.0 wcwidth==0.1.7 ; sys_platform != 'win32' diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 0cd2549e372..00ad1d6cc2c 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -39,7 +39,7 @@ cffi==1.12.3 chardet==3.0.4 cheetah3==3.2.3 cliff==2.15.0 -cloudauthz==0.4.0 +cloudauthz==0.6.0 cloudbridge==2.0.0 cmd2==0.8.9 contextlib2==0.5.5 ; python_version < '3.5' @@ -109,17 +109,17 @@ oslo.log==3.44.0 oslo.serialization==2.29.1 oslo.utils==3.41.0 packaging==19.0 -paramiko==2.5.0 +paramiko==2.6.0 parsley==1.3 -paste==3.0.8 ; python_version == '2.7' +paste==3.0.8 pastedeploy==2.0.1 pastescript==3.1.0 pathlib2==2.3.2 ; python_version < '3' -pbr==5.3.0 +pbr==5.3.1 prettytable==0.7.2 prov==1.5.1 psutil==5.6.3 -pulsar-galaxy-lib==0.12.1 +pulsar-galaxy-lib==0.13.0 pyasn1-modules==0.2.5 pyasn1==0.4.5 pycparser==2.19 @@ -165,10 +165,10 @@ schema-salad==2.7.20181126142424 shellescape==3.4.1 simplejson==3.16.0 six==1.11.0 -social-auth-core[openidconnect]==3.1.0 +social-auth-core[openidconnect]==3.1.0+gx0 sqlalchemy-migrate==0.12.0 sqlalchemy-utils==0.34.0 -sqlalchemy==1.3.4 +sqlalchemy==1.3.5 sqlparse==0.3.0 stevedore==1.30.1 subprocess32==3.5.4 ; python_version < '3.0' From 54302eb76f6a8d151ebd7ac26eff0eacaf99bfab Mon Sep 17 00:00:00 2001 From: Vahid Date: Sun, 30 Jun 2019 15:03:23 -0700 Subject: [PATCH 10/79] Update lib/galaxy/authnz/managers.py Co-Authored-By: Nicola Soranzo --- lib/galaxy/authnz/managers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index b633e18d517..36c490d255d 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -311,7 +311,7 @@ class AuthnzManager(object): raise exceptions.AuthenticationFailed(e) except NotImplementedError as e: log.info(e.message) - raise exceptions.RequestParameterInvalidException(e.message) + raise exceptions.RequestParameterInvalidException(e) def get_cloud_access_credentials_in_file(self, new_file_path, cloudauthz, sa_session, user_id, request=None): """ From 4b7db93176b55f407c8f976d227cec98742eb413 Mon Sep 17 00:00:00 2001 From: Vahid Date: Sun, 30 Jun 2019 15:03:29 -0700 Subject: [PATCH 11/79] Update lib/galaxy/authnz/managers.py Co-Authored-By: Nicola Soranzo --- lib/galaxy/authnz/managers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 36c490d255d..4f6cefeb29f 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -310,7 +310,7 @@ class AuthnzManager(object): log.info(e) raise exceptions.AuthenticationFailed(e) except NotImplementedError as e: - log.info(e.message) + log.info(e) raise exceptions.RequestParameterInvalidException(e) def get_cloud_access_credentials_in_file(self, new_file_path, cloudauthz, sa_session, user_id, request=None): From dff2d0b1b09dc111bab49bbd79184067034d57b0 Mon Sep 17 00:00:00 2001 From: vahid Date: Mon, 1 Jul 2019 09:16:21 -0700 Subject: [PATCH 12/79] Pinned Pulsar to 0.12.1 since 0.13.0 has a backward incompatible update. --- lib/galaxy/dependencies/pipfiles/default/Pipfile | 2 +- .../dependencies/pipfiles/default/pinned-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index ed21e8daecd..6a802601d07 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -68,7 +68,7 @@ bioblend = "*" boto = "*" kombu = "*" psutil = "*" -pulsar-galaxy-lib = "*" +pulsar-galaxy-lib = "==0.12.1" sqlalchemy-migrate = "*" sqlparse = "*" svgwrite = "*" diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 00ad1d6cc2c..8acabe3110f 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -119,7 +119,7 @@ pbr==5.3.1 prettytable==0.7.2 prov==1.5.1 psutil==5.6.3 -pulsar-galaxy-lib==0.13.0 +pulsar-galaxy-lib==0.12.1 pyasn1-modules==0.2.5 pyasn1==0.4.5 pycparser==2.19 From 80febae523c133b296bf88d07d0022adc1146a43 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Sun, 7 Jul 2019 17:20:24 +0200 Subject: [PATCH 13/79] Pass through tool defined environment_variables to Docker. --- lib/galaxy/tools/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 7beabc880b3..e9490bb99d0 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -705,6 +705,10 @@ class Tool(Dictifiable): self.home_target = home_target self.tmp_target = tmp_target self.docker_env_pass_through = tool_source.parse_docker_env_pass_through() + if self.environment_variables: + if not self.docker_env_pass_through: + self.docker_env_pass_through = [] + self.docker_env_pass_through.extend(map(lambda x: x['name'], self.environment_variables)) # Parameters used to build URL for redirection to external app redirect_url_params = tool_source.parse_redirect_url_params_elem() From 1b2c6b189aad15f4b60dcdf2ce538a2679265965 Mon Sep 17 00:00:00 2001 From: Juleen Graham Date: Mon, 8 Jul 2019 10:58:10 +0200 Subject: [PATCH 14/79] implement zoom feature --- .../scripts/mvc/workflow/workflow-canvas.js | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/client/galaxy/scripts/mvc/workflow/workflow-canvas.js b/client/galaxy/scripts/mvc/workflow/workflow-canvas.js index 2f529d124d2..7dba6172f29 100644 --- a/client/galaxy/scripts/mvc/workflow/workflow-canvas.js +++ b/client/galaxy/scripts/mvc/workflow/workflow-canvas.js @@ -97,7 +97,7 @@ class CanvasManager { this.init_copy_paste(); } setZoom(zoomLevel) { - this.zoomLevel = Math.min(Math.max(0, zoomLevel), zoomLevels.length); + this.zoomLevel = Math.min(Math.max(0, zoomLevel), zoomLevels.length-1); this.canvasZoom = zoomLevels[this.zoomLevel]; // Set CSS transform to appropriate zoom level this.cv.css("transform-origin", "top left"); @@ -109,16 +109,33 @@ class CanvasManager { this.app.workflow.fit_canvas_to_nodes(); } initZoomControls() { - var zoomControl = $('
').css({ + var zoomControl = $('
').css({ position: "absolute", left: "1rem", - bottom: "1rem" + bottom: "1rem", + cursor: "pointer" }); + const zoomButton = $(`${zoomLevels[defaultZoomLevel] * 100}%`).css({ + width: "4rem" + }); + zoomControl.append( - $('
').click(() => this.setZoom(this.zoomLevel + 1)) + $('').click(() => { + this.setZoom(this.zoomLevel - 1); + zoomButton.text(Math.floor(zoomLevels[this.zoomLevel] * 100) + '%'); + }) ); zoomControl.append( - $('
').click(() => this.setZoom(this.zoomLevel - 1)) + zoomButton.click(() => { + this.setZoom(defaultZoomLevel); + zoomButton.text(Math.floor(zoomLevels[this.zoomLevel] * 100) + '%'); + }) + ); + zoomControl.append( + $('').click(() => { + this.setZoom(this.zoomLevel + 1); + zoomButton.text(Math.floor(zoomLevels[this.zoomLevel] * 100) + '%'); + }) ); this.cv.closest("#workflow-canvas-body").append(zoomControl); } From 377e48918d9006a17e880c1bce4616f18afc5d42 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 7 Jul 2019 20:07:59 +0200 Subject: [PATCH 15/79] Create galaxy-selenium package. This is *not* the tests, it is the components for interacting with Galaxy via Selenium that can be used in any Galaxy UI automation context - not just tests. This includes a tiny CLI framework for building apps that drive Galaxy this way. --- .../galaxy/selenium}/__init__.py | 0 .../galaxy/selenium}/cli.py | 0 .../galaxy/selenium}/components.py | 4 +- .../galaxy/selenium}/data.py | 0 .../galaxy/selenium}/driver_factory.py | 0 .../galaxy/selenium}/has_driver.py | 0 .../galaxy/selenium}/navigates_galaxy.py | 0 .../galaxy/selenium}/navigation-data.yml | 0 .../galaxy/selenium}/navigation.yml | 0 .../galaxy/selenium}/sizzle.py | 0 .../galaxy/selenium}/smart_components.py | 2 +- packages/selenium/HISTORY.rst | 12 +++ packages/selenium/LICENSE | 1 + packages/selenium/MANIFEST.in | 2 + packages/selenium/Makefile | 1 + packages/selenium/README.rst | 15 +++ packages/selenium/dev-requirements.txt | 1 + packages/selenium/galaxy/__init__.py | 1 + .../galaxy/project_galaxy_selenium.py | 13 +++ packages/selenium/galaxy/selenium | 1 + packages/selenium/requirements.txt | 2 + packages/selenium/scripts | 1 + packages/selenium/setup.cfg | 1 + packages/selenium/setup.py | 101 ++++++++++++++++++ test/selenium_tests/framework.py | 4 +- test/selenium_tests/test_navigates_galaxy.py | 3 +- test/selenium_tests/test_tool_form.py | 2 +- 27 files changed, 159 insertions(+), 8 deletions(-) rename {test/galaxy_selenium => lib/galaxy/selenium}/__init__.py (100%) rename {test/galaxy_selenium => lib/galaxy/selenium}/cli.py (100%) rename {test/galaxy_selenium => lib/galaxy/selenium}/components.py (96%) rename {test/galaxy_selenium => lib/galaxy/selenium}/data.py (100%) rename {test/galaxy_selenium => lib/galaxy/selenium}/driver_factory.py (100%) rename {test/galaxy_selenium => lib/galaxy/selenium}/has_driver.py (100%) rename {test/galaxy_selenium => lib/galaxy/selenium}/navigates_galaxy.py (100%) rename {test/galaxy_selenium => lib/galaxy/selenium}/navigation-data.yml (100%) rename {test/galaxy_selenium => lib/galaxy/selenium}/navigation.yml (100%) rename {test/galaxy_selenium => lib/galaxy/selenium}/sizzle.py (100%) rename {test/galaxy_selenium => lib/galaxy/selenium}/smart_components.py (97%) create mode 100644 packages/selenium/HISTORY.rst create mode 120000 packages/selenium/LICENSE create mode 100644 packages/selenium/MANIFEST.in create mode 120000 packages/selenium/Makefile create mode 100644 packages/selenium/README.rst create mode 120000 packages/selenium/dev-requirements.txt create mode 100644 packages/selenium/galaxy/__init__.py create mode 100644 packages/selenium/galaxy/project_galaxy_selenium.py create mode 120000 packages/selenium/galaxy/selenium create mode 100644 packages/selenium/requirements.txt create mode 120000 packages/selenium/scripts create mode 120000 packages/selenium/setup.cfg create mode 100644 packages/selenium/setup.py diff --git a/test/galaxy_selenium/__init__.py b/lib/galaxy/selenium/__init__.py similarity index 100% rename from test/galaxy_selenium/__init__.py rename to lib/galaxy/selenium/__init__.py diff --git a/test/galaxy_selenium/cli.py b/lib/galaxy/selenium/cli.py similarity index 100% rename from test/galaxy_selenium/cli.py rename to lib/galaxy/selenium/cli.py diff --git a/test/galaxy_selenium/components.py b/lib/galaxy/selenium/components.py similarity index 96% rename from test/galaxy_selenium/components.py rename to lib/galaxy/selenium/components.py index 10760bb90ab..16fa64012dd 100644 --- a/test/galaxy_selenium/components.py +++ b/lib/galaxy/selenium/components.py @@ -98,7 +98,7 @@ class SelectorTemplate(Target): if name in self._children: return self._children[name](**{"_": self.selector}) else: - raise KeyError("Could not find child [%s] in %s" % (name, self._children)) + raise AttributeError("Could not find child [%s] in %s" % (name, self._children)) __getitem__ = __getattr__ @@ -193,7 +193,7 @@ class Component(object): elif attr in self._text: return self._text[attr] else: - raise Exception("Failed to find referenced sub-component/selector/label/text [%s]" % attr) + raise KeyError("Failed to find referenced sub-component/selector/label/text [%s]" % attr) __getitem__ = __getattr__ diff --git a/test/galaxy_selenium/data.py b/lib/galaxy/selenium/data.py similarity index 100% rename from test/galaxy_selenium/data.py rename to lib/galaxy/selenium/data.py diff --git a/test/galaxy_selenium/driver_factory.py b/lib/galaxy/selenium/driver_factory.py similarity index 100% rename from test/galaxy_selenium/driver_factory.py rename to lib/galaxy/selenium/driver_factory.py diff --git a/test/galaxy_selenium/has_driver.py b/lib/galaxy/selenium/has_driver.py similarity index 100% rename from test/galaxy_selenium/has_driver.py rename to lib/galaxy/selenium/has_driver.py diff --git a/test/galaxy_selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py similarity index 100% rename from test/galaxy_selenium/navigates_galaxy.py rename to lib/galaxy/selenium/navigates_galaxy.py diff --git a/test/galaxy_selenium/navigation-data.yml b/lib/galaxy/selenium/navigation-data.yml similarity index 100% rename from test/galaxy_selenium/navigation-data.yml rename to lib/galaxy/selenium/navigation-data.yml diff --git a/test/galaxy_selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml similarity index 100% rename from test/galaxy_selenium/navigation.yml rename to lib/galaxy/selenium/navigation.yml diff --git a/test/galaxy_selenium/sizzle.py b/lib/galaxy/selenium/sizzle.py similarity index 100% rename from test/galaxy_selenium/sizzle.py rename to lib/galaxy/selenium/sizzle.py diff --git a/test/galaxy_selenium/smart_components.py b/lib/galaxy/selenium/smart_components.py similarity index 97% rename from test/galaxy_selenium/smart_components.py rename to lib/galaxy/selenium/smart_components.py index 9f4304f0f74..8c085ebfc12 100644 --- a/test/galaxy_selenium/smart_components.py +++ b/lib/galaxy/selenium/smart_components.py @@ -8,7 +8,7 @@ class SmartComponent(object): """Wrap a Component with driver aware methods. Allows smarter selectors that know how to wait for themselves, test themselves, - click themselvers, etc.... More "magic", but much cleaner usage. + click themselves, etc.... More "magic", but much cleaner usage. """ def __init__(self, component, has_driver): diff --git a/packages/selenium/HISTORY.rst b/packages/selenium/HISTORY.rst new file mode 100644 index 00000000000..ec2f0e746e9 --- /dev/null +++ b/packages/selenium/HISTORY.rst @@ -0,0 +1,12 @@ +.. :changelog: + +History +------- + +.. to_doc + +--------------------- +19.9.0.dev0 +--------------------- + +* Initial import from dev branch of Galaxy during 19.09 development cycle. diff --git a/packages/selenium/LICENSE b/packages/selenium/LICENSE new file mode 120000 index 00000000000..1ef648f64b3 --- /dev/null +++ b/packages/selenium/LICENSE @@ -0,0 +1 @@ +../../LICENSE.txt \ No newline at end of file diff --git a/packages/selenium/MANIFEST.in b/packages/selenium/MANIFEST.in new file mode 100644 index 00000000000..99afa949c6c --- /dev/null +++ b/packages/selenium/MANIFEST.in @@ -0,0 +1,2 @@ +include *.rst LICENSE +include galaxy/selenium/*yml diff --git a/packages/selenium/Makefile b/packages/selenium/Makefile new file mode 120000 index 00000000000..37af8bae5ba --- /dev/null +++ b/packages/selenium/Makefile @@ -0,0 +1 @@ +../package.Makefile \ No newline at end of file diff --git a/packages/selenium/README.rst b/packages/selenium/README.rst new file mode 100644 index 00000000000..b66238cd3b1 --- /dev/null +++ b/packages/selenium/README.rst @@ -0,0 +1,15 @@ + +.. image:: https://badge.fury.io/py/galaxy-selenium.svg + :target: https://pypi.org/project/galaxy-selenium/ + + + +Overview +-------- + +The Galaxy_ selenium framework. + +* Free software: Academic Free License version 3.0 +* Code: https://github.com/galaxyproject/galaxy + +.. _Galaxy: http://galaxyproject.org/ diff --git a/packages/selenium/dev-requirements.txt b/packages/selenium/dev-requirements.txt new file mode 120000 index 00000000000..467b90d7a23 --- /dev/null +++ b/packages/selenium/dev-requirements.txt @@ -0,0 +1 @@ +../package-dev-requirements.txt \ No newline at end of file diff --git a/packages/selenium/galaxy/__init__.py b/packages/selenium/galaxy/__init__.py new file mode 100644 index 00000000000..69e3be50dac --- /dev/null +++ b/packages/selenium/galaxy/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/packages/selenium/galaxy/project_galaxy_selenium.py b/packages/selenium/galaxy/project_galaxy_selenium.py new file mode 100644 index 00000000000..73ec5c1cf31 --- /dev/null +++ b/packages/selenium/galaxy/project_galaxy_selenium.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- + +__version__ = '19.9.0.dev0' + +PROJECT_NAME = "galaxy-selenium" +PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" +PROJECT_URL = "https://github.com/galaxyproject/galaxy" +PROJECT_AUTHOR = 'Galaxy Project and Community' +PROJECT_DESCRIPTION = 'Galaxy Selenium Interaction Framework' +PROJECT_EMAIL = 'galaxy-committers@lists.galaxyproject.org' +RAW_CONTENT_URL = "https://raw.github.com/%s/%s/master/" % ( + PROJECT_USERAME, PROJECT_NAME +) diff --git a/packages/selenium/galaxy/selenium b/packages/selenium/galaxy/selenium new file mode 120000 index 00000000000..60a2948815b --- /dev/null +++ b/packages/selenium/galaxy/selenium @@ -0,0 +1 @@ +../../../lib/galaxy/selenium/ \ No newline at end of file diff --git a/packages/selenium/requirements.txt b/packages/selenium/requirements.txt new file mode 100644 index 00000000000..ea07fcb13dc --- /dev/null +++ b/packages/selenium/requirements.txt @@ -0,0 +1,2 @@ +galaxy-util +selenium diff --git a/packages/selenium/scripts b/packages/selenium/scripts new file mode 120000 index 00000000000..9aec9dc5a06 --- /dev/null +++ b/packages/selenium/scripts @@ -0,0 +1 @@ +../build_scripts \ No newline at end of file diff --git a/packages/selenium/setup.cfg b/packages/selenium/setup.cfg new file mode 120000 index 00000000000..eb7cf09393f --- /dev/null +++ b/packages/selenium/setup.cfg @@ -0,0 +1 @@ +../../setup.cfg \ No newline at end of file diff --git a/packages/selenium/setup.py b/packages/selenium/setup.py new file mode 100644 index 00000000000..38720afe67f --- /dev/null +++ b/packages/selenium/setup.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import ast +import os +import re +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + +SOURCE_DIR = "galaxy" + +_version_re = re.compile(r'__version__\s+=\s+(.*)') + +project_short_name = os.path.basename(os.path.dirname(os.path.realpath(__file__))) +with open('%s/project_galaxy_%s.py' % (SOURCE_DIR, project_short_name), 'rb') as f: + init_contents = f.read().decode('utf-8') + + def get_var(var_name): + pattern = re.compile(r'%s\s+=\s+(.*)' % var_name) + match = pattern.search(init_contents).group(1) + return str(ast.literal_eval(match)) + + version = get_var("__version__") + PROJECT_NAME = get_var("PROJECT_NAME") + PROJECT_URL = get_var("PROJECT_URL") + PROJECT_AUTHOR = get_var("PROJECT_AUTHOR") + PROJECT_EMAIL = get_var("PROJECT_EMAIL") + PROJECT_DESCRIPTION = get_var("PROJECT_DESCRIPTION") + +TEST_DIR = 'tests' +PACKAGES = [ + 'galaxy', + 'galaxy.selenium', +] +ENTRY_POINTS = ''' + [console_scripts] +''' +PACKAGE_DATA = { + # Be sure to update MANIFEST.in for source dist. + 'galaxy': [ + 'selenium/*yml', + ], +} +PACKAGE_DIR = { + SOURCE_DIR: SOURCE_DIR, +} + +readme = open('README.rst').read() +history = open('HISTORY.rst').read().replace('.. :changelog:', '') + +if os.path.exists("requirements.txt"): + requirements = open("requirements.txt").read().split("\n") +else: + # In tox, it will cover them anyway. + requirements = [] + + +test_requirements = [ + # TODO: put package test requirements here +] + + +setup( + name=PROJECT_NAME, + version=version, + description=PROJECT_DESCRIPTION, + long_description=readme + '\n\n' + history, + long_description_content_type='text/x-rst', + author=PROJECT_AUTHOR, + author_email=PROJECT_EMAIL, + url=PROJECT_URL, + packages=PACKAGES, + entry_points=ENTRY_POINTS, + package_data=PACKAGE_DATA, + package_dir=PACKAGE_DIR, + include_package_data=True, + install_requires=requirements, + license="AFL", + zip_safe=False, + keywords='galaxy', + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Environment :: Console', + 'License :: OSI Approved :: Academic Free License (AFL)', + 'Operating System :: POSIX', + 'Topic :: Software Development', + 'Topic :: Software Development :: Code Generators', + 'Topic :: Software Development :: Testing', + 'Natural Language :: English', + "Programming Language :: Python :: 2", + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + ], + test_suite=TEST_DIR, + tests_require=test_requirements +) diff --git a/test/selenium_tests/framework.py b/test/selenium_tests/framework.py index 853c2a04ad1..3ad89b820c5 100644 --- a/test/selenium_tests/framework.py +++ b/test/selenium_tests/framework.py @@ -24,10 +24,10 @@ from base import populators # noqa: I100,I202 from base.api import UsesApiTestCaseMixin # noqa: I100 from base.driver_util import classproperty, DEFAULT_WEB_HOST, get_ip_address # noqa: I100 from base.testcase import FunctionalTestCase # noqa: I100 -from galaxy_selenium import ( # noqa: I100,I201 +from galaxy.selenium import ( # noqa: I100,I201 driver_factory, ) -from galaxy_selenium.navigates_galaxy import ( # noqa: I100 +from galaxy.selenium.navigates_galaxy import ( # noqa: I100 NavigatesGalaxy, retry_during_transitions ) diff --git a/test/selenium_tests/test_navigates_galaxy.py b/test/selenium_tests/test_navigates_galaxy.py index 5fa9c3355ff..c7906791ac2 100644 --- a/test/selenium_tests/test_navigates_galaxy.py +++ b/test/selenium_tests/test_navigates_galaxy.py @@ -1,8 +1,7 @@ -from galaxy_selenium.navigates_galaxy import ( +from galaxy.selenium.navigates_galaxy import ( exception_indicates_not_clickable, exception_seems_to_indicate_transition, ) - from .framework import ( selenium_test, SeleniumTestCase, diff --git a/test/selenium_tests/test_tool_form.py b/test/selenium_tests/test_tool_form.py index cf2c909a3da..f8aef81abf2 100644 --- a/test/selenium_tests/test_tool_form.py +++ b/test/selenium_tests/test_tool_form.py @@ -2,8 +2,8 @@ import json from base import rules_test_data from base.populators import flakey, load_data_dict -from galaxy_selenium.navigates_galaxy import retry_call_during_transitions +from galaxy.selenium.navigates_galaxy import retry_call_during_transitions from .framework import ( managed_history, retry_assertion_during_transitions, From b6d294911ef13fd9cfe5d6014a78103f7fde2ce3 Mon Sep 17 00:00:00 2001 From: Juleen Graham Date: Mon, 8 Jul 2019 12:20:56 +0200 Subject: [PATCH 16/79] change zoom ranges --- client/galaxy/scripts/mvc/workflow/workflow-canvas.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/galaxy/scripts/mvc/workflow/workflow-canvas.js b/client/galaxy/scripts/mvc/workflow/workflow-canvas.js index 7dba6172f29..44d92d9e16c 100644 --- a/client/galaxy/scripts/mvc/workflow/workflow-canvas.js +++ b/client/galaxy/scripts/mvc/workflow/workflow-canvas.js @@ -74,7 +74,7 @@ class ScrollPanel { } // Zoom levels to use for zooming the workflow canvas -const zoomLevels = [0.25, 0.33, 0.5, 0.67, 0.75, 0.8, 0.9, 1, 1.1, 1.25, 1.5, 1.75, 2, 2.5, 3, 4]; +const zoomLevels = [0.25, 0.33, 0.5, 0.67, 0.75, 0.8, 0.9, 1, 1.1, 1.25, 1.33, 1.5, 2, 2.5, 3, 4]; // Default zoome level (1) const defaultZoomLevel = 7; From 1304c9abf90ce93d7ef76d3a9b2962f9d28dfd4d Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Mon, 8 Jul 2019 13:10:42 +0200 Subject: [PATCH 17/79] make admin/users grid filter on purged and make 'nonpurged' the default filter --- lib/galaxy/web/framework/helpers/grids.py | 21 +++++++++++++++++++ .../webapps/galaxy/controllers/admin.py | 5 ++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/web/framework/helpers/grids.py b/lib/galaxy/web/framework/helpers/grids.py index 08f7603b2c4..6031898535b 100644 --- a/lib/galaxy/web/framework/helpers/grids.py +++ b/lib/galaxy/web/framework/helpers/grids.py @@ -777,6 +777,27 @@ class DeletedColumn(GridColumn): return query +class PurgedColumn(GridColumn): + """ Column that tracks and filters for items with purged attribute. """ + + def get_accepted_filters(self): + """ Returns a list of accepted filters for this column. """ + accepted_filter_labels_and_vals = {"nonpurged" : "False", "purged" : "True", "all": "All"} + accepted_filters = [] + for label, val in accepted_filter_labels_and_vals.items(): + args = {self.key: val} + accepted_filters.append(GridColumnFilter(label, args)) + return accepted_filters + + def filter(self, trans, user, query, column_filter): + """Modify query to filter self.model_class by state.""" + if column_filter == "All": + pass + elif column_filter in ["True", "False"]: + query = query.filter(self.model_class.purged == (column_filter == "True")) + return query + + class StateColumn(GridColumn): """ Column that tracks and filters for items with state attribute. diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py index f023ad177bd..b4addd2f130 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -130,7 +130,8 @@ class UserListGrid(grids.Grid): ActivatedColumn("Activated", attach_popup=False), APIKeyColumn("API Key", attach_popup=False), # Columns that are valid for filtering but are not visible. - grids.DeletedColumn("Deleted", key="deleted", visible=False, filterable="advanced") + grids.DeletedColumn("Deleted", key="deleted", visible=False, filterable="advanced"), + grids.PurgedColumn("Purged", key="purged", visible=False, filterable="advanced") ] columns.append(grids.MulticolFilterColumn("Search", cols_to_filter=[columns[0], columns[1]], @@ -170,6 +171,8 @@ class UserListGrid(grids.Grid): ] num_rows_per_page = 50 use_paging = True + default_filter = dict(purged="False") + use_default_filter = True def get_current_item(self, trans, **kwargs): return trans.user From 753a151c7636b679e84375719ad3b812a3c17a49 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 8 Jul 2019 12:36:36 +0200 Subject: [PATCH 18/79] Add dump tour script to galaxy-selenium package. --- lib/galaxy/selenium/components.py | 2 +- lib/galaxy/selenium/scripts/__init__.py | 0 {scripts => lib/galaxy/selenium/scripts}/dump_tour.py | 4 +--- packages/selenium/setup.py | 2 ++ 4 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 lib/galaxy/selenium/scripts/__init__.py rename {scripts => lib/galaxy/selenium/scripts}/dump_tour.py (91%) diff --git a/lib/galaxy/selenium/components.py b/lib/galaxy/selenium/components.py index 16fa64012dd..2eac23ad8ae 100644 --- a/lib/galaxy/selenium/components.py +++ b/lib/galaxy/selenium/components.py @@ -193,7 +193,7 @@ class Component(object): elif attr in self._text: return self._text[attr] else: - raise KeyError("Failed to find referenced sub-component/selector/label/text [%s]" % attr) + raise AttributeError("Failed to find referenced sub-component/selector/label/text [%s]" % attr) __getitem__ = __getattr__ diff --git a/lib/galaxy/selenium/scripts/__init__.py b/lib/galaxy/selenium/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/dump_tour.py b/lib/galaxy/selenium/scripts/dump_tour.py similarity index 91% rename from scripts/dump_tour.py rename to lib/galaxy/selenium/scripts/dump_tour.py index b13020581cb..5ce5f6c2a31 100755 --- a/scripts/dump_tour.py +++ b/lib/galaxy/selenium/scripts/dump_tour.py @@ -5,9 +5,7 @@ import os import sys import time -sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'test'))) - -from galaxy_selenium import cli +from galaxy.selenium import cli DESCRIPTION = "Walk a Galaxy tour and dump screenshots." diff --git a/packages/selenium/setup.py b/packages/selenium/setup.py index 38720afe67f..a53fe8e6602 100644 --- a/packages/selenium/setup.py +++ b/packages/selenium/setup.py @@ -33,9 +33,11 @@ TEST_DIR = 'tests' PACKAGES = [ 'galaxy', 'galaxy.selenium', + 'galaxy.selenium.scripts', ] ENTRY_POINTS = ''' [console_scripts] + gx-dump-tour=galaxy.selenium.scripts.dump_tour:main ''' PACKAGE_DATA = { # Be sure to update MANIFEST.in for source dist. From 9b287ef06c3a3f4bc72bdd52a59fc0b2ae98016a Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 8 Jul 2019 16:17:56 +0200 Subject: [PATCH 19/79] Push scripts into subpackages. --- packages/app/setup.py | 2 + packages/data/setup.py | 3 +- packages/meta/setup.py | 4 - scripts/build_universe_config.py | 35 --- scripts/check_galaxy.py | 416 ------------------------------- 5 files changed, 4 insertions(+), 456 deletions(-) delete mode 100644 scripts/build_universe_config.py delete mode 100755 scripts/check_galaxy.py diff --git a/packages/app/setup.py b/packages/app/setup.py index 1f1b2fa3b07..61c68cbb3a8 100644 --- a/packages/app/setup.py +++ b/packages/app/setup.py @@ -102,6 +102,8 @@ PACKAGES = [ ] ENTRY_POINTS = ''' [console_scripts] + galaxy-main=galaxy.main:main + galaxy-config=galaxy.config.script:main ''' PACKAGE_DATA = { # Be sure to update MANIFEST.in for source dist. diff --git a/packages/data/setup.py b/packages/data/setup.py index 12556d60ed4..28faa87c3d9 100644 --- a/packages/data/setup.py +++ b/packages/data/setup.py @@ -47,7 +47,8 @@ PACKAGES = [ ] ENTRY_POINTS = ''' [console_scripts] - gx-build-objects=galaxy.model.store.build_objects:main + galaxy-build-objects=galaxy.model.store.build_objects:main + galaxy-manage-db=galaxy.model.orm.scripts:manage_db ''' PACKAGE_DATA = { # Be sure to update MANIFEST.in for source dist. diff --git a/packages/meta/setup.py b/packages/meta/setup.py index ba0cd02ba7b..07e0d7a2d4f 100644 --- a/packages/meta/setup.py +++ b/packages/meta/setup.py @@ -32,10 +32,6 @@ TEST_DIR = 'tests' PACKAGES = [] ENTRY_POINTS = ''' [console_scripts] - galaxy-paster=galaxy.util.pastescript.serve:run - galaxy-main=galaxy.main:main - galaxy-config=galaxy.config.script:main - galaxy-manage-db=galaxy.model.orm.scripts:manage_db ''' PACKAGE_DATA = { # Be sure to update MANIFEST.in for source dist. diff --git a/scripts/build_universe_config.py b/scripts/build_universe_config.py deleted file mode 100644 index 44c15be7c29..00000000000 --- a/scripts/build_universe_config.py +++ /dev/null @@ -1,35 +0,0 @@ -from os import listdir -from os.path import join -from re import match -from sys import argv - -from six.moves.configparser import ConfigParser - - -def merge(): - """ - Merges all .ini files in a specified directory into a file (defaults to - ./config/galaxy.ini ). - """ - if len(argv) < 2: - message = "%s: Must specify directory to merge configuration files from." % argv[0] - raise Exception(message) - conf_directory = argv[1] - conf_files = [f for f in listdir(conf_directory) if match(r'.*\.ini', f)] - conf_files.sort() - - parser = ConfigParser() - for conf_file in conf_files: - parser.read([join(conf_directory, conf_file)]) - # TODO: Expand enviroment variables here, that would - # also make Galaxy much easier to configure. - - destination = "config/galaxy.ini" - if len(argv) > 2: - destination = argv[2] - - parser.write(open(destination, 'w')) - - -if __name__ == '__main__': - merge() diff --git a/scripts/check_galaxy.py b/scripts/check_galaxy.py deleted file mode 100755 index 22f96a6bf0c..00000000000 --- a/scripts/check_galaxy.py +++ /dev/null @@ -1,416 +0,0 @@ -#!/usr/bin/env python -""" -check_galaxy can be run by hand, although it is meant to run from cron -via the check_galaxy.sh script in Galaxy's cron/ directory. -""" -from __future__ import print_function - -import filecmp -import formatter -import getopt -import htmllib -import os -import socket -import sys -import tempfile -import time - -import twill -import twill.commands as tc - -# options -if "DEBUG" in os.environ: - debug = os.environ["DEBUG"] -else: - debug = False -scripts_dir = os.path.abspath(os.path.dirname(sys.argv[0])) -test_data_dir = os.path.join(scripts_dir, os.pardir, "test-data") -# what tools to run - not so pretty -tools = { - "gops_intersect_1": - [ - { - "inputs": - ( - os.path.join(test_data_dir, "1.bed"), - os.path.join(test_data_dir, "2.bed") - ) - }, - {"check_file": os.path.join(test_data_dir, "gops_intersect_out.bed")}, - { - "tool_run_options": - { - "input1": "1.bed", - "input2": "2.bed", - "min": "1", - "returntype": "" - } - } - ] -} - - -# handle arg(s) -def usage(): - sys.exit("usage: check_galaxy.py ") - - -try: - opts, args = getopt.getopt(sys.argv[1:], 'n') -except getopt.GetoptError as e: - print(str(e)) - usage() -if len(args) < 1: - usage() -server = args[0] -if server.endswith(".g2.bx.psu.edu"): - if debug: - print("Checking a PSU Galaxy server, using maint file") - maint = "/errordocument/502/%s/maint" % args[0].split('.', 1)[0] -else: - maint = None -new_history = False -for o, a in opts: - if o == "-n": - if debug: - print("Specified -n, will create a new history") - new_history = True - else: - usage() - -# state information -var_dir = os.path.join(os.path.expanduser('~'), ".check_galaxy", server) -if not os.access(var_dir, os.F_OK): - os.makedirs(var_dir, 0o700) - -# get user/pass -login_file = os.path.join(var_dir, "login") -try: - f = open(login_file, 'r') -except Exception: - message = """Please create the file: -%s -This should contain a username and password to log in to Galaxy with, -on one line, separated by whitespace, e.g.: - -check_galaxy@example.com password - -If the user does not exist, check_galaxy will create it for you.""" % login_file - sys.exit(message) -(username, password) = f.readline().split() - -# default timeout for twill browser is never -socket.setdefaulttimeout(300) - -# user-agent -tc.agent("Mozilla/5.0 (compatible; check_galaxy/0.1)") -tc.config('use_tidy', 0) - - -class Browser(object): - - def __init__(self): - self.server = server - self.maint = maint - self.tool = None - self.tool_opts = None - self.id = None - self.status = None - self.check_file = None - self.hid = None - self.cookie_jar = os.path.join(var_dir, "cookie_jar") - dprint("cookie jar path: %s" % self.cookie_jar) - if not os.access(self.cookie_jar, os.R_OK): - dprint("no cookie jar at above path, creating") - tc.save_cookies(self.cookie_jar) - tc.load_cookies(self.cookie_jar) - - def get(self, path): - tc.go("http://%s%s" % (self.server, path)) - tc.code(200) - - def reset(self): - self.tool = None - self.tool_opts = None - self.id = None - self.status = None - self.check_file = None - self.delete_datasets() - self.get("/root/history") - p = didParser() - p.feed(tc.browser.get_html()) - if len(p.dids) > 0: - print("Remaining datasets ids:", " ".join(p.dids)) - raise Exception("History still contains datasets after attempting to delete them") - if new_history: - self.get("/history/delete_current") - tc.save_cookies(self.cookie_jar) - - def check_redir(self, url): - try: - tc.get_browser()._browser.set_handle_redirect(False) - tc.go(url) - tc.code(302) - tc.get_browser()._browser.set_handle_redirect(True) - dprint("%s is returning redirect (302)" % url) - return(True) - except twill.errors.TwillAssertionError as e: - tc.get_browser()._browser.set_handle_redirect(True) - dprint("%s is not returning redirect (302): %s" % (url, e)) - code = tc.browser.get_code() - if code == 502: - is_maint = self.check_maint() - if is_maint: - dprint("Galaxy is down, but a maint file was found, so not sending alert") - sys.exit(0) - else: - sys.exit("Galaxy is down (code 502)") - return(False) - - # checks for a maint file - def check_maint(self): - if self.maint is None: - # dprint( "Warning: unable to check maint file for %s" % self.server ) - return(False) - try: - self.get(self.maint) - return(True) - except twill.errors.TwillAssertionError: - return(False) - - def login(self, user, pw): - self.get("/user/login") - tc.fv("1", "email", user) - tc.fv("1", "password", pw) - tc.submit("Login") - tc.code(200) - if len(tc.get_browser().get_all_forms()) > 0: - # uh ohs, fail - p = userParser() - p.feed(tc.browser.get_html()) - if p.no_user: - dprint("user does not exist, will try creating") - self.create_user(user, pw) - elif p.bad_pw: - raise Exception("Password is incorrect") - else: - raise Exception("Unknown error logging in") - tc.save_cookies(self.cookie_jar) - - def create_user(self, user, pw): - self.get("/user/create") - tc.fv("1", "email", user) - tc.fv("1", "password", pw) - tc.fv("1", "confirm", pw) - tc.submit("Submit") - tc.code(200) - if len(tc.get_browser().get_all_forms()) > 0: - p = userParser() - p.feed(tc.browser.get_html()) - if p.already_exists: - raise Exception('The user you were trying to create already exists') - - def upload(self, file): - self.get("/tool_runner/index?tool_id=upload1") - tc.fv("1", "file_type", "bed") - tc.formfile("1", "file_data", file) - tc.submit("runtool_btn") - tc.code(200) - - def runtool(self): - self.get("/tool_runner/index?tool_id=%s" % self.tool) - for k, v in self.tool_opts.items(): - tc.fv("1", k, v) - tc.submit("runtool_btn") - tc.code(200) - - def wait(self): - sleep_amount = 1 - count = 0 - maxiter = 16 - while count < maxiter: - count += 1 - self.get("/root/history") - page = tc.browser.get_html() - if page.find('') > -1: - time.sleep(sleep_amount) - sleep_amount += 1 - else: - break - if count == maxiter: - raise Exception("Tool never finished") - - def check_status(self): - self.get("/root/history") - p = historyParser() - p.feed(tc.browser.get_html()) - if p.status != "ok": - raise Exception("JOB %s NOT OK: %s" % (p.id, p.status)) - self.id = p.id - self.status = p.status - # return((p.id, p.status)) - - def diff(self): - self.get("/datasets/%s/display/display?to_ext=bed" % self.id) - data = tc.browser.get_html() - tmp = tempfile.mkstemp() - dprint("tmp file: %s" % tmp[1]) - tmpfh = os.fdopen(tmp[0], 'w') - tmpfh.write(data) - tmpfh.close() - if filecmp.cmp(tmp[1], self.check_file): - dprint("Tool output is as expected") - else: - if not debug: - os.remove(tmp[1]) - raise Exception("Tool output differs from expected") - if not debug: - os.remove(tmp[1]) - - def delete_datasets(self): - self.get("/root/history") - p = didParser() - p.feed(tc.browser.get_html()) - dids = p.dids - for did in dids: - self.get("/datasets/%s/delete" % did) - - def check_if_logged_in(self): - self.get("/user?cntrller=user") - p = loggedinParser() - p.feed(tc.browser.get_html()) - return p.logged_in - - -class userParser(htmllib.HTMLParser): - def __init__(self): - htmllib.HTMLParser.__init__(self, formatter.NullFormatter()) - self.in_span = False - self.in_div = False - self.no_user = False - self.bad_pw = False - self.already_exists = False - - def start_span(self, attrs): - self.in_span = True - - def start_div(self, attrs): - self.in_div = True - - def end_span(self): - self.in_span = False - - def end_div(self): - self.in_div = False - - def handle_data(self, data): - if self.in_span or self.in_div: - if data == "No such user (please note that login is case sensitive)": - self.no_user = True - elif data == "Invalid password": - self.bad_pw = True - elif data == "User with that email already exists": - self.already_exists = True - - -class historyParser(htmllib.HTMLParser): - def __init__(self): - htmllib.HTMLParser.__init__(self, formatter.NullFormatter()) - self.status = None - self.id = None - - def start_div(self, attrs): - # find the top history item - for i in attrs: - if i[0] == "class" and i[1].startswith("historyItemWrapper historyItem historyItem-"): - self.status = i[1].rsplit("historyItemWrapper historyItem historyItem-", 1)[1] - dprint("status: %s" % self.status) - if i[0] == "id" and i[1].startswith("historyItem-"): - self.id = i[1].rsplit("historyItem-", 1)[1] - dprint("id: %s" % self.id) - if self.status is not None: - self.reset() - - -class didParser(htmllib.HTMLParser): - def __init__(self): - htmllib.HTMLParser.__init__(self, formatter.NullFormatter()) - self.dids = [] - - def start_div(self, attrs): - for i in attrs: - if i[0] == "id" and i[1].startswith("historyItemContainer-"): - self.dids.append(i[1].rsplit("historyItemContainer-", 1)[1]) - dprint("got a dataset id: %s" % self.dids[-1]) - - -class loggedinParser(htmllib.HTMLParser): - def __init__(self): - htmllib.HTMLParser.__init__(self, formatter.NullFormatter()) - self.in_p = False - self.logged_in = False - - def start_p(self, attrs): - self.in_p = True - - def end_p(self): - self.in_p = False - - def handle_data(self, data): - if self.in_p: - if data == "You are currently not logged in.": - self.logged_in = False - elif data.startswith("You are currently logged in as "): - self.logged_in = True - - -def dprint(str): - if debug: - print(str) - - -if __name__ == "__main__": - dprint("checking %s" % server) - - b = Browser() - - # login (or not) - if b.check_if_logged_in(): - dprint("we are already logged in (via cookies), hooray!") - else: - dprint("not logged in... logging in") - b.login(username, password) - - for tool, params in tools.items(): - - check_file = "" - - # make sure history and state is clean - b.reset() - b.tool = tool - - # get all the tool run conditions - for dict in params: - for k, v in dict.items(): - if k == 'inputs': - for file in v: - b.upload(file) - elif k == 'check_file': - b.check_file = v - elif k == 'tool_run_options': - b.tool_opts = v - else: - raise Exception("Unknown key in tools dict: %s" % k) - - b.runtool() - b.wait() - b.check_status() - b.diff() - b.delete_datasets() - - # by this point, everything else has succeeded. there should be no maint. - is_maint = b.check_maint() - if is_maint: - sys.exit("Galaxy is up and fully functional, but a maint file is in place.") - - sys.exit(0) From 6e78cc6847dd6bb72ebcabd361ba6b159d3fb9a6 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 8 Jul 2019 17:59:25 +0200 Subject: [PATCH 20/79] Fix bug in a unit test There was a bug in a context manager inside a unit test. This caused nondeterministic behavior of some unit tests after a refactoring. Bug fixed; added 3 unit tests to test the context manager. --- test/unit/tools/test_tool_deps.py | 56 +++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/test/unit/tools/test_tool_deps.py b/test/unit/tools/test_tool_deps.py index c28818bd526..1a4efa6c09a 100644 --- a/test/unit/tools/test_tool_deps.py +++ b/test/unit/tools/test_tool_deps.py @@ -593,7 +593,7 @@ def test_config_MODULEPATH(): def test_config_MODULESHOME(): # Test fallbacks to read MODULESHOME if modulepath is not specified and # neither is MODULEPATH. - with __environ({"MODULESHOME": "/opt/modules"}, remove="MODULEPATH"): + with __environ({"MODULESHOME": "/opt/modules"}, keys_to_remove=["MODULEPATH"]): with __parse_resolvers(''' @@ -719,25 +719,69 @@ def _first_conda_resolver_options(dm): @contextmanager -def __environ(values, remove=[]): +def __environ(values, keys_to_remove=[]): """ Modify the environment for a test, adding/updating values in dict `values` and removing any environment variables mentioned in list `remove`. """ - new_keys = set(environ.keys()) - set(values.keys()) + new_keys = set(values.keys()) - set(environ.keys()) old_environ = environ.copy() try: environ.update(values) - for to_remove in remove: + for key in keys_to_remove: try: - del environ[remove] + del environ[key] + new_keys.remove(key) # Because key no longer exists except KeyError: pass yield finally: - environ.update(old_environ) for key in new_keys: del environ[key] + environ.update(old_environ) + + +def test_environ_contextmanager_env_restored(): + """ os.environ must be preserved across calls to __environ """ + key, val = 'foo_test_678363', '42' + os.environ[key] = val + with __environ({}, []): + pass + assert os.environ[key] == val + del os.environ[key] + + +def test_environ_contextmanager_env_updated_and_restored(): + """ + os.environ must be preserved across calls to __environ + when updating existing or adding new key:value pairs + """ + key, val = 'foo_test_678363', '42' + os.environ[key] = val + new_key, new_val1 = 'foo_test_new', '53' # add new key + value + new_val2 = '43' # update old key's value + to_update = { new_key: new_val1, key: new_val2 } + with __environ(values=to_update): + pass + assert os.environ[key] == val # value should not change + assert new_key not in os.environ # key should not be added + del os.environ[key] + + +def test_environ_contextmanager_env_removed_and_restored(): + """ + os.environ must be preserved across calls to __environ + when removing key:value pairs + """ + key, val = 'foo_test_678363', '42' + os.environ[key] = val + new_key, new_val = 'foo_test_new', '53' # add new key + value + to_update = { new_key: new_val } + with __environ(values=to_update, keys_to_remove=[key, new_key]): + pass + assert os.environ[key] == val # key should be present w/value + assert new_key not in os.environ # key should not be added + del os.environ[key] @contextmanager From ce84ee7f3dfa591c8ce2eb07f5c98b03827c4c4c Mon Sep 17 00:00:00 2001 From: guerler Date: Mon, 8 Jul 2019 22:59:42 +0200 Subject: [PATCH 21/79] Fix indention in base styles --- client/galaxy/style/scss/base.scss | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/client/galaxy/style/scss/base.scss b/client/galaxy/style/scss/base.scss index 74b47b27bcb..ac8f6eb205b 100644 --- a/client/galaxy/style/scss/base.scss +++ b/client/galaxy/style/scss/base.scss @@ -1507,26 +1507,26 @@ div.toolSectionTitle, div.toolTitle, div.toolTitleNoSection { } a { - @extend .px-3; - @extend .py-1; - text-decoration: none; - display: block; - &:hover { + @extend .px-3; + @extend .py-1; + text-decoration: none; + display: block; + &:hover { background: darken($panel-bg-color, 5%); - } + } } &.text-muted a { - &:hover { - background: inherit; - } + &:hover { + background: inherit; + } } } div.toolSectionWrapper { - div.toolTitle a, div.toolPanelLabel { - @extend .pl-4; - font-size: inherit; + div.toolTitle a, div.toolPanelLabel { + @extend .pl-4; + font-size: inherit; } } From 74c1f68e7161cfdfe780fdfa821efaa9c4e5f34f Mon Sep 17 00:00:00 2001 From: guerler Date: Mon, 8 Jul 2019 23:08:40 +0200 Subject: [PATCH 22/79] Fix indention for admin panel sections --- client/galaxy/scripts/entry/panels/admin-panel.js | 2 +- client/galaxy/style/scss/base.scss | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/client/galaxy/scripts/entry/panels/admin-panel.js b/client/galaxy/scripts/entry/panels/admin-panel.js index e67252912a6..9ae48429d2a 100644 --- a/client/galaxy/scripts/entry/panels/admin-panel.js +++ b/client/galaxy/scripts/entry/panels/admin-panel.js @@ -181,7 +181,7 @@ const AdminPanel = Backbone.View.extend({ _templateSection: function(options) { return `
-
${_l(options.title)}
+
${_l(options.title)}
`; }, diff --git a/client/galaxy/style/scss/base.scss b/client/galaxy/style/scss/base.scss index ac8f6eb205b..8e2fd2d86af 100644 --- a/client/galaxy/style/scss/base.scss +++ b/client/galaxy/style/scss/base.scss @@ -1486,9 +1486,6 @@ div.permissionContainer { div.toolSectionTitle { font-weight: 500; font-size: $h4-font-size; - &:hover { - background: darken($panel-bg-color, 5%); - } } div.toolPanelLabel { From 9cd3f2aa0bdd691eae054cdbe4daeff509264291 Mon Sep 17 00:00:00 2001 From: guerler Date: Mon, 8 Jul 2019 23:13:31 +0200 Subject: [PATCH 23/79] Fix additional indentions in base styles --- client/galaxy/style/scss/base.scss | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/client/galaxy/style/scss/base.scss b/client/galaxy/style/scss/base.scss index 8e2fd2d86af..398c5acf815 100644 --- a/client/galaxy/style/scss/base.scss +++ b/client/galaxy/style/scss/base.scss @@ -1477,7 +1477,7 @@ div.permissionContainer { .toolMenuContainer { color: $panel-text-color; a { - color: $panel-text-color; + color: $panel-text-color; } background: $panel-bg-color; min-height: 100%; @@ -1502,7 +1502,6 @@ div.toolSectionTitle, div.toolTitle, div.toolTitleNoSection { .labels { float: right; } - a { @extend .px-3; @extend .py-1; @@ -1512,7 +1511,6 @@ div.toolSectionTitle, div.toolTitle, div.toolTitleNoSection { background: darken($panel-bg-color, 5%); } } - &.text-muted a { &:hover { background: inherit; @@ -1521,7 +1519,7 @@ div.toolSectionTitle, div.toolTitle, div.toolTitleNoSection { } div.toolSectionWrapper { - div.toolTitle a, div.toolPanelLabel { + div.toolTitle a, div.toolPanelLabel { @extend .pl-4; font-size: inherit; } From d96769d311936c4857fd84e4149e9a5ac17081d8 Mon Sep 17 00:00:00 2001 From: guerler Date: Mon, 8 Jul 2019 23:23:32 +0200 Subject: [PATCH 24/79] Remove indention from tool panel labels --- client/galaxy/style/scss/base.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/galaxy/style/scss/base.scss b/client/galaxy/style/scss/base.scss index 398c5acf815..c048084f93d 100644 --- a/client/galaxy/style/scss/base.scss +++ b/client/galaxy/style/scss/base.scss @@ -1519,7 +1519,7 @@ div.toolSectionTitle, div.toolTitle, div.toolTitleNoSection { } div.toolSectionWrapper { - div.toolTitle a, div.toolPanelLabel { + div.toolTitle a { @extend .pl-4; font-size: inherit; } From 73cd69a13fdb72866b02ab0d988f9d04b9d81461 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 9 Jul 2019 00:39:21 +0200 Subject: [PATCH 25/79] Fix lint errors --- test/unit/tools/test_tool_deps.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test/unit/tools/test_tool_deps.py b/test/unit/tools/test_tool_deps.py index 1a4efa6c09a..c8f3c5366e9 100644 --- a/test/unit/tools/test_tool_deps.py +++ b/test/unit/tools/test_tool_deps.py @@ -731,7 +731,7 @@ def __environ(values, keys_to_remove=[]): for key in keys_to_remove: try: del environ[key] - new_keys.remove(key) # Because key no longer exists + new_keys.remove(key) # Because key no longer exists except KeyError: pass yield @@ -752,35 +752,35 @@ def test_environ_contextmanager_env_restored(): def test_environ_contextmanager_env_updated_and_restored(): - """ - os.environ must be preserved across calls to __environ + """ + os.environ must be preserved across calls to __environ when updating existing or adding new key:value pairs """ key, val = 'foo_test_678363', '42' os.environ[key] = val - new_key, new_val1 = 'foo_test_new', '53' # add new key + value - new_val2 = '43' # update old key's value - to_update = { new_key: new_val1, key: new_val2 } + new_key, new_val1 = 'foo_test_new', '53' # add new key + value + new_val2 = '43' # update old key's value + to_update = {new_key: new_val1, key: new_val2} with __environ(values=to_update): pass - assert os.environ[key] == val # value should not change - assert new_key not in os.environ # key should not be added + assert os.environ[key] == val # value should not change + assert new_key not in os.environ # key should not be added del os.environ[key] def test_environ_contextmanager_env_removed_and_restored(): - """ - os.environ must be preserved across calls to __environ + """ + os.environ must be preserved across calls to __environ when removing key:value pairs """ key, val = 'foo_test_678363', '42' os.environ[key] = val - new_key, new_val = 'foo_test_new', '53' # add new key + value - to_update = { new_key: new_val } + new_key, new_val = 'foo_test_new', '53' # add new key + value + to_update = {new_key: new_val} with __environ(values=to_update, keys_to_remove=[key, new_key]): pass - assert os.environ[key] == val # key should be present w/value - assert new_key not in os.environ # key should not be added + assert os.environ[key] == val # key should be present w/value + assert new_key not in os.environ # key should not be added del os.environ[key] From 34bde631f42c82d53d8bc6de9d2b2d1e964f9be9 Mon Sep 17 00:00:00 2001 From: Kiran Telukunta saraswathi Date: Mon, 8 Jul 2019 08:56:35 +0200 Subject: [PATCH 26/79] * walltime_limit added for the k8s jobs * default is also set to the k8s_walltime_limit in job specification --- lib/galaxy/jobs/runners/kubernetes.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 1dba8c47281..fadfff4fc0e 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -58,7 +58,8 @@ class KubernetesJobRunner(AsynchronousJobRunner): k8s_default_limits_cpu=dict(map=str, default=None), k8s_default_limits_memory=dict(map=str, default=None), k8s_pod_retries=dict(map=int, valid=lambda x: int >= 0, default=3), - k8s_pod_retrials=dict(map=int, valid=lambda x: int >= 0, default=3)) + k8s_pod_retrials=dict(map=int, valid=lambda x: int >= 0, default=3), + k8s_walltime_limit=dict(map=int, valid=lambda x: int(x) >= 0, default=172800)) if 'runner_param_specs' not in kwargs: kwargs['runner_param_specs'] = dict() @@ -220,8 +221,10 @@ class KubernetesJobRunner(AsynchronousJobRunner): return produce_unique_k8s_job_name(app_prefix='galaxy', instance_id=instance_id, job_id=galaxy_internal_job_id) def __get_k8s_job_spec(self, ajs): - """Creates the k8s Job spec. For a Job spec, the only requirement is to have a .spec.template.""" - k8s_job_spec = {"template": self.__get_k8s_job_spec_template(ajs)} + """Creates the k8s Job spec. For a Job spec, the only requirement is to have a .spec.template. + If the job hangs around unlimited it will be ended after k8s wall time limit, which sets activeDeadlineSeconds""" + k8s_job_spec = {"template": self.__get_k8s_job_spec_template(ajs), + "activeDeadlineSeconds": int(self.runner_params['k8s_walltime_limit'])} return k8s_job_spec def __get_k8s_job_spec_template(self, ajs): From 8fb5936f92641e8dc0d673274ce1cf0eb502b2b5 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 10 Jul 2019 13:49:50 +0200 Subject: [PATCH 27/79] Fix kubernetes runner on py3 instance_id is a string, so this can't be compared to an integer on python 3. Broken in ac1b25ed4c069abf1c8d810ba26d5be43717766d, prior to that the comparison was `len(self._galaxy_instance_id) > 0:` --- lib/galaxy/jobs/runners/util/pykube_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/runners/util/pykube_util.py b/lib/galaxy/jobs/runners/util/pykube_util.py index a86601c324f..0203b9f5046 100644 --- a/lib/galaxy/jobs/runners/util/pykube_util.py +++ b/lib/galaxy/jobs/runners/util/pykube_util.py @@ -47,7 +47,7 @@ def produce_unique_k8s_job_name(app_prefix=None, instance_id=None, job_id=None): if app_prefix: job_name += "%s-" % app_prefix - if instance_id and instance_id > 0: + if instance_id and len(instance_id) > 0: job_name += "%s-" % instance_id return job_name + job_id From b2cd5ba8d052d22ef29b3725e8d46052a3b5f3a0 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 10 Jul 2019 15:02:59 +0200 Subject: [PATCH 28/79] Add k8s_walltime_limit to job_conf.xml.sample_advanced --- lib/galaxy/config/sample/job_conf.xml.sample_advanced | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/galaxy/config/sample/job_conf.xml.sample_advanced b/lib/galaxy/config/sample/job_conf.xml.sample_advanced index 471946d0e3a..ebb74e9514d 100644 --- a/lib/galaxy/config/sample/job_conf.xml.sample_advanced +++ b/lib/galaxy/config/sample/job_conf.xml.sample_advanced @@ -216,6 +216,11 @@ zero (no execution) and the stderr/stdout of the k8s job is reported in galaxy (and the galaxy job set to failed) --> + + + + + value, name, url + +
diff --git a/tool-data/icn3d_simple_display.loc.sample b/tool-data/icn3d_simple_display.loc.sample new file mode 100644 index 00000000000..a0d3d91277b --- /dev/null +++ b/tool-data/icn3d_simple_display.loc.sample @@ -0,0 +1,3 @@ +# Table used for listing simple iCn3D Structure Viewer servers +# +ncbi_icn3d NCBI https://www.ncbi.nlm.nih.gov/Structure/icn3d/full.html?type=%(icn3d_file_type)s&url=%(icn3d_file_url_qp)s From 8dd9b2fe8c678fcab05054e4bf3669baa195067b Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Wed, 17 Jul 2019 09:58:17 -0400 Subject: [PATCH 64/79] Missing space --- display_applications/icn3d/icn3d_simple.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/display_applications/icn3d/icn3d_simple.xml b/display_applications/icn3d/icn3d_simple.xml index 1d41a423708..25e182be0fd 100644 --- a/display_applications/icn3d/icn3d_simple.xml +++ b/display_applications/icn3d/icn3d_simple.xml @@ -1,5 +1,5 @@ - + ${ url % { 'icn3d_file_type': $icn3d_file.ext, 'icn3d_file_url_qp': $icn3d_file.qp } } From 0b90afcfcbd14a5bfa67746f1c32cd94ad0dd0a0 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Wed, 17 Jul 2019 11:54:15 -0400 Subject: [PATCH 65/79] Enable iCn3D for mol2 and sdf files --- lib/galaxy/config/sample/datatypes_conf.xml.sample | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index 37b5df825e0..8293795508a 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -593,6 +593,7 @@ + @@ -613,6 +614,7 @@ + From e8c017892fca3ac83186f96858afcca4696de2eb Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 17 Jul 2019 19:49:46 -0400 Subject: [PATCH 66/79] Add links to admin entry page This is a temporary solution. Ideally, this should be generated using the same logic that is used for generating the links on the left menu panel. And no page reloads, of course. --- templates/webapps/galaxy/admin/center.mako | 26 +++++++++++----------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/templates/webapps/galaxy/admin/center.mako b/templates/webapps/galaxy/admin/center.mako index 5a8bd5df23e..9ca501692e0 100644 --- a/templates/webapps/galaxy/admin/center.mako +++ b/templates/webapps/galaxy/admin/center.mako @@ -11,41 +11,41 @@ Please visit the Galax

Server

  • - Data types - See all datatypes available in this Galaxy. + Data types - See all datatypes available in this Galaxy.
  • - Data tables - See all data tables available in this Galaxy. + Data tables - See all data tables available in this Galaxy.
  • - Display applications - See all display applications configured in this Galaxy. + Display applications - See all display applications configured in this Galaxy.
  • - Manage jobs - Display all jobs that are currently not finished (i.e., their state is new, waiting, queued, or running). Administrators are able to cleanly stop long-running jobs. + Manage jobs - Display all jobs that are currently not finished (i.e., their state is new, waiting, queued, or running). Administrators are able to cleanly stop long-running jobs.
  • - Local data - Manage the reference (and other) data that is stored within Tool Data Tables. See wiki for details. + Local data - Manage the reference (and other) data that is stored within Tool Data Tables. See wiki for details.

User Management

  • - Users - The primary user management interface, displaying information associated with each user and providing operations for resetting passwords, updating user information, impersonating a user, and more. + Users - The primary user management interface, displaying information associated with each user and providing operations for resetting passwords, updating user information, impersonating a user, and more.
    • %if trans.app.config.enable_quotas:
  • - Quotas - Manage user space quotas. See wiki for details. + Quotas - Manage user space quotas. See wiki for details.
    • %endif
  • - Groups - A view of all groups along with the members of the group and the roles associated with each group. + Groups - A view of all groups along with the members of the group and the roles associated with each group.
  • - Roles - A view of all non-private roles along with the role type, and the users and groups that are associated with the role. + Roles - A view of all non-private roles along with the role type, and the users and groups that are associated with the role. Also includes a view of the data library datasets that are associated with the role and the permissions applied to each dataset.
  • - Forms - Manage local form definitions. + Forms - Manage local form definitions.
@@ -53,7 +53,7 @@ Please visit the Galax %endif From a70595a23e09b002946e85bf69e9910dd48448a9 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 18 Jul 2019 16:44:08 -0400 Subject: [PATCH 67/79] Better separation for integration tests. --- test/base/driver_util.py | 5 +++++ test/base/integration_util.py | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/test/base/driver_util.py b/test/base/driver_util.py index f4591e2812a..4a140ecc339 100644 --- a/test/base/driver_util.py +++ b/test/base/driver_util.py @@ -933,6 +933,11 @@ class GalaxyTestDriver(TestDriver): galaxy_db_path, **setup_galaxy_config_kwds ) + + isolate_galaxy_config = getattr(config_object, "isolate_galaxy_config", False) + if isolate_galaxy_config: + galaxy_config["config_dir"] = tempdir + self._saved_galaxy_config = galaxy_config if galaxy_config is not None: diff --git a/test/base/integration_util.py b/test/base/integration_util.py index 699c98888cc..37a16a33383 100644 --- a/test/base/integration_util.py +++ b/test/base/integration_util.py @@ -60,6 +60,10 @@ class IntegrationInstance(UsesApiTestCaseMixin): # Subclasses can override this to force uwsgi for tests. require_uwsgi = False + # Don't pull in default configs for un-configured things from Galaxy's + # config directory and such. + isolate_galaxy_config = True + @classmethod def setUpClass(cls): """Configure and start Galaxy for a test.""" From bbf17354c5b53b303d94a4257044e54a6a1d358d Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 19 Jul 2019 02:35:24 +0100 Subject: [PATCH 68/79] Fix broken link in ``galaxy-app`` package Broken in commit 42cf61fd3608e24e74837da6706d506dd276db36 . --- packages/app/galaxy/config | 1 + packages/app/galaxy/config.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 120000 packages/app/galaxy/config delete mode 120000 packages/app/galaxy/config.py diff --git a/packages/app/galaxy/config b/packages/app/galaxy/config new file mode 120000 index 00000000000..ce9417b1a26 --- /dev/null +++ b/packages/app/galaxy/config @@ -0,0 +1 @@ +../../../lib/galaxy/config \ No newline at end of file diff --git a/packages/app/galaxy/config.py b/packages/app/galaxy/config.py deleted file mode 120000 index 7e61acb3d31..00000000000 --- a/packages/app/galaxy/config.py +++ /dev/null @@ -1 +0,0 @@ -../../../lib/galaxy/config.py \ No newline at end of file From 7be55d36f2cff64c0e8aa33cd72827da6d66924f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 19 Jul 2019 16:33:19 +0200 Subject: [PATCH 69/79] Fix default involucro path BaseAppConfiguration.root is now an absolute path, and so is BaseAppConfiguration.data_dir, so the result of `resolve_path(kwargs.get("tool_dependency_dir", "dependencies"), self.data_dir)` is not "none" as it used to be, but `"/none"` and so the fallback will not be used. This can be fixed by setting the default target dir to `dependencies` and join it to self.data_dir, which defaults to `galaxy_root/database` on classic installations. Fixes the issues in https://github.com/galaxyproject/galaxy/pull/8195#issuecomment-513246346 --- lib/galaxy/config/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 1df467d617a..a48e9c661a0 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -556,9 +556,10 @@ class GalaxyAppConfiguration(BaseAppConfiguration): involucro_path = kwargs.get('involucro_path', None) if involucro_path is None: - target_dir = resolve_path(kwargs.get("tool_dependency_dir", "dependencies"), self.data_dir) + target_dir = kwargs.get("tool_dependency_dir", "dependencies") if target_dir == "none": - target_dir = "database" + target_dir = "dependencies" + target_dir = resolve_path(target_dir, self.data_dir) involucro_path = os.path.join(target_dir, "involucro") self.involucro_path = resolve_path(involucro_path, self.root) self.involucro_auto_init = string_as_bool(kwargs.get('involucro_auto_init', True)) From 31869d385d7ddde1ab0b9e90a84d1d8ba00f6b31 Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Sun, 21 Jul 2019 00:08:12 +0200 Subject: [PATCH 70/79] Fix zipped files autodetection --- lib/galaxy/datatypes/test/1.excel.xlsx | Bin 0 -> 6105 bytes lib/galaxy/util/compression_utils.py | 6 ++++-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 lib/galaxy/datatypes/test/1.excel.xlsx diff --git a/lib/galaxy/datatypes/test/1.excel.xlsx b/lib/galaxy/datatypes/test/1.excel.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..f7c85d3305d126f958c9ddf4eaebc8065f97bbaa GIT binary patch literal 6105 zcmaJ_bzD^Kwx+wJ8$m!67`i*8V*qJ}5QZGOQz_{IQP4peNy(A!ZX|^v1r#JjsSyyk z!}r~D^m@1{y^{7Tc79^O`WZ~4g)Zl z6M*+7o=?mjwX2=%2q*2t_9_wKl;vYZclG(v8{W+8vuN`ZJn zqt0jc?nvAe@TfaQ3iM`3L#K3Zs5x?9k8k8qeL53;jLM+G9JAJq8`$FAeO+xkM@(W> z5S+t5NQ^(`#r#ZYQa}08DmR}X?y>9HhRlW>8x!Kra8+cEzNkO`2P=E#gCIh8{KX1j z*xpa}qr(Q+gNmw^lRz|Q1n5^&V zkV_TRhn;EY`Oz1Z9%#7%q-bc(F%u1zNeC`;Iy_-z$Yx)a&cG01$lzpZ`W$lVww{R_ zPA{hIe6Tnz>>H>}+!URg1GG>wl4#mDjQQk{2O*DjS_t)eVT!bPJVR|`>%PVXbJ$jxO_XbH;<2~lOIB-fQ#lVtG1Lsux@(<59@i&`* zUJyg}nv;v?o)$RtQYJ@)Clt4$18!@(^7g#D%cbq2yc-xYS7joG(cavc(_>-UNNSW8 z4}tWFNFl0E?IZ}iB${cvh#Bt!Rzl>hwqPVO`1yOYr>hf+uEU7q3%_;0g7lVfZ*)*usuf^av7&+t6wV zw=??8Y*Romq+*Ht4P+`|23e=>Uoo!2gg2$y1oXMv+3vIFp!3v{B`lgO5#kG`_WD|U zW66(j5dY~z4sLri(*D(MntHOXZQ#_(wdk<;)g}5(WVV)y=9-|Q(gEk6B9r#-$h^r` zZ$~iL$D8l3lfX@Mw!s$N7RAY)9w6^wEd$CheF2O#bxe1Or-~4@=GF(43DD@_{Pg@R zPTdZ6j*-K}b-VDHHZ_i&IUH#anq9)CH``wC(>iI*7#tBO4^JJ{RqC(L%33seWcU432uo$r(%+*ha-Ga7Q;CHLZ8 zW%w5)_k5pAGi8?gvz6tk2msdD`N1+onKf%kw(`Jal5TKIaG6rkSE1Q9WUf6;Mpj5O zZB8-6F3n4z@Il_Xa8mSCSn8?LN8xN*^N)ZjMf)M&%#}-;OG${~S*s4qyh?FlXj%LQ zv?ixaeco3%E|tE+?JXNkSr@9}f)K2a=(iO=n>(3Ntuj$WRDAFTy;7X~v=^72k))Fp zVdUiL;^bh%vVPURove4g_=s6>8hLijGY^VBuJTV4noHx=tik3OSx>#cB}m*Gj{!3_ zC2c9H$1sUJ&v0{MwVIHMD~udx!2DEjc&I%1Yr}^D&M^bIr!{ z2LJ5_z$NQTXd1iwfuJJ@CrcCVziq1U;}p;{3q~{@p910@tT*`6)d31vU#_{^&@})) zv8JOgV6}kf=r7sNc=ej!yWRJAniU_g3!fxH3+kRgEzAS^Bt?jIU-S4aGFyTAYG+{q~@H6+1R>UkMXf zaXVLdkWd+vqlwywKKRrp^)OS6V9KENj+Yy2S01XAwYC=`G=3_5=zBsy+YDh>sG9x^ z{Ei=kEg2f!<5k``jYxIEP0ys~LAQSCC70@#pZql^b<*qh9=?K2@Qe@=>IWxX6!H@qi9YQk*&V&hW@3EnM~|J3R7D_!g_%((<2WaCzsB(2{NE;yZ7* zc9x;Rnu#pmK<`8fNt0+Pve+|cX9Nah22KdzDrhSbQC99^k`GFUU>}j`eorSQ8dGYu zG-qAvw|VBy;cWPfW| z8*g`Ax_c4X*}%y6bT=p}MTMH*PvA}5su3XQmJ|swU{_pF{Sr1WBP(+9L;R$qfFwVo zm-b!?>d#YxeiG~mSkA>MW2X!RP>zIj5lb6IQ-@D{Liu}eHMu^k8|TS$156z#CRD1^ z+CfO2U#8Ksha5x94IhWdVfzS+bQkD0i1$QH_glO66;BM|D0X^Y&c;wel&52g$03o; z$>nyhm2u{$#OF|fC1c$MV^Xq+ zcwSQLW{9DxTo(~mtZerx8m_%(3$mhOJ@B5}{ZK*2a>p{LFjCbUd0{Jo@X~yMlu7yi zGcJvdx2Tr#1+tyl?H`g@#{Wl3YEF*gZ_i2`4WzW9X z`iC^-J`e!A_y(B2r5Mz&cXgJGX5&wh&(F?8JHdSPrwn;Q%G_rG(YG#SV2Q!|nP*gr-Ky#t%wGqd6 z22k#jZEim1_c4uUoRyTD$*sl|xGg~*Y#!GS-^iyV&u)Gyy&{|ws9kFS@TTVZ==fN~ zvj6BZZyxB^tpVT#7Np6B4YeylXKuYr4a8QBwO=dIJ;PMI1J`L}Z zx%R#_8r%>SGaHV|ldveerAK+z!M5lJOqPCkxwdUuqR2v_!Yyevuj(M1fWd9HYA4Qg z;^2SVkBSpQKlW&ob)0!Yfdj%gRi{szP#LSVHZw#BY@0238u1po^!~i6cuBsY)s6j! zvF6EQSHsd!J6~)(1&RLGglaYyTYCVP_16SM+LmYObtW|%8PP`Zd2X!1OsX(2YzkYU z9B-fe@#1WsRiv^`Ij#+DUA0GdGW9Z&jkk2@O4Cx`4o6Z!!d2kFTDd!{HK+Hvk7V3s z02T2iS2NQE%hQ6Ij;3XZW9qhKmTOq*d&Zs2?AikN$qC&fw2f1(7Fpj6a|sP}u&?_7 z&lU&#C-zP7T+VJjDNt-F*Tf;J-loz6r?vH9uwO$Exb3mwsl1>d$MJ zpOm|#Ig!3D<4EJIFxZGdf7`Fp@1*qj5<-M^URv}#T}^r(d7i!dDUy#?>eBv=&_vHe%))6mo zNd*&6tC@nC>%Nsh4~Cwj481f2cQLsBQu%WUQ()62Gb9wTFcmwJrMu9IsAQxmwDp$; z^vha(-6}U~LL0}IFt4=5OX~54OL;_=WQ?WA3%BmO} zZCBOkT=XIbR5ambFPt}-%Pnr-=2|D3JerAS&8K@>xBKEFe0W=eGDe?+nmyi^JJ5NX zUH?$o_kLpbf?cJQa}bMxp*&CB>&r2jz_!6{pRYjPk;GQMB6xn4~vt32; zy5L%?-#Po^m{YrF3QI*0)GCS;CU(~P?7WC+M{3^ag=^yr*Kd;ZSD-9NwsX}yq|F97 zcUh0I?`LM5!&hRT_VGi%ujwv>d2o396Q4EG&?8NTZqOKhS`^(#5T5v&dPD}tGTNRT zK3d1VoQUHLMiMjjSaj#;xXY*lp-=$HbjEIVv{V3*|I_HIYz1NoXNA!RTl!jHo`b8l zj+-RO1*wVbpprx$H7We}0hbV!B!AESf6w}<;~JO(WHAS!@hQRn^;F7s@7iJ~UaN&Z zc{O@vBL?HA^9>D-#4{@`#7$NYI~o&H%2+11KZ(y&jaP9mGviK#{-g~q$RR=QGPJoE zkKK81q`^*VJ6tx?3pu`>>V^(y;73^Kb(>g9u3Hx}HpSR2+XY9tf~+vF?Y2&1OHWrE zS1yu_VvtcUt4nuPV*Ec_VyH+=t{dm4@FKZX_=H+->5eNK80OOJv_Sa$&w_RExMt03 zR%1Q08Xvw(eAVGA-4x@MB}efks`AQPq7hFe`KmQldwaz3L+9jOM>&)uaPjQRA{tgv z%a;1b$)61MhMl`D(97M!o6p9><7TePO3}SxLi!FaD7%g`#7$MX`aI{kWT@(uMEwXW zD{7gqVk%x7FODzf?gxs=$b5f(jIggS5?9h%Xn_hFY7=-&aqmN!0uTH*;wkZ?4Y~tx znVRI;7_K^3@=3|`l^310OFb-y8skXCTDY%469Uj|8cc-0PN1 zs5GYiY%fl!X(2o7i_4eSm&>5VfUpt`chZGeGtO8n)VvHNe$a5E zsoDf2b64{?q_0tLy?qv|GnkpIRyPgC9h7)Vem`Z_nw%`;)NCTHDJa(fkeODkifvpG zDc($~lGtD56k$JQy8IKXgEz-`xvV{9-_TRuDTF@Ci+EJ=s|4EqBJuF<=kNE-b7utX z!^FQAI8l>p*uPk6ANVnMMvWM#(=k8EY!t`ssRPco+3;O)HMZKp5lw`;3g^3q4s@FP z2I-@S!ry82aE&9}iG#vRk(;my($9%*sMYan% zu6`^zovs~IuNn~q=89I#-E!9gmpC^&B{I4d>Vn#(#ea-w3e--SE4jP*fZcqo9tL`V zy+MDG1rNt`du~&9To?;mXxkiFCptAmvZ#XAr5U2;sOK=yMH{YHRKKK)x9He@it7yS z;A3_b1&R5>F}|ctO3+Q;QkN&ZlLqiUl9}}YviD+B+@`j+xQfTu5HZgL> zWWwZLqDw$$1vhNtQJ1DQO~E7wg$nKjEAg{PEx0+?4>-xtGJSGGGv3{yv3fp@jVN<- z?k>!9({i{@X5<1A&De~&QBy8UPq|*EU4J%*#kjgznhdC|MLphf%bmKJ)fHVJIbc~w zUCn+e8h6DS(|m4^-<7wWH!FkT7|As})1>U7N|$*!%(g+3JvjwIWu zzyyS1lks=ae5H_mbWHulv?USu6#nHpJb4)pr;WhEsV7ZVB)PiO=9@z8o6zA!B$CCQ zg!g@BO*ZTI99zYohwOthY#fcb=LTQvqCLL_e+SLRUR4I9*PN Date: Sun, 21 Jul 2019 00:09:17 +0200 Subject: [PATCH 71/79] Add unit test for get_fileobj_raw --- test/unit/test_compression_util.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/test/unit/test_compression_util.py b/test/unit/test_compression_util.py index 582190d5def..8bd191c53db 100644 --- a/test/unit/test_compression_util.py +++ b/test/unit/test_compression_util.py @@ -2,13 +2,12 @@ import shutil import tempfile import unittest -from galaxy.util.compression_utils import CompressedFile +from galaxy.util.compression_utils import CompressedFile, get_fileobj_raw class CompressionUtilTestCase(unittest.TestCase): def test_compression_safety(self): - self.assert_safety("test-data/unsafe.tar", False) self.assert_safety("test-data/unsafe_relative_symlink.tar", False) self.assert_safety("test-data/unsafe.zip", False) self.assert_safety("test-data/4.bed.zip", True) @@ -16,6 +15,20 @@ class CompressionUtilTestCase(unittest.TestCase): self.assert_safety("test-data/safetar_with_symlink.tar", True) self.assert_safety("test-data/safe_relative_symlink.tar", True) + def test_get_fileobj_raw(self): + self.assert_format_detected("test-data/4.bed.zip", "zip") + self.assert_format_detected( + "test-data/4.bed.zip", None, ["bz2", "gzip"] + ) + self.assert_format_detected("test-data/4.bed.gz", "gzip") + self.assert_format_detected( + "test-data/4.bed.gz", None, ["bz2", "zip"] + ) + self.assert_format_detected("test-data/4.bed.bz2", "bz2") + self.assert_format_detected( + "test-data/4.bed.bz2", None, ["gzip", "zip"] + ) + def assert_safety(self, path, expected_to_be_safe): temp_dir = tempfile.mkdtemp() try: @@ -26,3 +39,13 @@ class CompressionUtilTestCase(unittest.TestCase): CompressedFile(path).extract(temp_dir) finally: shutil.rmtree(temp_dir, ignore_errors=True) + + def assert_format_detected(self, path, expected_fmt, allowed_fmts=None): + fmt1 = get_fileobj_raw(path, "r", allowed_fmts)[0] + fmt2 = get_fileobj_raw(path, "rb", allowed_fmts)[0] + fmt3 = get_fileobj_raw(path, "rt", allowed_fmts)[0] + fmt4 = get_fileobj_raw(path, "U", allowed_fmts)[0] + self.assertEqual(fmt1, fmt2) + self.assertEqual(fmt1, fmt3) + self.assertEqual(fmt1, fmt4) + self.assertEqual(fmt1, expected_fmt) From b66eeea526943ce04471e061f49f139af5d180e6 Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Sun, 21 Jul 2019 10:48:15 +0200 Subject: [PATCH 72/79] Fix and improve unit tests --- lib/galaxy/util/compression_utils.py | 3 +++ test/unit/test_compression_util.py | 25 ++++++++++++++++--------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/util/compression_utils.py b/lib/galaxy/util/compression_utils.py index 9576c9b97b9..1102b437f4a 100644 --- a/lib/galaxy/util/compression_utils.py +++ b/lib/galaxy/util/compression_utils.py @@ -48,6 +48,9 @@ def get_fileobj_raw(filename, mode="r", compressed_formats=None): elif 'zip' in compressed_formats and zipfile.is_zipfile(filename): # Return fileobj for the first file in a zip file. # 'b' is not allowed in the ZipFile mode argument + # since it always opens files in binary mode. + # For emulating text mode, we will be returning the binary fh in a + # TextIOWrapper. zf_mode = mode.replace('b', '') with zipfile.ZipFile(filename, zf_mode) as zh: fh = zh.open(zh.namelist()[0], zf_mode) diff --git a/test/unit/test_compression_util.py b/test/unit/test_compression_util.py index 8bd191c53db..f2b95a52a49 100644 --- a/test/unit/test_compression_util.py +++ b/test/unit/test_compression_util.py @@ -2,12 +2,18 @@ import shutil import tempfile import unittest -from galaxy.util.compression_utils import CompressedFile, get_fileobj_raw +import six + +from galaxy.util.compression_utils import ( + CompressedFile, + get_fileobj_raw +) class CompressionUtilTestCase(unittest.TestCase): def test_compression_safety(self): + self.assert_safety("test-data/unsafe.tar", False) self.assert_safety("test-data/unsafe_relative_symlink.tar", False) self.assert_safety("test-data/unsafe.zip", False) self.assert_safety("test-data/4.bed.zip", True) @@ -41,11 +47,12 @@ class CompressionUtilTestCase(unittest.TestCase): shutil.rmtree(temp_dir, ignore_errors=True) def assert_format_detected(self, path, expected_fmt, allowed_fmts=None): - fmt1 = get_fileobj_raw(path, "r", allowed_fmts)[0] - fmt2 = get_fileobj_raw(path, "rb", allowed_fmts)[0] - fmt3 = get_fileobj_raw(path, "rt", allowed_fmts)[0] - fmt4 = get_fileobj_raw(path, "U", allowed_fmts)[0] - self.assertEqual(fmt1, fmt2) - self.assertEqual(fmt1, fmt3) - self.assertEqual(fmt1, fmt4) - self.assertEqual(fmt1, expected_fmt) + for mode in ['r', 'rb', 'rt', 'U']: + if 'b' in mode: + expected_type = six.binary_type + else: + expected_type = six.text_type + fmt, fh = get_fileobj_raw(path, mode, allowed_fmts) + assert fmt == expected_fmt + if 'b' in mode or fmt is not None: + assert isinstance(fh.read(1), expected_type) From 652f19a422850617e3661ce34e8d990ef5a97bd0 Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Sun, 21 Jul 2019 10:59:09 +0200 Subject: [PATCH 73/79] Use more elegant way to test stream type --- test/unit/test_compression_util.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/unit/test_compression_util.py b/test/unit/test_compression_util.py index f2b95a52a49..c35c8f5ce95 100644 --- a/test/unit/test_compression_util.py +++ b/test/unit/test_compression_util.py @@ -54,5 +54,4 @@ class CompressionUtilTestCase(unittest.TestCase): expected_type = six.text_type fmt, fh = get_fileobj_raw(path, mode, allowed_fmts) assert fmt == expected_fmt - if 'b' in mode or fmt is not None: - assert isinstance(fh.read(1), expected_type) + assert isinstance(fh.read(0), expected_type) From df5252c887ff211f934916d432f8b727bf32b563 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sun, 21 Jul 2019 12:42:58 -0400 Subject: [PATCH 74/79] Update RST files to reflect code structure changes Also, clean out obsolete RST files. --- doc/source/lib/galaxy.config.rst | 20 ++ doc/source/lib/galaxy.datatypes.util.rst | 8 + doc/source/lib/galaxy.rst | 8 +- doc/source/lib/galaxy.selenium.rst | 83 +++++++ doc/source/lib/galaxy.selenium.scripts.rst | 20 ++ doc/source/lib/galaxy.tool_util.linters.rst | 92 ++++++++ doc/source/lib/galaxy.tool_util.rst | 10 + .../lib/galaxy.tool_util.verify.asserts.rst | 52 +++++ doc/source/lib/galaxy.tool_util.verify.rst | 43 ++++ doc/source/lib/galaxy.tools.linters.rst | 92 -------- doc/source/lib/galaxy.tools.rst | 18 -- .../lib/galaxy.tools.verify.asserts.rst | 52 ----- doc/source/lib/galaxy.tools.verify.rst | 43 ---- doc/source/lib/galaxy.util.rst | 1 + doc/source/lib/galaxy.util.tool_shed.rst | 36 +++ doc/source/lib/galaxy.web.base.rst | 20 -- doc/source/lib/galaxy.web.rst | 1 - doc/source/lib/galaxy.webapps.base.rst | 20 ++ doc/source/lib/galaxy.webapps.galaxy.rst | 8 - doc/source/lib/galaxy.webapps.rst | 1 + ...galaxy.webapps.tool_shed.model.migrate.rst | 7 + ...bapps.tool_shed.model.migrate.versions.rst | 220 ++++++++++++++++++ .../lib/tool_shed.galaxy_install.migrate.rst | 7 + ...l_shed.galaxy_install.migrate.versions.rst | 108 +++++++++ 24 files changed, 733 insertions(+), 237 deletions(-) create mode 100644 doc/source/lib/galaxy.config.rst create mode 100644 doc/source/lib/galaxy.selenium.rst create mode 100644 doc/source/lib/galaxy.selenium.scripts.rst create mode 100644 doc/source/lib/galaxy.tool_util.linters.rst create mode 100644 doc/source/lib/galaxy.tool_util.verify.asserts.rst create mode 100644 doc/source/lib/galaxy.tool_util.verify.rst delete mode 100644 doc/source/lib/galaxy.tools.linters.rst delete mode 100644 doc/source/lib/galaxy.tools.verify.asserts.rst delete mode 100644 doc/source/lib/galaxy.tools.verify.rst create mode 100644 doc/source/lib/galaxy.util.tool_shed.rst delete mode 100644 doc/source/lib/galaxy.web.base.rst create mode 100644 doc/source/lib/galaxy.webapps.base.rst create mode 100644 doc/source/lib/galaxy.webapps.tool_shed.model.migrate.versions.rst create mode 100644 doc/source/lib/tool_shed.galaxy_install.migrate.versions.rst diff --git a/doc/source/lib/galaxy.config.rst b/doc/source/lib/galaxy.config.rst new file mode 100644 index 00000000000..30d5b67c3ee --- /dev/null +++ b/doc/source/lib/galaxy.config.rst @@ -0,0 +1,20 @@ +galaxy.config package +===================== + +.. automodule:: galaxy.config + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +galaxy.config.script module +--------------------------- + +.. automodule:: galaxy.config.script + :members: + :undoc-members: + :show-inheritance: + + diff --git a/doc/source/lib/galaxy.datatypes.util.rst b/doc/source/lib/galaxy.datatypes.util.rst index 8ac6f2c599d..9812c396a99 100644 --- a/doc/source/lib/galaxy.datatypes.util.rst +++ b/doc/source/lib/galaxy.datatypes.util.rst @@ -25,4 +25,12 @@ galaxy.datatypes.util.gff\_util module :undoc-members: :show-inheritance: +galaxy.datatypes.util.maf\_utilities module +------------------------------------------- + +.. automodule:: galaxy.datatypes.util.maf_utilities + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/source/lib/galaxy.rst b/doc/source/lib/galaxy.rst index 321bfcfcd7c..4c2425c5326 100644 --- a/doc/source/lib/galaxy.rst +++ b/doc/source/lib/galaxy.rst @@ -14,6 +14,7 @@ Subpackages galaxy.actions galaxy.auth galaxy.authnz + galaxy.config galaxy.containers galaxy.datatypes galaxy.dependencies @@ -30,6 +31,7 @@ Subpackages galaxy.openid galaxy.quota galaxy.security + galaxy.selenium galaxy.tool_util galaxy.tools galaxy.tours @@ -53,10 +55,10 @@ galaxy.app module :undoc-members: :show-inheritance: -galaxy.config module --------------------- +galaxy.config\_watchers module +------------------------------ -.. automodule:: galaxy.config +.. automodule:: galaxy.config_watchers :members: :undoc-members: :show-inheritance: diff --git a/doc/source/lib/galaxy.selenium.rst b/doc/source/lib/galaxy.selenium.rst new file mode 100644 index 00000000000..db4f0adfa64 --- /dev/null +++ b/doc/source/lib/galaxy.selenium.rst @@ -0,0 +1,83 @@ +galaxy.selenium package +======================= + +.. automodule:: galaxy.selenium + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + + galaxy.selenium.scripts + +Submodules +---------- + +galaxy.selenium.cli module +-------------------------- + +.. automodule:: galaxy.selenium.cli + :members: + :undoc-members: + :show-inheritance: + +galaxy.selenium.components module +--------------------------------- + +.. automodule:: galaxy.selenium.components + :members: + :undoc-members: + :show-inheritance: + +galaxy.selenium.data module +--------------------------- + +.. automodule:: galaxy.selenium.data + :members: + :undoc-members: + :show-inheritance: + +galaxy.selenium.driver\_factory module +-------------------------------------- + +.. automodule:: galaxy.selenium.driver_factory + :members: + :undoc-members: + :show-inheritance: + +galaxy.selenium.has\_driver module +---------------------------------- + +.. automodule:: galaxy.selenium.has_driver + :members: + :undoc-members: + :show-inheritance: + +galaxy.selenium.navigates\_galaxy module +---------------------------------------- + +.. automodule:: galaxy.selenium.navigates_galaxy + :members: + :undoc-members: + :show-inheritance: + +galaxy.selenium.sizzle module +----------------------------- + +.. automodule:: galaxy.selenium.sizzle + :members: + :undoc-members: + :show-inheritance: + +galaxy.selenium.smart\_components module +---------------------------------------- + +.. automodule:: galaxy.selenium.smart_components + :members: + :undoc-members: + :show-inheritance: + + diff --git a/doc/source/lib/galaxy.selenium.scripts.rst b/doc/source/lib/galaxy.selenium.scripts.rst new file mode 100644 index 00000000000..9b1ca62862e --- /dev/null +++ b/doc/source/lib/galaxy.selenium.scripts.rst @@ -0,0 +1,20 @@ +galaxy.selenium.scripts package +=============================== + +.. automodule:: galaxy.selenium.scripts + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +galaxy.selenium.scripts.dump\_tour module +----------------------------------------- + +.. automodule:: galaxy.selenium.scripts.dump_tour + :members: + :undoc-members: + :show-inheritance: + + diff --git a/doc/source/lib/galaxy.tool_util.linters.rst b/doc/source/lib/galaxy.tool_util.linters.rst new file mode 100644 index 00000000000..e87307309f8 --- /dev/null +++ b/doc/source/lib/galaxy.tool_util.linters.rst @@ -0,0 +1,92 @@ +galaxy.tool\_util.linters package +================================= + +.. automodule:: galaxy.tool_util.linters + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +galaxy.tool\_util.linters.citations module +------------------------------------------ + +.. automodule:: galaxy.tool_util.linters.citations + :members: + :undoc-members: + :show-inheritance: + +galaxy.tool\_util.linters.command module +---------------------------------------- + +.. automodule:: galaxy.tool_util.linters.command + :members: + :undoc-members: + :show-inheritance: + +galaxy.tool\_util.linters.cwl module +------------------------------------ + +.. automodule:: galaxy.tool_util.linters.cwl + :members: + :undoc-members: + :show-inheritance: + +galaxy.tool\_util.linters.general module +---------------------------------------- + +.. automodule:: galaxy.tool_util.linters.general + :members: + :undoc-members: + :show-inheritance: + +galaxy.tool\_util.linters.help module +------------------------------------- + +.. automodule:: galaxy.tool_util.linters.help + :members: + :undoc-members: + :show-inheritance: + +galaxy.tool\_util.linters.inputs module +--------------------------------------- + +.. automodule:: galaxy.tool_util.linters.inputs + :members: + :undoc-members: + :show-inheritance: + +galaxy.tool\_util.linters.outputs module +---------------------------------------- + +.. automodule:: galaxy.tool_util.linters.outputs + :members: + :undoc-members: + :show-inheritance: + +galaxy.tool\_util.linters.stdio module +-------------------------------------- + +.. automodule:: galaxy.tool_util.linters.stdio + :members: + :undoc-members: + :show-inheritance: + +galaxy.tool\_util.linters.tests module +-------------------------------------- + +.. automodule:: galaxy.tool_util.linters.tests + :members: + :undoc-members: + :show-inheritance: + +galaxy.tool\_util.linters.xml\_order module +------------------------------------------- + +.. automodule:: galaxy.tool_util.linters.xml_order + :members: + :undoc-members: + :show-inheritance: + + diff --git a/doc/source/lib/galaxy.tool_util.rst b/doc/source/lib/galaxy.tool_util.rst index 02818574534..f09d5aa37f5 100644 --- a/doc/source/lib/galaxy.tool_util.rst +++ b/doc/source/lib/galaxy.tool_util.rst @@ -13,8 +13,10 @@ Subpackages galaxy.tool_util.cwl galaxy.tool_util.deps + galaxy.tool_util.linters galaxy.tool_util.locations galaxy.tool_util.parser + galaxy.tool_util.verify Submodules ---------- @@ -27,6 +29,14 @@ galaxy.tool\_util.fetcher module :undoc-members: :show-inheritance: +galaxy.tool\_util.lint module +----------------------------- + +.. automodule:: galaxy.tool_util.lint + :members: + :undoc-members: + :show-inheritance: + galaxy.tool\_util.loader module ------------------------------- diff --git a/doc/source/lib/galaxy.tool_util.verify.asserts.rst b/doc/source/lib/galaxy.tool_util.verify.asserts.rst new file mode 100644 index 00000000000..b252cd0806c --- /dev/null +++ b/doc/source/lib/galaxy.tool_util.verify.asserts.rst @@ -0,0 +1,52 @@ +galaxy.tool\_util.verify.asserts package +======================================== + +.. automodule:: galaxy.tool_util.verify.asserts + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +galaxy.tool\_util.verify.asserts.archive module +----------------------------------------------- + +.. automodule:: galaxy.tool_util.verify.asserts.archive + :members: + :undoc-members: + :show-inheritance: + +galaxy.tool\_util.verify.asserts.hdf5 module +-------------------------------------------- + +.. automodule:: galaxy.tool_util.verify.asserts.hdf5 + :members: + :undoc-members: + :show-inheritance: + +galaxy.tool\_util.verify.asserts.tabular module +----------------------------------------------- + +.. automodule:: galaxy.tool_util.verify.asserts.tabular + :members: + :undoc-members: + :show-inheritance: + +galaxy.tool\_util.verify.asserts.text module +-------------------------------------------- + +.. automodule:: galaxy.tool_util.verify.asserts.text + :members: + :undoc-members: + :show-inheritance: + +galaxy.tool\_util.verify.asserts.xml module +------------------------------------------- + +.. automodule:: galaxy.tool_util.verify.asserts.xml + :members: + :undoc-members: + :show-inheritance: + + diff --git a/doc/source/lib/galaxy.tool_util.verify.rst b/doc/source/lib/galaxy.tool_util.verify.rst new file mode 100644 index 00000000000..99469adf02d --- /dev/null +++ b/doc/source/lib/galaxy.tool_util.verify.rst @@ -0,0 +1,43 @@ +galaxy.tool\_util.verify package +================================ + +.. automodule:: galaxy.tool_util.verify + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + + galaxy.tool_util.verify.asserts + +Submodules +---------- + +galaxy.tool\_util.verify.interactor module +------------------------------------------ + +.. automodule:: galaxy.tool_util.verify.interactor + :members: + :undoc-members: + :show-inheritance: + +galaxy.tool\_util.verify.script module +-------------------------------------- + +.. automodule:: galaxy.tool_util.verify.script + :members: + :undoc-members: + :show-inheritance: + +galaxy.tool\_util.verify.test\_data module +------------------------------------------ + +.. automodule:: galaxy.tool_util.verify.test_data + :members: + :undoc-members: + :show-inheritance: + + diff --git a/doc/source/lib/galaxy.tools.linters.rst b/doc/source/lib/galaxy.tools.linters.rst deleted file mode 100644 index f79ce27f18b..00000000000 --- a/doc/source/lib/galaxy.tools.linters.rst +++ /dev/null @@ -1,92 +0,0 @@ -galaxy.tools.linters package -============================ - -.. automodule:: galaxy.tools.linters - :members: - :undoc-members: - :show-inheritance: - -Submodules ----------- - -galaxy.tools.linters.citations module -------------------------------------- - -.. automodule:: galaxy.tools.linters.citations - :members: - :undoc-members: - :show-inheritance: - -galaxy.tools.linters.command module ------------------------------------ - -.. automodule:: galaxy.tools.linters.command - :members: - :undoc-members: - :show-inheritance: - -galaxy.tools.linters.cwl module -------------------------------- - -.. automodule:: galaxy.tools.linters.cwl - :members: - :undoc-members: - :show-inheritance: - -galaxy.tools.linters.general module ------------------------------------ - -.. automodule:: galaxy.tools.linters.general - :members: - :undoc-members: - :show-inheritance: - -galaxy.tools.linters.help module --------------------------------- - -.. automodule:: galaxy.tools.linters.help - :members: - :undoc-members: - :show-inheritance: - -galaxy.tools.linters.inputs module ----------------------------------- - -.. automodule:: galaxy.tools.linters.inputs - :members: - :undoc-members: - :show-inheritance: - -galaxy.tools.linters.outputs module ------------------------------------ - -.. automodule:: galaxy.tools.linters.outputs - :members: - :undoc-members: - :show-inheritance: - -galaxy.tools.linters.stdio module ---------------------------------- - -.. automodule:: galaxy.tools.linters.stdio - :members: - :undoc-members: - :show-inheritance: - -galaxy.tools.linters.tests module ---------------------------------- - -.. automodule:: galaxy.tools.linters.tests - :members: - :undoc-members: - :show-inheritance: - -galaxy.tools.linters.xml\_order module --------------------------------------- - -.. automodule:: galaxy.tools.linters.xml_order - :members: - :undoc-members: - :show-inheritance: - - diff --git a/doc/source/lib/galaxy.tools.rst b/doc/source/lib/galaxy.tools.rst index e544594648a..ad6b25686dc 100644 --- a/doc/source/lib/galaxy.tools.rst +++ b/doc/source/lib/galaxy.tools.rst @@ -18,12 +18,10 @@ Subpackages galaxy.tools.expressions galaxy.tools.filters galaxy.tools.imp_exp - galaxy.tools.linters galaxy.tools.parameters galaxy.tools.search galaxy.tools.toolbox galaxy.tools.util - galaxy.tools.verify Submodules ---------- @@ -76,22 +74,6 @@ galaxy.tools.execute module :undoc-members: :show-inheritance: -galaxy.tools.lint module ------------------------- - -.. automodule:: galaxy.tools.lint - :members: - :undoc-members: - :show-inheritance: - -galaxy.tools.lint\_util module ------------------------------- - -.. automodule:: galaxy.tools.lint_util - :members: - :undoc-members: - :show-inheritance: - galaxy.tools.repositories module -------------------------------- diff --git a/doc/source/lib/galaxy.tools.verify.asserts.rst b/doc/source/lib/galaxy.tools.verify.asserts.rst deleted file mode 100644 index 66b69426efb..00000000000 --- a/doc/source/lib/galaxy.tools.verify.asserts.rst +++ /dev/null @@ -1,52 +0,0 @@ -galaxy.tools.verify.asserts package -=================================== - -.. automodule:: galaxy.tools.verify.asserts - :members: - :undoc-members: - :show-inheritance: - -Submodules ----------- - -galaxy.tools.verify.asserts.archive module ------------------------------------------- - -.. automodule:: galaxy.tools.verify.asserts.archive - :members: - :undoc-members: - :show-inheritance: - -galaxy.tools.verify.asserts.hdf5 module ---------------------------------------- - -.. automodule:: galaxy.tools.verify.asserts.hdf5 - :members: - :undoc-members: - :show-inheritance: - -galaxy.tools.verify.asserts.tabular module ------------------------------------------- - -.. automodule:: galaxy.tools.verify.asserts.tabular - :members: - :undoc-members: - :show-inheritance: - -galaxy.tools.verify.asserts.text module ---------------------------------------- - -.. automodule:: galaxy.tools.verify.asserts.text - :members: - :undoc-members: - :show-inheritance: - -galaxy.tools.verify.asserts.xml module --------------------------------------- - -.. automodule:: galaxy.tools.verify.asserts.xml - :members: - :undoc-members: - :show-inheritance: - - diff --git a/doc/source/lib/galaxy.tools.verify.rst b/doc/source/lib/galaxy.tools.verify.rst deleted file mode 100644 index 4daf18a03de..00000000000 --- a/doc/source/lib/galaxy.tools.verify.rst +++ /dev/null @@ -1,43 +0,0 @@ -galaxy.tools.verify package -=========================== - -.. automodule:: galaxy.tools.verify - :members: - :undoc-members: - :show-inheritance: - -Subpackages ------------ - -.. toctree:: - - galaxy.tools.verify.asserts - -Submodules ----------- - -galaxy.tools.verify.interactor module -------------------------------------- - -.. automodule:: galaxy.tools.verify.interactor - :members: - :undoc-members: - :show-inheritance: - -galaxy.tools.verify.script module ---------------------------------- - -.. automodule:: galaxy.tools.verify.script - :members: - :undoc-members: - :show-inheritance: - -galaxy.tools.verify.test\_data module -------------------------------------- - -.. automodule:: galaxy.tools.verify.test_data - :members: - :undoc-members: - :show-inheritance: - - diff --git a/doc/source/lib/galaxy.util.rst b/doc/source/lib/galaxy.util.rst index 792c0dfabb9..06cfeee6814 100644 --- a/doc/source/lib/galaxy.util.rst +++ b/doc/source/lib/galaxy.util.rst @@ -14,6 +14,7 @@ Subpackages galaxy.util.logging galaxy.util.pastescript galaxy.util.path + galaxy.util.tool_shed Submodules ---------- diff --git a/doc/source/lib/galaxy.util.tool_shed.rst b/doc/source/lib/galaxy.util.tool_shed.rst new file mode 100644 index 00000000000..42b92b9bc7e --- /dev/null +++ b/doc/source/lib/galaxy.util.tool_shed.rst @@ -0,0 +1,36 @@ +galaxy.util.tool\_shed package +============================== + +.. automodule:: galaxy.util.tool_shed + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +galaxy.util.tool\_shed.common\_util module +------------------------------------------ + +.. automodule:: galaxy.util.tool_shed.common_util + :members: + :undoc-members: + :show-inheritance: + +galaxy.util.tool\_shed.encoding\_util module +-------------------------------------------- + +.. automodule:: galaxy.util.tool_shed.encoding_util + :members: + :undoc-members: + :show-inheritance: + +galaxy.util.tool\_shed.xml\_util module +--------------------------------------- + +.. automodule:: galaxy.util.tool_shed.xml_util + :members: + :undoc-members: + :show-inheritance: + + diff --git a/doc/source/lib/galaxy.web.base.rst b/doc/source/lib/galaxy.web.base.rst deleted file mode 100644 index f81bd29947a..00000000000 --- a/doc/source/lib/galaxy.web.base.rst +++ /dev/null @@ -1,20 +0,0 @@ -galaxy.web.base package -======================= - -.. automodule:: galaxy.web.base - :members: - :undoc-members: - :show-inheritance: - -Submodules ----------- - -galaxy.web.base.controller module ---------------------------------- - -.. automodule:: galaxy.web.base.controller - :members: - :undoc-members: - :show-inheritance: - - diff --git a/doc/source/lib/galaxy.web.rst b/doc/source/lib/galaxy.web.rst index dc3fd75cfd6..8bf2e29cc68 100644 --- a/doc/source/lib/galaxy.web.rst +++ b/doc/source/lib/galaxy.web.rst @@ -11,7 +11,6 @@ Subpackages .. toctree:: - galaxy.web.base galaxy.web.framework galaxy.web.proxy diff --git a/doc/source/lib/galaxy.webapps.base.rst b/doc/source/lib/galaxy.webapps.base.rst new file mode 100644 index 00000000000..57a5db43038 --- /dev/null +++ b/doc/source/lib/galaxy.webapps.base.rst @@ -0,0 +1,20 @@ +galaxy.webapps.base package +=========================== + +.. automodule:: galaxy.webapps.base + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +galaxy.webapps.base.controller module +------------------------------------- + +.. automodule:: galaxy.webapps.base.controller + :members: + :undoc-members: + :show-inheritance: + + diff --git a/doc/source/lib/galaxy.webapps.galaxy.rst b/doc/source/lib/galaxy.webapps.galaxy.rst index f5e0f2027ef..5256155b1ed 100644 --- a/doc/source/lib/galaxy.webapps.galaxy.rst +++ b/doc/source/lib/galaxy.webapps.galaxy.rst @@ -25,12 +25,4 @@ galaxy.webapps.galaxy.buildapp module :undoc-members: :show-inheritance: -galaxy.webapps.galaxy.config\_watchers module ---------------------------------------------- - -.. automodule:: galaxy.webapps.galaxy.config_watchers - :members: - :undoc-members: - :show-inheritance: - diff --git a/doc/source/lib/galaxy.webapps.rst b/doc/source/lib/galaxy.webapps.rst index 0ca4f9f0d3a..bcf4c1e750f 100644 --- a/doc/source/lib/galaxy.webapps.rst +++ b/doc/source/lib/galaxy.webapps.rst @@ -11,6 +11,7 @@ Subpackages .. toctree:: + galaxy.webapps.base galaxy.webapps.galaxy galaxy.webapps.reports galaxy.webapps.tool_shed diff --git a/doc/source/lib/galaxy.webapps.tool_shed.model.migrate.rst b/doc/source/lib/galaxy.webapps.tool_shed.model.migrate.rst index 3faeb2ab908..90e8d042ed1 100644 --- a/doc/source/lib/galaxy.webapps.tool_shed.model.migrate.rst +++ b/doc/source/lib/galaxy.webapps.tool_shed.model.migrate.rst @@ -6,6 +6,13 @@ galaxy.webapps.tool\_shed.model.migrate package :undoc-members: :show-inheritance: +Subpackages +----------- + +.. toctree:: + + galaxy.webapps.tool_shed.model.migrate.versions + Submodules ---------- diff --git a/doc/source/lib/galaxy.webapps.tool_shed.model.migrate.versions.rst b/doc/source/lib/galaxy.webapps.tool_shed.model.migrate.versions.rst new file mode 100644 index 00000000000..dd416885749 --- /dev/null +++ b/doc/source/lib/galaxy.webapps.tool_shed.model.migrate.versions.rst @@ -0,0 +1,220 @@ +galaxy.webapps.tool\_shed.model.migrate.versions package +======================================================== + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +galaxy.webapps.tool\_shed.model.migrate.versions.0001\_initial\_tables module +----------------------------------------------------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0001_initial_tables + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0002\_add\_tool\_suite\_column module +-------------------------------------------------------------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0002_add_tool_suite_column + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0003\_review\_and\_review\_association\_tables module +------------------------------------------------------------------------------------------------------ + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0003_review_and_review_association_tables + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0004\_repository\_tables module +-------------------------------------------------------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0004_repository_tables + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0005\_drop\_tool\_related\_tables module +----------------------------------------------------------------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0005_drop_tool_related_tables + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0006\_add\_email\_alerts\_column module +---------------------------------------------------------------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0006_add_email_alerts_column + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0007\_add\_long\_description\_times\_downloaded\_columns module +---------------------------------------------------------------------------------------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0007_add_long_description_times_downloaded_columns + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0008\_add\_repository\_metadata\_table module +---------------------------------------------------------------------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0008_add_repository_metadata_table + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0009\_add\_malicious\_column module +------------------------------------------------------------------------------------ + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0009_add_malicious_column + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0010\_add\_new\_repo\_alert\_column module +------------------------------------------------------------------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0010_add_new_repo_alert_column + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0011\_add\_tool\_versions\_column module +----------------------------------------------------------------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0011_add_tool_versions_column + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0012\_add\_downloadable\_column module +--------------------------------------------------------------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0012_add_downloadable_column + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0013\_add\_review\_tables module +--------------------------------------------------------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0013_add_review_tables + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0014\_add\_deprecated\_column module +------------------------------------------------------------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0014_add_deprecated_column + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0015\_add\_api\_keys\_table module +----------------------------------------------------------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0015_add_api_keys_table + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0016\_add\_do\_not\_test\_tools\_functionally\_correct\_errors\_columns module +------------------------------------------------------------------------------------------------------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0016_add_do_not_test_tools_functionally_correct_errors_columns + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0017\_add\_galaxy\_utility\_columns\_to\_repository\_metadata\_table module +---------------------------------------------------------------------------------------------------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0017_add_galaxy_utility_columns_to_repository_metadata_table + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0018\_add\_repository\_metadata\_flag\_columns module +------------------------------------------------------------------------------------------------------ + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0018_add_repository_metadata_flag_columns + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0019\_add\_skip\_tool\_test\_table\_and\_test\_install\_error\_column module +----------------------------------------------------------------------------------------------------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0019_add_skip_tool_test_table_and_test_install_error_column + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0020\_add\_repository\_type\_column module +------------------------------------------------------------------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0020_add_repository_type_column + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0021\_change\_repository\_type\_value module +--------------------------------------------------------------------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0021_change_repository_type_value + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0022\_add\_repository\_admin\_roles module +------------------------------------------------------------------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0022_add_repository_admin_roles + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0023\_add\_repository\_url\_and\_hompeage\_url module +------------------------------------------------------------------------------------------------------ + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0023_add_repository_url_and_hompeage_url + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0024\_password\_reset module +----------------------------------------------------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0024_password_reset + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0025\_session\_timeout module +------------------------------------------------------------------------------ + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0025_session_timeout + :members: + :undoc-members: + :show-inheritance: + +galaxy.webapps.tool\_shed.model.migrate.versions.0026\_add\_numeric\_revision\_column module +-------------------------------------------------------------------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.versions.0026_add_numeric_revision_column + :members: + :undoc-members: + :show-inheritance: + + diff --git a/doc/source/lib/tool_shed.galaxy_install.migrate.rst b/doc/source/lib/tool_shed.galaxy_install.migrate.rst index 12d29998abc..578387dfd5f 100644 --- a/doc/source/lib/tool_shed.galaxy_install.migrate.rst +++ b/doc/source/lib/tool_shed.galaxy_install.migrate.rst @@ -6,6 +6,13 @@ tool\_shed.galaxy\_install.migrate package :undoc-members: :show-inheritance: +Subpackages +----------- + +.. toctree:: + + tool_shed.galaxy_install.migrate.versions + Submodules ---------- diff --git a/doc/source/lib/tool_shed.galaxy_install.migrate.versions.rst b/doc/source/lib/tool_shed.galaxy_install.migrate.versions.rst new file mode 100644 index 00000000000..f1db6f74f58 --- /dev/null +++ b/doc/source/lib/tool_shed.galaxy_install.migrate.versions.rst @@ -0,0 +1,108 @@ +tool\_shed.galaxy\_install.migrate.versions package +=================================================== + +.. automodule:: tool_shed.galaxy_install.migrate.versions + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +tool\_shed.galaxy\_install.migrate.versions.0001\_tools module +-------------------------------------------------------------- + +.. automodule:: tool_shed.galaxy_install.migrate.versions.0001_tools + :members: + :undoc-members: + :show-inheritance: + +tool\_shed.galaxy\_install.migrate.versions.0002\_tools module +-------------------------------------------------------------- + +.. automodule:: tool_shed.galaxy_install.migrate.versions.0002_tools + :members: + :undoc-members: + :show-inheritance: + +tool\_shed.galaxy\_install.migrate.versions.0003\_tools module +-------------------------------------------------------------- + +.. automodule:: tool_shed.galaxy_install.migrate.versions.0003_tools + :members: + :undoc-members: + :show-inheritance: + +tool\_shed.galaxy\_install.migrate.versions.0004\_tools module +-------------------------------------------------------------- + +.. automodule:: tool_shed.galaxy_install.migrate.versions.0004_tools + :members: + :undoc-members: + :show-inheritance: + +tool\_shed.galaxy\_install.migrate.versions.0005\_tools module +-------------------------------------------------------------- + +.. automodule:: tool_shed.galaxy_install.migrate.versions.0005_tools + :members: + :undoc-members: + :show-inheritance: + +tool\_shed.galaxy\_install.migrate.versions.0006\_tools module +-------------------------------------------------------------- + +.. automodule:: tool_shed.galaxy_install.migrate.versions.0006_tools + :members: + :undoc-members: + :show-inheritance: + +tool\_shed.galaxy\_install.migrate.versions.0007\_tools module +-------------------------------------------------------------- + +.. automodule:: tool_shed.galaxy_install.migrate.versions.0007_tools + :members: + :undoc-members: + :show-inheritance: + +tool\_shed.galaxy\_install.migrate.versions.0008\_tools module +-------------------------------------------------------------- + +.. automodule:: tool_shed.galaxy_install.migrate.versions.0008_tools + :members: + :undoc-members: + :show-inheritance: + +tool\_shed.galaxy\_install.migrate.versions.0009\_tools module +-------------------------------------------------------------- + +.. automodule:: tool_shed.galaxy_install.migrate.versions.0009_tools + :members: + :undoc-members: + :show-inheritance: + +tool\_shed.galaxy\_install.migrate.versions.0010\_tools module +-------------------------------------------------------------- + +.. automodule:: tool_shed.galaxy_install.migrate.versions.0010_tools + :members: + :undoc-members: + :show-inheritance: + +tool\_shed.galaxy\_install.migrate.versions.0011\_tools module +-------------------------------------------------------------- + +.. automodule:: tool_shed.galaxy_install.migrate.versions.0011_tools + :members: + :undoc-members: + :show-inheritance: + +tool\_shed.galaxy\_install.migrate.versions.0012\_tools module +-------------------------------------------------------------- + +.. automodule:: tool_shed.galaxy_install.migrate.versions.0012_tools + :members: + :undoc-members: + :show-inheritance: + + From e345bcab98c9f5b4a74ca4654e3754eba38166e3 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sun, 21 Jul 2019 19:41:10 -0400 Subject: [PATCH 75/79] Remove some test interdependencies in unit/tools Factor out BaseParameterTestCase into a helper module. --- test/unit/tools/test_column_parameters.py | 2 +- test/unit/tools/test_data_parameters.py | 2 +- test/unit/tools/test_parameter_parsing.py | 22 +--------------------- test/unit/tools/test_select_parameters.py | 2 +- test/unit/tools/util.py | 23 +++++++++++++++++++++++ 5 files changed, 27 insertions(+), 24 deletions(-) create mode 100644 test/unit/tools/util.py diff --git a/test/unit/tools/test_column_parameters.py b/test/unit/tools/test_column_parameters.py index 19fc9ea4029..4cbb39d131f 100644 --- a/test/unit/tools/test_column_parameters.py +++ b/test/unit/tools/test_column_parameters.py @@ -3,7 +3,7 @@ test_select_parameters.py. """ from galaxy import model from galaxy.util import bunch -from .test_parameter_parsing import BaseParameterTestCase +from .util import BaseParameterTestCase from ..tools_support import datatypes_registry diff --git a/test/unit/tools/test_data_parameters.py b/test/unit/tools/test_data_parameters.py index 6d18fe3458a..a89425c03b6 100644 --- a/test/unit/tools/test_data_parameters.py +++ b/test/unit/tools/test_data_parameters.py @@ -1,5 +1,5 @@ from galaxy import model -from .test_parameter_parsing import BaseParameterTestCase +from .util import BaseParameterTestCase from ..unittest_utils import galaxy_mock diff --git a/test/unit/tools/test_parameter_parsing.py b/test/unit/tools/test_parameter_parsing.py index f858ca00e40..bdf9c9de65d 100644 --- a/test/unit/tools/test_parameter_parsing.py +++ b/test/unit/tools/test_parameter_parsing.py @@ -1,11 +1,7 @@ from unittest import TestCase -from xml.etree.ElementTree import XML -from galaxy import model -from galaxy.tools.parameters import basic from galaxy.tools.parameters.meta import process_key -from galaxy.util import bunch -from ..tools_support import UsesApp +from .util import BaseParameterTestCase class ProcessKeyTestCase(TestCase): @@ -46,22 +42,6 @@ class ProcessKeyTestCase(TestCase): self.assertEqual(nested_dict, expected_dict) -class BaseParameterTestCase(TestCase, UsesApp): - - def setUp(self): - self.setup_app() - self.mock_tool = bunch.Bunch( - app=self.app, - tool_type="default", - valid_input_states=model.Dataset.valid_input_states, - ) - - def _parameter_for(self, **kwds): - content = kwds["xml"] - param_xml = XML(content) - return basic.ToolParameter.build(self.mock_tool, param_xml) - - class ParameterParsingTestCase(BaseParameterTestCase): """ Test the parsing of XML for most parameter types - in many ways these are not very good tests since they break the abstraction diff --git a/test/unit/tools/test_select_parameters.py b/test/unit/tools/test_select_parameters.py index 28cfe50e33d..f00e64d6f40 100644 --- a/test/unit/tools/test_select_parameters.py +++ b/test/unit/tools/test_select_parameters.py @@ -1,7 +1,7 @@ from galaxy import model from galaxy.tools.parameters import basic from galaxy.util import bunch -from .test_parameter_parsing import BaseParameterTestCase +from .util import BaseParameterTestCase class SelectToolParameterTestCase(BaseParameterTestCase): diff --git a/test/unit/tools/util.py b/test/unit/tools/util.py new file mode 100644 index 00000000000..384de1052de --- /dev/null +++ b/test/unit/tools/util.py @@ -0,0 +1,23 @@ +from unittest import TestCase +from xml.etree.ElementTree import XML + +from galaxy import model +from galaxy.tools.parameters import basic +from galaxy.util import bunch +from ..tools_support import UsesApp + + +class BaseParameterTestCase(TestCase, UsesApp): + + def setUp(self): + self.setup_app() + self.mock_tool = bunch.Bunch( + app=self.app, + tool_type="default", + valid_input_states=model.Dataset.valid_input_states, + ) + + def _parameter_for(self, **kwds): + content = kwds["xml"] + param_xml = XML(content) + return basic.ToolParameter.build(self.mock_tool, param_xml) From f060d3c1f2aef8f182d7eeb1b1357c2e5ebdef61 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sun, 21 Jul 2019 21:33:27 -0400 Subject: [PATCH 76/79] [Commit to squash] (restart tests) --- test/unit/tools/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/tools/util.py b/test/unit/tools/util.py index 384de1052de..6535683124e 100644 --- a/test/unit/tools/util.py +++ b/test/unit/tools/util.py @@ -7,6 +7,7 @@ from galaxy.util import bunch from ..tools_support import UsesApp + class BaseParameterTestCase(TestCase, UsesApp): def setUp(self): From a3c4d753b902f8a0eb96d626f46903c50e6150b7 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sun, 21 Jul 2019 21:35:31 -0400 Subject: [PATCH 77/79] [Commit to squash] (restart test/undo prev edit) --- test/unit/tools/util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/tools/util.py b/test/unit/tools/util.py index 6535683124e..384de1052de 100644 --- a/test/unit/tools/util.py +++ b/test/unit/tools/util.py @@ -7,7 +7,6 @@ from galaxy.util import bunch from ..tools_support import UsesApp - class BaseParameterTestCase(TestCase, UsesApp): def setUp(self): From a2e2139d5d2541d5da2ac5fc7c5d3b19add1a211 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 22 Jul 2019 10:23:33 +0200 Subject: [PATCH 78/79] Sort Collection: add missing when tags --- lib/galaxy/tools/sort_collection_list.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/tools/sort_collection_list.xml b/lib/galaxy/tools/sort_collection_list.xml index 1d7207e22dd..7c8d3467a52 100644 --- a/lib/galaxy/tools/sort_collection_list.xml +++ b/lib/galaxy/tools/sort_collection_list.xml @@ -14,6 +14,8 @@ + + From cef7eeb7beac4e7dd2682f98e3f0552b1c2cbee6 Mon Sep 17 00:00:00 2001 From: guerler Date: Tue, 23 Jul 2019 00:57:28 -0400 Subject: [PATCH 79/79] Hide references card if array is empty --- client/galaxy/scripts/mvc/tool/tool-form-base.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/galaxy/scripts/mvc/tool/tool-form-base.js b/client/galaxy/scripts/mvc/tool/tool-form-base.js index b98d5f1929b..ba499e4d9b9 100644 --- a/client/galaxy/scripts/mvc/tool/tool-form-base.js +++ b/client/galaxy/scripts/mvc/tool/tool-form-base.js @@ -280,7 +280,7 @@ export default FormBase.extend({ } }).$mount(vm); } - if (options.xrefs) { + if (options.xrefs && options.xrefs.length) { var xrefInstance = Vue.extend(xrefs); vm = document.createElement("div"); $el.append(vm);