Merge pull request #13251 from mvdbeek/openpbs_support

Add openPBS cli support, rebuild CLI test images
2026-09-24 16:30:27 +08:00 · 2022-01-29 15:59:05 +00:00
parent ca35dcadb7 4b46c898a7
commit 4ffc7e8194
6 changed files with 90 additions and 50 deletions
@@ -4,7 +4,7 @@ adal==1.2.7
 aiofiles==0.8.0; python_version >= "3.6" and python_version < "4.0"
 alabaster==0.7.12; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 amqp==5.0.9; python_version >= "3.7"
-anyio==3.4.0; python_version >= "3.7" and python_full_version >= "3.6.2"
+anyio==3.5.0; python_version >= "3.7" and python_full_version >= "3.6.2"
 appdirs==1.4.4
 argcomplete==2.0.0; python_version >= "3.6" and python_version < "4"
 asgiref==3.4.1; python_version >= "3.6"
@@ -58,7 +58,7 @@ ecdsa==0.17.0; python_version >= "2.6" and python_full_version < "3.0.0" or pyth
 edam-ontology==1.25.2
 fabric3==1.14.post1
 fastapi-utils==0.2.1; python_version >= "3.6" and python_version < "4.0"
-fastapi==0.71.0; python_full_version >= "3.6.1"
+fastapi==0.72.0; python_full_version >= "3.6.1"
 fluent-logger==0.10.0; python_version >= "3.5"
 fs==2.4.14
 funcsigs==1.0.2
@@ -70,18 +70,18 @@ gunicorn==20.1.0; python_version >= "3.5"
 gxformat2==0.15.0
 h11==0.12.0; python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.6.1"
 h5py==3.6.0; python_version >= "3.7"
-httpcore==0.14.4; python_version >= "3.6"
+httpcore==0.14.5; python_version >= "3.6"
 httpx==0.21.3; python_version >= "3.6"
 humanfriendly==10.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0"
 idna==3.3
 imagesize==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-importlib-metadata==4.10.0; python_version == "3.7" and (python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.6.0" and python_version < "3.8" and python_version >= "3.7")
-importlib-resources==5.4.0; python_version < "3.9" and python_version >= "3.7"
+importlib-metadata==4.10.1; python_version == "3.7" and (python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.6.0" and python_version < "3.8" and python_version >= "3.7") and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6")
+importlib-resources==5.4.0; python_version >= "3.6"
 iniconfig==1.1.1; python_version >= "3.7"
 isa-rwval==0.10.10
 isodate==0.6.1; python_version >= "3.7" and python_version < "4"
 jinja2==3.0.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-jsonschema==4.3.3; python_version >= "3.7"
+jsonschema==4.4.0; python_version >= "3.7"
 junit-xml==1.9; python_version >= "3.6" and python_version < "4"
 kombu==5.2.3; python_version >= "3.7"
 lagom==1.7.0; python_version >= "3.6" and python_version < "4.0"
@@ -106,7 +106,7 @@ oauthlib==3.1.1; python_version >= "3.6" and python_full_version < "3.0.0" or py
 outcome==1.1.0; python_version >= "3.7" and python_version < "4.0"
 oyaml==1.0
 packaging==21.3; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.7"
-paramiko==2.8.1
+paramiko==2.9.2
 parsley==1.3
 paste==3.5.0
 pastedeploy==2.1.1
@@ -134,12 +134,12 @@ pyjwt==2.3.0; python_version >= "3.6"
 pykwalify==1.8.0
 pynacl==1.5.0; python_version >= "3.6"
 pyopenssl==21.0.0; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.7"
-pyparsing==3.0.6; python_version >= "3.6"
+pyparsing==3.0.7; python_version >= "3.6"
 pyreadline3==3.3; sys_platform == "win32" and python_version >= "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0")
 pyreadline==2.1; sys_platform == "win32" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0")
-pyrsistent==0.18.0; python_version >= "3.7"
+pyrsistent==0.18.1; python_version >= "3.7"
 pysam==0.18.0
-pytest-asyncio==0.16.0; python_version >= "3.6"
+pytest-asyncio==0.17.2; python_version >= "3.7"
 pytest-celery==0.0.0
 pytest-cov==3.0.0; python_version >= "3.6"
 pytest-html==3.1.1; python_version >= "3.6"
@@ -151,7 +151,7 @@ pytest-pythonpath==0.7.3
 pytest-shard==0.1.2; python_version >= "3.6"
 pytest==6.2.5; python_version >= "3.6"
 python-dateutil==2.8.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.3.0"
-python-irodsclient==1.0.0
+python-irodsclient==1.1.0
 python-jose==3.3.0
 python-multipart==0.0.5
 python3-openid==3.2.0; python_version >= "3.0"
@@ -166,9 +166,9 @@ repoze.lru==0.7
 requests-oauthlib==1.3.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
 requests-toolbelt==0.9.1; python_version >= "3.6"
 requests==2.27.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
-responses==0.16.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
+responses==0.17.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
 rfc3986==1.5.0; python_version >= "3.6"
-rich==10.16.2; python_full_version >= "3.6.2" and python_full_version < "4.0.0"
+rich==11.0.0; python_full_version >= "3.6.2" and python_full_version < "4.0.0"
 routes==2.5.1
 rsa==4.8; python_version >= "3.6" and python_version < "4"
 ruamel.yaml.clib==0.2.6; platform_python_implementation == "CPython" and python_version < "3.10" and python_version >= "3.6"
@@ -184,7 +184,7 @@ social-auth-core==4.0.3
 sortedcontainers==2.4.0
 sphinx-markdown-tables==0.0.15
 sphinx-rtd-theme==1.0.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
-sphinx==4.3.2; python_version >= "3.6"
+sphinx==4.4.0; python_version >= "3.6"
 sphinxcontrib-applehelp==1.0.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 sphinxcontrib-devhelp==1.0.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 sphinxcontrib-htmlhelp==2.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
@@ -192,7 +192,7 @@ sphinxcontrib-jsmath==1.0.1; python_version >= "3.6" and python_full_version < "
 sphinxcontrib-qthelp==1.0.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 sphinxcontrib-serializinghtml==1.1.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 sqlalchemy-migrate==0.13.0
-sqlalchemy==1.4.29; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
+sqlalchemy==1.4.31; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
 sqlitedict==1.7.0
 sqlparse==0.4.2; python_version >= "3.5"
 starlette-context==0.3.3; python_version >= "3.7"
@@ -203,7 +203,7 @@ tempita==0.5.2
 tenacity==8.0.1; python_version >= "3.6"
 testfixtures==6.18.3
 tifffile==2021.11.2; python_version >= "3.7"
-tinydb==4.5.0; python_version >= "3.5" and python_version < "4.0"
+tinydb==4.6.1; python_version >= "3.6" and python_version < "4.0"
 toml==0.10.2; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.7"
 tomli==1.2.3; python_full_version >= "3.6.2" and python_version >= "3.6"
 tornado==6.1; python_version >= "3.5"
@@ -218,7 +218,7 @@ typing-extensions==4.0.1; python_version >= "3.6"
 tzlocal==2.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4"
 ubiquerg==0.6.2
 urllib3==1.26.8; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.7"
-uvicorn==0.16.0
+uvicorn==0.17.0
 uvloop==0.16.0
 vine==5.0.0; python_version >= "3.7"
 watchdog==2.1.6; python_version >= "3.6"
@@ -228,7 +228,6 @@ webob==1.8.7; (python_version >= "2.7" and python_full_version < "3.0.0") or (py
 whoosh==2.7.4
 wrapt==1.13.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 wsproto==1.0.0; python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.6.1"
-xmlrunner==1.7.7
 yacman==0.8.4
 zipp==3.7.0; python_version == "3.7" and (python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.6.0" and python_version < "3.8" and python_version >= "3.7")
 zipstream-new==1.1.8
@@ -1,6 +1,6 @@
 attrs==21.4.0
 flake8==4.0.1
-flake8-bugbear==21.11.29
+flake8-bugbear==22.1.11
 flake8-import-order==0.18.1
 importlib-metadata==4.2.0
 mccabe==0.6.1
@@ -11,19 +11,20 @@ pyflakes==2.4.0
 toml==0.10.2
 typed-ast==1.4.3
 types-bleach==4.1.4
-types-boto==2.49.5
+types-boto==2.49.6
 types-contextvars==2.4.1
-types-cryptography==3.3.11
-types-dataclasses==0.6.3
-types-docutils==0.17.3
-types-enum34==1.1.2
-types-ipaddress==1.0.2
-types-Markdown==3.3.11
-types-paramiko==2.8.7
+types-cryptography==3.3.14
+types-dataclasses==0.6.4
+types-docutils==0.17.4
+types-enum34==1.1.8
+types-ipaddress==1.0.7
+types-Markdown==3.3.12
+types-paramiko==2.8.10
 types-pkg-resources==0.1.3
-types-python-dateutil==2.8.5
-types-PyYAML==6.0.2
-types-requests==2.27.2
-types-six==1.16.8
+types-python-dateutil==2.8.8
+types-PyYAML==6.0.3
+types-requests==2.27.7
+types-six==1.16.10
+types-urllib3==1.26.7
 typing_extensions==4.0.1
 zipp==3.7.0
@@ -3,7 +3,7 @@
 adal==1.2.7
 aiofiles==0.8.0; python_version >= "3.6" and python_version < "4.0"
 amqp==5.0.9; python_version >= "3.7"
-anyio==3.4.0; python_version >= "3.7" and python_full_version >= "3.6.2"
+anyio==3.5.0; python_version >= "3.7" and python_full_version >= "3.6.2"
 appdirs==1.4.4
 argcomplete==2.0.0; python_version >= "3.6" and python_version < "4"
 asgiref==3.4.1; python_version >= "3.6"
@@ -50,7 +50,7 @@ ecdsa==0.17.0; python_version >= "2.6" and python_full_version < "3.0.0" or pyth
 edam-ontology==1.25.2
 fabric3==1.14.post1
 fastapi-utils==0.2.1; python_version >= "3.6" and python_version < "4.0"
-fastapi==0.71.0; python_full_version >= "3.6.1"
+fastapi==0.72.0; python_full_version >= "3.6.1"
 fs==2.4.14
 funcsigs==1.0.2
 future==0.18.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0")
@@ -63,11 +63,11 @@ h11==0.12.0; python_version >= "3.6"
 h5py==3.6.0; python_version >= "3.7"
 humanfriendly==10.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0"
 idna==3.3; python_full_version >= "3.6.2" and python_version >= "3.7" and python_version < "4"
-importlib-metadata==4.10.0; python_version == "3.7" and (python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.6.0" and python_version < "3.8" and python_version >= "3.7")
-importlib-resources==5.4.0; python_version < "3.9" and python_version >= "3.7"
+importlib-metadata==4.10.1; python_version == "3.7" and (python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.6.0" and python_version < "3.8" and python_version >= "3.7")
+importlib-resources==5.4.0; python_version >= "3.6"
 isa-rwval==0.10.10
 isodate==0.6.1; python_version >= "3.7" and python_version < "4"
-jsonschema==4.3.3; python_version >= "3.7"
+jsonschema==4.4.0; python_version >= "3.7"
 kombu==5.2.3; python_version >= "3.7"
 lagom==1.7.0; python_version >= "3.6" and python_version < "4.0"
 lockfile==0.12.2; python_version >= "3.7" and python_version < "4"
@@ -87,7 +87,7 @@ numpy==1.21.1; python_version >= "3.7"
 oauthlib==3.1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 oyaml==1.0
 packaging==21.3; python_version >= "3.6"
-paramiko==2.8.1
+paramiko==2.9.2
 parsley==1.3
 paste==3.5.0
 pastedeploy==2.1.1
@@ -108,10 +108,10 @@ pygments==2.11.2; python_full_version >= "3.6.2" and python_full_version < "4.0.
 pyjwt==2.3.0; python_version >= "3.6"
 pykwalify==1.8.0
 pynacl==1.5.0; python_version >= "3.6"
-pyparsing==3.0.6; python_version >= "3.6"
+pyparsing==3.0.7; python_version >= "3.6"
 pyreadline3==3.3; sys_platform == "win32" and python_version >= "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0")
 pyreadline==2.1; sys_platform == "win32" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0")
-pyrsistent==0.18.0; python_version >= "3.7"
+pyrsistent==0.18.1; python_version >= "3.7"
 pysam==0.18.0
 python-dateutil==2.8.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.3.0"
 python-jose==3.3.0
@@ -127,7 +127,7 @@ repoze.lru==0.7
 requests-oauthlib==1.3.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
 requests-toolbelt==0.9.1; python_version >= "3.6"
 requests==2.27.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
-rich==10.16.2; python_full_version >= "3.6.2" and python_full_version < "4.0.0"
+rich==11.0.0; python_full_version >= "3.6.2" and python_full_version < "4.0.0"
 routes==2.5.1
 rsa==4.8; python_version >= "3.6" and python_version < "4"
 ruamel.yaml.clib==0.2.6; platform_python_implementation == "CPython" and python_version < "3.10" and python_version >= "3.6"
@@ -140,7 +140,7 @@ sniffio==1.2.0; python_version >= "3.7" and python_full_version >= "3.6.2"
 social-auth-core==4.0.3
 sortedcontainers==2.4.0
 sqlalchemy-migrate==0.13.0
-sqlalchemy==1.4.29; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
+sqlalchemy==1.4.31; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
 sqlitedict==1.7.0
 sqlparse==0.4.2; python_version >= "3.5"
 starlette-context==0.3.3; python_version >= "3.7"
@@ -156,7 +156,7 @@ typing-extensions==4.0.1; python_version >= "3.6"
 tzlocal==2.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4"
 ubiquerg==0.6.2
 urllib3==1.26.8; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6"
-uvicorn==0.16.0
+uvicorn==0.17.0
 uvloop==0.16.0
 vine==5.0.0; python_version >= "3.7"
 wcwidth==0.2.5; python_full_version >= "3.6.2" and python_version >= "3.7"
@@ -0,0 +1,32 @@
+import json
+from logging import getLogger
+
+from .torque import Torque
+
+log = getLogger(__name__)
+
+
+class OpenPBS(Torque):
+
+    ERROR_MESSAGE_UNRECOGNIZED_ARG = 'Unrecognized long argument passed to OpenPBS CLI plugin: %s'
+
+    def get_status(self, job_ids=None):
+        return 'qstat -f -F json'
+
+    def get_single_status(self, job_id):
+        return f'qstat -f {job_id}'
+
+    def parse_status(self, status, job_ids):
+        try:
+            data = json.loads(status)
+        except Exception:
+            log.warning(f'No valid qstat JSON return from `qstat -f -F json`, got the following: {status}')
+        rval = {}
+        for job_id, job in data.get('Jobs', {}).items():
+            if job_id in job_ids:
+                # map PBS job states to Galaxy job states.
+                rval[id] = self._get_job_state(job['job_state'])
+        return rval
+
+
+__all__ = ('OpenPBS',)
@@ -5,7 +5,6 @@ from ..job import BaseJobExec, job_states

 log = getLogger(__name__)

-ERROR_MESSAGE_UNRECOGNIZED_ARG = 'Unrecognized long argument passed to Torque CLI plugin: %s'

 argmap = {'destination': '-q',
          'Execution_Time': '-a',
@@ -31,6 +30,8 @@ argmap = {'destination': '-q',

 class Torque(BaseJobExec):

+    ERROR_MESSAGE_UNRECOGNIZED_ARG = 'Unrecognized long argument passed to Torque CLI plugin: %s'
+
    def job_script_kwargs(self, ofile, efile, job_name):
        pbsargs = {'-o': ofile,
                   '-e': efile,
@@ -43,7 +44,7 @@ class Torque(BaseJobExec):
                    k = argmap[k]
                pbsargs[k] = v
            except KeyError:
-                log.warning(ERROR_MESSAGE_UNRECOGNIZED_ARG % k)
+                log.warning(self.ERROR_MESSAGE_UNRECOGNIZED_ARG, k)
        template_pbsargs = ''
        for k, v in pbsargs.items():
            template_pbsargs += f'#PBS {k} {v}\n'
@@ -4,6 +4,7 @@ import string
 import subprocess
 import sys
 import tempfile
+import time
 import unittest
 from typing import ClassVar, NamedTuple

@@ -12,6 +13,8 @@ from galaxy_test.base.populators import skip_without_tool
 from galaxy_test.driver import integration_util
 from .test_job_environments import BaseJobEnvironmentIntegrationTestCase

+PBS_STARTUP_DELAY = 5
+

 class RemoteConnection(NamedTuple):
    hostname: str
@@ -42,7 +45,11 @@ def start_ssh_docker(container_name, jobs_directory, port=10022, image='agaveapi
                          'nofile=2048:2048',
                          image]
    subprocess.check_call(START_SLURM_DOCKER)
+    if 'openpbs' in image:
+        time.sleep(PBS_STARTUP_DELAY)
    if sys.platform != 'darwin':
+        # Change testuser's uid to match current user id. This ensures that /home/testuser/.ssh/authorized_keys
+        # is owned by the right user and that job outputs can be cleaned up.
        subprocess.check_call(['docker', 'exec', container_name, 'usermod', '-u', str(os.getuid()), 'testuser'])
    return RemoteConnection('localhost', 'testuser', port, ssh_keys.private_key_file, ssh_keys.public_key_file)

@@ -120,14 +127,14 @@ class BaseCliIntegrationTestCase(BaseJobEnvironmentIntegrationTestCase):
        assert job_env.some_env == '42'


-class TorqueSetup:
-    job_plugin = 'Torque'
-    image = 'mvdbeek/galaxy-integration-docker-images:torque_latest'
+class OpenPBSSetup:
+    job_plugin = 'OpenPBS'
+    image = 'mvdbeek/galaxy-integration-docker-images:openpbs-22.01'


 class SlurmSetup:
    job_plugin = 'Slurm'
-    image = 'mvdbeek/galaxy-integration-docker-images:slurm_latest'
+    image = 'mvdbeek/galaxy-integration-docker-images:slurm-22.01'


 class ParamikoShell:
@@ -146,9 +153,9 @@ class ShellJobCliSlurmIntegrationTestCase(SlurmSetup, SecureShell, BaseCliIntegr
    pass


-class ParamikoCliTorqueIntegrationTestCase(TorqueSetup, ParamikoShell, BaseCliIntegrationTestCase):
+class ParamikoCliOpenPBSIntegrationTestCase(OpenPBSSetup, ParamikoShell, BaseCliIntegrationTestCase):
    pass


-class ShellJobCliTorqueIntegrationTestCase(TorqueSetup, SecureShell, BaseCliIntegrationTestCase):
+class ShellJobCliOpenPBSIntegrationTestCase(OpenPBSSetup, SecureShell, BaseCliIntegrationTestCase):
    pass