From f9baa0b4839c70e14d5e66f988214c3b05abebdd Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 5 Jun 2019 17:47:59 -0400 Subject: [PATCH] Stub out job_exection package for remote job finish code. --- lib/galaxy/job_execution/__init__.py | 0 .../output_collect.py | 0 lib/galaxy/tools/__init__.py | 4 +- packages/job_execution/HISTORY.rst | 12 +++ packages/job_execution/LICENSE | 1 + packages/job_execution/MANIFEST.in | 2 + packages/job_execution/Makefile | 1 + packages/job_execution/README.rst | 14 +++ packages/job_execution/dev-requirements.txt | 1 + packages/job_execution/galaxy/__init__.py | 1 + packages/job_execution/galaxy/job_execution | 1 + .../galaxy/project_galaxy_job_execution.py | 13 +++ packages/job_execution/requirements.txt | 2 + packages/job_execution/scripts | 1 + packages/job_execution/setup.cfg | 1 + packages/job_execution/setup.py | 100 ++++++++++++++++++ packages/test.sh | 3 +- 17 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 lib/galaxy/job_execution/__init__.py rename lib/galaxy/{tool_util => job_execution}/output_collect.py (100%) create mode 100644 packages/job_execution/HISTORY.rst create mode 120000 packages/job_execution/LICENSE create mode 100644 packages/job_execution/MANIFEST.in create mode 120000 packages/job_execution/Makefile create mode 100644 packages/job_execution/README.rst create mode 120000 packages/job_execution/dev-requirements.txt create mode 100644 packages/job_execution/galaxy/__init__.py create mode 120000 packages/job_execution/galaxy/job_execution create mode 100644 packages/job_execution/galaxy/project_galaxy_job_execution.py create mode 100644 packages/job_execution/requirements.txt create mode 120000 packages/job_execution/scripts create mode 120000 packages/job_execution/setup.cfg create mode 100644 packages/job_execution/setup.py diff --git a/lib/galaxy/job_execution/__init__.py b/lib/galaxy/job_execution/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/galaxy/tool_util/output_collect.py b/lib/galaxy/job_execution/output_collect.py similarity index 100% rename from lib/galaxy/tool_util/output_collect.py rename to lib/galaxy/job_execution/output_collect.py diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 8227ef08ba0..7aa13edf6d9 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -25,11 +25,11 @@ from galaxy import ( exceptions, model ) +from galaxy.job_execution import output_collect from galaxy.managers.jobs import JobSearch from galaxy.metadata import get_metadata_compute_strategy from galaxy.model.tags import GalaxyTagHandler from galaxy.queue_worker import send_control_task -from galaxy.tool_util import output_collect from galaxy.tool_util.deps import ( CachedDependencyManager, ) @@ -1737,6 +1737,8 @@ class Tool(Dictifiable): Find any additional datasets generated by a tool and attach (for cases where number of outputs is not known in advance). """ + # given the job_execution import is the only one, probably makes sense to refactor this out + # into job_wrapper. tool = self permission_provider = output_collect.PermissionProvider(inp_data, tool.app.security_agent, job) metadata_source_provider = output_collect.MetadataSourceProvider(inp_data) diff --git a/packages/job_execution/HISTORY.rst b/packages/job_execution/HISTORY.rst new file mode 100644 index 00000000000..ec2f0e746e9 --- /dev/null +++ b/packages/job_execution/HISTORY.rst @@ -0,0 +1,12 @@ +.. :changelog: + +History +------- + +.. to_doc + +--------------------- +19.9.0.dev0 +--------------------- + +* Initial import from dev branch of Galaxy during 19.09 development cycle. diff --git a/packages/job_execution/LICENSE b/packages/job_execution/LICENSE new file mode 120000 index 00000000000..1ef648f64b3 --- /dev/null +++ b/packages/job_execution/LICENSE @@ -0,0 +1 @@ +../../LICENSE.txt \ No newline at end of file diff --git a/packages/job_execution/MANIFEST.in b/packages/job_execution/MANIFEST.in new file mode 100644 index 00000000000..2bf148a5764 --- /dev/null +++ b/packages/job_execution/MANIFEST.in @@ -0,0 +1,2 @@ +include *.rst LICENSE + diff --git a/packages/job_execution/Makefile b/packages/job_execution/Makefile new file mode 120000 index 00000000000..37af8bae5ba --- /dev/null +++ b/packages/job_execution/Makefile @@ -0,0 +1 @@ +../package.Makefile \ No newline at end of file diff --git a/packages/job_execution/README.rst b/packages/job_execution/README.rst new file mode 100644 index 00000000000..897422a8aa3 --- /dev/null +++ b/packages/job_execution/README.rst @@ -0,0 +1,14 @@ + +.. image:: https://badge.fury.io/py/galaxy-util.svg + :target: https://pypi.python.org/pypi/galaxy-util/ + + +Overview +-------- + +The Galaxy_ job execution runtime module. + +* Free software: Academic Free License version 3.0 +* Code: https://github.com/galaxyproject/galaxy + +.. _Galaxy: http://galaxyproject.org/ diff --git a/packages/job_execution/dev-requirements.txt b/packages/job_execution/dev-requirements.txt new file mode 120000 index 00000000000..467b90d7a23 --- /dev/null +++ b/packages/job_execution/dev-requirements.txt @@ -0,0 +1 @@ +../package-dev-requirements.txt \ No newline at end of file diff --git a/packages/job_execution/galaxy/__init__.py b/packages/job_execution/galaxy/__init__.py new file mode 100644 index 00000000000..69e3be50dac --- /dev/null +++ b/packages/job_execution/galaxy/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/packages/job_execution/galaxy/job_execution b/packages/job_execution/galaxy/job_execution new file mode 120000 index 00000000000..4ef157cced2 --- /dev/null +++ b/packages/job_execution/galaxy/job_execution @@ -0,0 +1 @@ +../../../lib/galaxy/job_execution \ No newline at end of file diff --git a/packages/job_execution/galaxy/project_galaxy_job_execution.py b/packages/job_execution/galaxy/project_galaxy_job_execution.py new file mode 100644 index 00000000000..a30f915133d --- /dev/null +++ b/packages/job_execution/galaxy/project_galaxy_job_execution.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- + +__version__ = '19.9.0.dev0' + +PROJECT_NAME = "galaxy-job-execution" +PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" +PROJECT_URL = "https://github.com/galaxyproject/galaxy" +PROJECT_AUTHOR = 'Galaxy Project and Community' +PROJECT_DESCRIPTION = 'Galaxy Job Execution Runtime Utilities' +PROJECT_EMAIL = 'jmchilton@gmail.com' +RAW_CONTENT_URL = "https://raw.github.com/%s/%s/master/" % ( + PROJECT_USERAME, PROJECT_NAME +) diff --git a/packages/job_execution/requirements.txt b/packages/job_execution/requirements.txt new file mode 100644 index 00000000000..77de615ea9c --- /dev/null +++ b/packages/job_execution/requirements.txt @@ -0,0 +1,2 @@ +galaxy-data + diff --git a/packages/job_execution/scripts b/packages/job_execution/scripts new file mode 120000 index 00000000000..9aec9dc5a06 --- /dev/null +++ b/packages/job_execution/scripts @@ -0,0 +1 @@ +../build_scripts \ No newline at end of file diff --git a/packages/job_execution/setup.cfg b/packages/job_execution/setup.cfg new file mode 120000 index 00000000000..eb7cf09393f --- /dev/null +++ b/packages/job_execution/setup.cfg @@ -0,0 +1 @@ +../../setup.cfg \ No newline at end of file diff --git a/packages/job_execution/setup.py b/packages/job_execution/setup.py new file mode 100644 index 00000000000..0ea5361f7d5 --- /dev/null +++ b/packages/job_execution/setup.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import ast +import os +import re +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + +SOURCE_DIR = "galaxy" + +_version_re = re.compile(r'__version__\s+=\s+(.*)') + +with open('%s/project_galaxy_job_execution.py' % SOURCE_DIR, 'rb') as f: + init_contents = f.read().decode('utf-8') + + def get_var(var_name): + pattern = re.compile(r'%s\s+=\s+(.*)' % var_name) + match = pattern.search(init_contents).group(1) + return str(ast.literal_eval(match)) + + version = get_var("__version__") + PROJECT_NAME = get_var("PROJECT_NAME") + PROJECT_URL = get_var("PROJECT_URL") + PROJECT_AUTHOR = get_var("PROJECT_AUTHOR") + PROJECT_EMAIL = get_var("PROJECT_EMAIL") + PROJECT_DESCRIPTION = get_var("PROJECT_DESCRIPTION") + +TEST_DIR = 'tests' +PACKAGES = [ + 'galaxy', + 'galaxy.job_execution', +] +ENTRY_POINTS = ''' + [console_scripts] +''' +PACKAGE_DATA = { + # Be sure to update MANIFEST.in for source dist. + 'galaxy': [ + ], +} +PACKAGE_DIR = { + SOURCE_DIR: SOURCE_DIR, +} + +readme = open('README.rst').read() +history = open('HISTORY.rst').read().replace('.. :changelog:', '') + +if os.path.exists("requirements.txt"): + requirements = open("requirements.txt").read().split("\n") +else: + # In tox, it will cover them anyway. + requirements = [] + + +test_requirements = [ + # TODO: put package test requirements here +] + + +setup( + name=PROJECT_NAME, + version=version, + description=PROJECT_DESCRIPTION, + long_description=readme + '\n\n' + history, + long_description_content_type='text/x-rst', + author=PROJECT_AUTHOR, + author_email=PROJECT_EMAIL, + url=PROJECT_URL, + packages=PACKAGES, + entry_points=ENTRY_POINTS, + package_data=PACKAGE_DATA, + package_dir=PACKAGE_DIR, + include_package_data=True, + install_requires=requirements, + extras_require={}, + license="AFL", + zip_safe=False, + keywords='galaxy', + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Environment :: Console', + 'License :: OSI Approved :: Academic Free License (AFL)', + 'Operating System :: POSIX', + 'Topic :: Software Development', + 'Topic :: Software Development :: Code Generators', + 'Topic :: Software Development :: Testing', + 'Natural Language :: English', + "Programming Language :: Python :: 2", + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + ], + test_suite=TEST_DIR, + tests_require=test_requirements +) diff --git a/packages/test.sh b/packages/test.sh index 5deb29bf919..a597d69ee1a 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -20,10 +20,11 @@ PACKAGE_DIRS=( containers tool_util data + job_execution ) # containers has no tests, tool_util not yet working 100%, # data has many problems quota, tool shed install database, etc.. -RUN_TESTS=(1 1 1 0 0 0 0) +RUN_TESTS=(1 1 1 0 0 0 0 0) for ((i=0; i<${#PACKAGE_DIRS[@]}; i++)); do package_dir=${PACKAGE_DIRS[$i]}