opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into remote_chart_plugins

Browse Source
This commit is contained in:
guerler
2016-10-14 16:24:39 -04:00
parent fe73009db5 a895c823da
commit f053f9bb3d
123 changed files with 2319 additions and 653 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.ci/flake8_blacklist.txt
+2 -2
View File
@@ -1,8 +1,8 @@
.venv/
.tox/
eggs/
client/
client/node_modules/
database/
doc/patch.py
doc/source/conf.py
eggs/
lib/galaxy/util/jstree.py
.ci/flake8_lint_include_list.txt
+34 -7
View File
@@ -1,8 +1,10 @@
client/galaxy/style/source_material/circle.py
contrib/
cron/add_manual_builds.py
cron/cleanup_datasets.py
cron/parse_builds_3_sites.py
cron/parse_builds.py
doc/parse_gx_xsd.py
lib/galaxy/actions/
lib/galaxy/auth/__init__.py
lib/galaxy/auth/providers/__init__.py
@@ -93,6 +95,7 @@ lib/galaxy/main.py
lib/galaxy/managers/annotatable.py
lib/galaxy/managers/api_keys.py
lib/galaxy/managers/base.py
lib/galaxy/managers/collections_util.py
lib/galaxy/managers/context.py
lib/galaxy/managers/deletable.py
lib/galaxy/managers/__init__.py
@@ -101,7 +104,9 @@ lib/galaxy/managers/libraries.py
lib/galaxy/managers/secured.py
lib/galaxy/managers/taggable.py
lib/galaxy/model/__init__.py
lib/galaxy/model/item_attrs.py
lib/galaxy/model/mapping.py
lib/galaxy/model/metadata.py
lib/galaxy/model/migrate/__init__.py
lib/galaxy/model/migrate/versions/0001_initial_tables.py
lib/galaxy/model/migrate/versions/0002_metadata_file_table.py
@@ -244,9 +249,11 @@ lib/galaxy/sample_tracking/sample.py
lib/galaxy/security/validate_user_input.py
lib/galaxy/tags/
lib/galaxy/tools/actions/metadata.py
lib/galaxy/tools/cwl
lib/galaxy/tools/cwl/
lib/galaxy/tools/data_manager/__init__.py
lib/galaxy/tools/deps
lib/galaxy/tools/deps/
lib/galaxy/tools/exception_handling.py
lib/galaxy/tools/execute.py
lib/galaxy/tools/filters/
lib/galaxy/tools/imp_exp/export_history.py
lib/galaxy/tools/imp_exp/__init__.py
@@ -254,11 +261,15 @@ lib/galaxy/tools/linters/
lib/galaxy/tools/lint.py
lib/galaxy/tools/lint_util.py
lib/galaxy/tools/loader_directory.py
lib/galaxy/tools/loader.py
lib/galaxy/tools/parameters/dataset_matcher.py
lib/galaxy/tools/parameters/history_query.py
lib/galaxy/tools/parameters/__init__.py
lib/galaxy/tools/parameters/input_translation.py
lib/galaxy/tools/parameters/sanitize.py
lib/galaxy/tools/parameters/validation.py
lib/galaxy/tools/parameters/wrapped_json.py
lib/galaxy/tools/parameters/wrapped.py
lib/galaxy/tools/parser/
lib/galaxy/tools/special_tools.py
lib/galaxy/tools/test.py
@@ -345,9 +356,11 @@ lib/galaxy/web/framework/middleware/statsd.py
lib/galaxy/web/framework/middleware/translogger.py
lib/galaxy/web/framework/middleware/xforwardedhost.py
lib/galaxy/web/params.py
lib/galaxy/workflow/extract.py
lib/galaxy/workflow/__init__.py
lib/galaxy/workflow/render.py
lib/galaxy/workflow/schedulers/__init__.py
lib/galaxy/workflow/run.py
lib/galaxy/workflow/schedulers/
lib/galaxy/workflow/steps.py
lib/galaxy/work/__init__.py
lib/mimeparse.py
@@ -440,6 +453,7 @@ scripts/tool_shed/migrate_tools_to_repositories.py
scripts/tool_shed/show_tool_dependency_installation_dir_contents.py
scripts/tools/re_escape_output.py
test/api/__init__.py
test/api/test_dataset_collections.py
test/api/test_datasets.py
test/api/test_datatypes.py
test/api/test_framework.py
@@ -447,27 +461,39 @@ test/api/test_histories.py
test/api/test_history_contents_provenance.py
test/api/test_jobs.py
test/api/test_libraries.py
test/api/test_page_revisions.py
test/api/test_search.py
test/api/test_tool_data.py
test/api/test_tools.py
test/api/test_tours.py
test/api/test_workflow_extraction.py
test/api/test_workflows_from_yaml.py
test/api/workflows_format_2/interface.py
test/api/workflows_format_2/main.py
test/api/workflows_format_2/
test/base/api_asserts.py
test/base/api.py
test/base/api_util.py
test/base/__init__.py
test/base/instrument.py
test/base/integration_util.py
test/base/interactor.py
test/base/nose_util.py
test/base/test_logging.py
test/base/tool_shed_util.py
test/base/twilltestcase.py
test/casperjs/
test/functional/database_contexts.py
test/functional/__init__.py
test/functional/test_data_managers.py
test/functional/test_toolbox.py
test/functional/tool-data/
test/functional/tools/
test/integration/__init__.py
test/integration/test_pulsar_embedded.py
test/functional/workflow.py
test/integration/
test/manual/__init__.py
test/shed_functional/base/common.py
test/shed_functional/base/__init__.py
test/shed_functional/functional/
test/shed_functional/functional_tests.py
test/shed_functional/__init__.py
test/unit/dataset_collections/__init__.py
test/unit/datatypes/dataproviders/__init__.py
@@ -502,6 +528,7 @@ test/unit/tools/test_tool_deps.py
test/unit/tools/test_tool_loader.py
test/unit/tools/test_toolbox_filters.py
test/unit/tools/test_watcher.py
test/unit/tools_support.py
test/unit/unittest_utils/__init__.py
test/unit/unittest_utils/utility.py
test/unit/visualizations/__init__.py
.ci/py3_sources.txt
+14
View File
@@ -1,5 +1,7 @@
client/galaxy/style/source_material/circle.py
contrib/
cron/
doc/parse_gx_xsd.py
lib/galaxy/actions/
lib/galaxy/auth/
lib/galaxy/config.py
@@ -26,8 +28,11 @@ lib/galaxy/forms/
lib/galaxy/jobs/
lib/galaxy/managers/
lib/galaxy/model/__init__.py
lib/galaxy/model/item_attrs.py
lib/galaxy/model/mapping.py
lib/galaxy/model/metadata.py
lib/galaxy/model/migrate/
lib/galaxy/model/orm/now.py
lib/galaxy/objectstore/
lib/galaxy/openid/
lib/galaxy/quota/
@@ -36,17 +41,26 @@ lib/galaxy/security/
lib/galaxy/tags/
lib/galaxy/tools/cwl/
lib/galaxy/tools/deps/
lib/galaxy/tools/exception_handling.py
lib/galaxy/tools/execute.py
lib/galaxy/tools/lint.py
lib/galaxy/tools/lint_util.py
lib/galaxy/tools/linters/
lib/galaxy/tools/loader.py
lib/galaxy/tools/loader_directory.py
lib/galaxy/tools/parameters/dataset_matcher.py
lib/galaxy/tools/parameters/__init__.py
lib/galaxy/tools/parameters/wrapped_json.py
lib/galaxy/tools/parameters/wrapped.py
lib/galaxy/tools/parser/
lib/galaxy/tools/toolbox/
lib/galaxy/tours/
lib/galaxy/util/
lib/galaxy/visualization/
lib/galaxy/work/
lib/galaxy/workflow/extract.py
lib/galaxy/workflow/run.py
lib/galaxy/workflow/schedulers/core.py
lib/galaxy_ext/
lib/galaxy_utils/
lib/log_tempfile.py
.travis.yml
-3
View File
@@ -24,9 +24,6 @@ matrix:
allow_failures:
- env: TOX_ENV=py27-lint-imports
before_install:
- if [ `uname` == "Darwin" ]; then bash -c "brew update && brew install python"; fi
install:
- pip install tox
- if [ "$TOX_ENV" == "validate-test-tools" ]; then sudo apt-get install libxml2-utils; fi
client/galaxy/scripts/mvc/tool/tools.js
+6
View File
@@ -633,6 +633,12 @@ var ToolSearchView = Backbone.View.extend({
if (!this.model.is_visible()) {
this.$el.hide();
}
// Adjust top for issue 2907 depending on whether the messagebox is visible.
if ($("#messagebox").is(":visible")) {
this.$el.css("top","95px");
}
this.$el.find('[title]').tooltip();
return this;
},
client/galaxy/style/source_material/circle.py
+2 -3
View File
@@ -1,15 +1,14 @@
#!/usr/bin/env python
"""
usage: %prog width height bg_color hatch_color [color alpha stop_pos] +
"""
from __future__ import division
import sys
import cairo
from math import pi
import cairo
assert sys.version_info[:2] >= ( 2, 4 )
config/galaxy.ini.sample
+27 -3
View File
@@ -203,7 +203,7 @@ paste.app_factory = galaxy.web.buildapp:app_factory
# Pass debug flag to conda commands.
#conda_debug = False
# conda channels to enable by default (http://conda.pydata.org/docs/custom-channels.html)
#conda_ensure_channels = r,bioconda,iuc
#conda_ensure_channels = conda-forge,r,bioconda,iuc
# Set to True to instruct Galaxy to look for and install missing tool
# dependencies before each job runs.
#conda_auto_install = False
@@ -228,6 +228,31 @@ paste.app_factory = galaxy.web.buildapp:app_factory
# than the watchdog default.
#watch_tools = False
# Enable Galaxy to fetch Docker containers registered with quay.io generated
# from tool requirements resolved through conda. These containers (when
# available) have been generated using mulled - https://github.com/mulled.
# These containers are highly beta and availablity will vary by tool.
# This option will additionally only be used for job destinations with
# Docker enabled.
#enable_beta_mulled_containers = False
# Container resolvers configuration (beta). Setup a file describing container
# resolvers to use when discovering containers for Galaxy. If this is set to
# None, the default containers loaded is determined by
# enable_beta_mulled_containers.
#containers_resolvers_config_file = None
# involucro is a tool used to build Docker containers for tools from Conda
# dependencies referenced in tools as `requirement`s. The following path is
# the location of involucro on the Galaxy host. This is ignored if the relevant
# container resolver isn't enabled, and will install on demand unless
# involucro_auto_init is set to False.
#involucro_path = database/dependencies/involucro
# Install involucro as needed to build Docker containers for tools. Ignored if
# relevant container resolver is not used.
#involucro_auto_init = True
# Enable automatic polling of relative tool sheds to see if any updates
# are available for installed repositories. Ideally only one Galaxy
# server process should be able to check for repository updates. The
@@ -861,8 +886,7 @@ use_interactive = True
# Galaxy encodes various internal values when these values will be output in
# some format (for example, in a URL or cookie). You should set a key to be
# used by the algorithm that encodes and decodes these values. It can be any
# string. If left unchanged, anyone could construct a cookie that would grant
# them access to others' sessions.
# string.
# One simple way to generate a value for this is with the shell command:
# python -c 'import time; print time.time()' | md5sum | cut -f 1 -d ' '
#id_secret = USING THE DEFAULT IS NOT SECURE!
config/job_conf.xml.sample_advanced
+6
View File
@@ -362,6 +362,12 @@
a fallback. -->
<!-- <param id="docker_default_container_id">busybox:ubuntu-14.04</param> -->
<!-- If the destination should be secured to only allow containerized jobs
the following parameter may be set for the job destination. Not all,
or even most, tools available in Galaxy core or in the Tool Shed
support Docker yet so this option may require a lot of extra work for
the deployer. -->
<!-- <param id="require_container">true</param> -->
</destination>
<destination id="pbs" runner="pbs" tags="mycluster"/>
<destination id="pbs_longjobs" runner="pbs" tags="mycluster,longjobs">
doc/parse_gx_xsd.py
+2 -4
View File
@@ -2,14 +2,12 @@
# TODO: Add examples, tables and best practice links to command
# TODO: Examples of truevalue, falsevalue
# TODO: Test param extra_file
from __future__ import print_function
# Things dropped from TOC (still documented inside schema).
# - request_parameter_translation
from __future__ import print_function
from lxml import etree
from StringIO import StringIO
from six import StringIO
with open("doc/schema_template.md", "r") as f:
MARKDOWN_TEMPLATE = f.read()
doc/source/admin/dependency_resolvers.rst
+1 -1
View File
@@ -182,7 +182,7 @@ debug
ensure_channels
conda channels to enable by default. See
http://conda.pydata.org/docs/custom-channels.html for more
information about channels. (default: r,bioconda).
information about channels. (default: conda-forge,r,bioconda,iuc).
auto_install
Set to True to instruct Galaxy to look for and install missing tool
doc/source/admin/index.rst
+2
View File
@@ -11,6 +11,8 @@ documentation. These resources should be used together.
dependency_resolvers.rst
conda_faq.rst
mulled_containers.rst
interactive_environments.rst
doc/source/admin/mulled_containers.rst
+128
View File
@@ -0,0 +1,128 @@
=================================
Containers for Tool Dependencies
=================================
Galaxy tools (also called wrappers) are able to use Conda packages
(see more information in our `Galaxy Conda documentation`_) and Docker containers as dependency resolvers.
The IUC_ recommends to use Conda packages as primary dependency resolver, mainly because Docker is not
available on every (HPC-) system. Conda on the other hand can be installed by Galaxy and maintained
entirely in user-space. Nevertheless, Docker (Containers in general) has some unique features and
there are many use-cases in the Galaxy community which makes containerized systems very appealing.
Since 2014 Galaxy supports running tools in Docker containers via a special `container annotation`_ inside of the
requirement field.
.. code-block:: xml
<requirements>
<!-- Container based dependency handling -->
<container type="docker">busybox:ubuntu-14.04</container>
<!-- Conda based dependency handling -->
<requirement type="package" version="8.22">gnu_coreutils</requirement>
</requirements>
This approach has shown two limitations that slowed down the adoption by tool developers.
First, every tool needs to be annotated with a container name (as shown above) and this container needs
to be created beforehand, usually manually. The second reason is that a Galaxy tool aims to be deployed everywhere,
independet of the underlying system, meaning if Docker is not available Galaxy should use Conda packages.
This puts an additional burden on tool developers who need to take care of two dependency resolvers. This setup can cause
different tool results depending on the resolver, because both the Conda package and the Docker container are
usually not created out of the same recipe and maybe were compiled in a different way, use different sources etc.
Not an ideal solution and something we wanted to solve.
Here we demonstrate a solution that can create Containers out of Conda packages automatically.
This can be either used to support communities like BioContainers_ to create Containers
before deploying a Galaxy tool, or this can be used by Galaxy to create Containers on-demand and on-the-fly if one
is not available already.
Automatic build of Linux containers
-----------------------------------
We utilize [mulled](https://github.com/mulled/mulled) with [involucro](https://github.com/involucro/involucro)
in an automatic way. This is for example used to convert all packages in bioconda_ into Linux Containers
(Docker and rkt at the moment) and made available at the `BioContainers Quay.io account`_.
We have developed small utilities around this technology stack which is currently included in galaxy-lib_.
Here is a short introduction:
Search for containers
^^^^^^^^^^^^^^^^^^^^^
This will search for containers in the biocontainers organisation.
.. code-block:: bash
$ mulled-search -s vsearch -o biocontainers
Build all packages from bioconda from the last 24h
^^^^^^^^^^^^^^^^^^^^^
The BioConda community is building a container for every package they create with a command similar to this.
.. code-block:: bash
$ mulled-build-channel --channel bioconda --namespace biocontainers \
--involucro-path ./involucro --recipes-dir ./bioconda-recipes --diff-hours 25 build
Building Docker containers for local Conda packages
^^^^^^^^^^^^^^^^^^^^^
Conda packages can be tested with creating a busybox based container for this particular package in the following way.
This also demonstrates how you can build a container locally and on-the-fly.
> we modified the samtools package to version 3.0 to make clear we are using a local version
1) build your recipe
.. code-block:: bash
$ conda build recipes/samtools
2) index your local builds
.. code-block:: bash
$ conda index /home/bag/miniconda2/conda-bld/linux-64/
3) build a container for your local package
.. code-block:: bash
$ mulled-build build-and-test 'samtools=3.0--0' \
--extra-channel file://home/bag/miniconda2/conda-bld/ --test 'samtools --help'
The ``--0`` indicates the build version of the conda package. It is recommended to specify this number otherwise
you will override already existing images. For Python Conda packages this extension might look like this ``--py35_1``.
Build, test and push a conda-forge package to biocontainers
^^^^^^^^^^^^^^^^^^^^^
> You need to have write access to the biocontainers repository
You can build packages from other Conda channels as well, not only from BioConda. ``pandoc`` is available from the
conda-forge channel and conda-forge is also enabled by default in Galaxy. To build ``pandoc`` and push it to biocontainrs
you could do something along these lines.
.. code-block:: bash
$ mulled-build build-and-test 'pandoc=1.17.2--0' --test 'pandoc --help' -n biocontainers
.. code-block:: bash
$ mulled-build push 'pandoc=1.17.2--0' --test 'pandoc --help' -n biocontainers
.. _Galaxy Conda documentation: ./conda_faq.rst
.. _IUC: https://wiki.galaxyproject.org/IUC
.. _container annotation: https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/catDocker.xml#L4
.. _BioContainers: https://github.com/biocontainers
.. _bioconda: https://github.com/bioconda/bioconda-recipes
.. _BioContainers Quay.io account: https://quay.io/organization/biocontainers
.. _galaxy-lib: https://github.com/galaxyproject/galaxy-lib
lib/galaxy/config.py
+19 -1
View File
@@ -226,6 +226,7 @@ class Configuration( object ):
self.local_task_queue_workers = int(kwargs.get("local_task_queue_workers", 2))
self.tool_submission_burst_threads = int( kwargs.get( 'tool_submission_burst_threads', '1' ) )
self.tool_submission_burst_at = int( kwargs.get( 'tool_submission_burst_at', '10' ) )
# Enable new interface for API installations from TS.
# Admin menu will list both if enabled.
self.enable_beta_ts_api_install = string_as_bool( kwargs.get( 'enable_beta_ts_api_install', 'False' ) )
@@ -323,6 +324,19 @@ class Configuration( object ):
else:
self.tool_dependency_dir = None
self.use_tool_dependencies = os.path.exists(self.dependency_resolvers_config_file)
self.enable_beta_mulled_containers = string_as_bool( kwargs.get( 'enable_beta_mulled_containers', 'False' ) )
containers_resolvers_config_file = kwargs.get( 'containers_resolvers_config_file', None )
if containers_resolvers_config_file:
containers_resolvers_config_file = resolve_path(containers_resolvers_config_file, self.root)
self.containers_resolvers_config_file = containers_resolvers_config_file
involucro_path = kwargs.get('involucro_path', None)
if involucro_path is None:
involucro_path = os.path.join(tool_dependency_dir, "involucro")
self.involucro_path = resolve_path(involucro_path, self.root)
self.involucro_auto_init = string_as_bool(kwargs.get( 'involucro_auto_init', True))
# Configuration options for taking advantage of nginx features
self.upstream_gzip = string_as_bool( kwargs.get( 'upstream_gzip', False ) )
self.apache_xsendfile = string_as_bool( kwargs.get( 'apache_xsendfile', False ) )
@@ -846,7 +860,11 @@ class ConfiguresGalaxyMixin:
default_file_path=file_path,
outputs_to_working_directory=self.config.outputs_to_working_directory,
container_image_cache_path=self.config.container_image_cache_path,
library_import_dir=self.config.library_import_dir
library_import_dir=self.config.library_import_dir,
enable_beta_mulled_containers=self.config.enable_beta_mulled_containers,
containers_resolvers_config_file=self.config.containers_resolvers_config_file,
involucro_path=self.config.involucro_path,
involucro_auto_init=self.config.involucro_auto_init,
)
self.container_finder = containers.ContainerFinder(app_info)
lib/galaxy/datatypes/images.py
+1 -9
View File
@@ -8,18 +8,10 @@ from six.moves.urllib.parse import quote_plus
from galaxy.datatypes.binary import Binary
from galaxy.datatypes.sniff import get_headers
from galaxy.datatypes.text import Html as HtmlFromText
from galaxy.datatypes.util.image_util import check_image_type
from galaxy.util import nice_size
from galaxy.util.image_util import check_image_type
from . import data
try:
import Image as PIL
except ImportError:
try:
from PIL import Image as PIL
except:
PIL = None
log = logging.getLogger(__name__)
# TODO: Uploading image files of various types is supported in Galaxy, but on
lib/galaxy/datatypes/registry.py
+6 -15
View File
@@ -434,27 +434,18 @@ class Registry( object ):
Return the datatype class where the datatype's `type` attribute
(as defined in the datatype_conf.xml file) contains `name`.
"""
# TODO: too roundabout - would be better to generate this once as a map and store in this object
# TODO: obviously not ideal but some of these base classes that are useful for testing datatypes
# aren't loaded into the datatypes registry, so we'd need to test for them here
if name == 'images.Image':
return images.Image
# TODO: too inefficient - would be better to generate this once as a map and store in this object
for ext, datatype_obj in self.datatypes_by_extension.items():
datatype_obj_class = datatype_obj.__class__
datatype_obj_class_str = str( datatype_obj_class )
if name in datatype_obj_class_str:
return datatype_obj_class
return None
# these seem to be connected to the dynamic classes being generated in this file, lines 157-158
# they appear when a one of the three are used in inheritance with subclass="True"
# TODO: a possible solution is to def a fn in datatypes __init__ for creating the dynamic classes
# remap = {
# 'galaxy.datatypes.registry.Tabular' : galaxy.datatypes.tabular.Tabular,
# 'galaxy.datatypes.registry.Text' : galaxy.datatypes.data.Text,
# 'galaxy.datatypes.registry.Binary' : galaxy.datatypes.binary.Binary
# }
# datatype_str = str( datatype )
# if datatype_str in remap:
# datatype = remap[ datatype_str ]
#
# return datatype
def get_available_tracks( self ):
return self.available_tracks
lib/galaxy/datatypes/sequence.py
+1 -1
View File
@@ -15,9 +15,9 @@ from six import PY3
from galaxy import util
from galaxy.datatypes import metadata
from galaxy.util.checkers import is_gzip
from galaxy.util.image_util import check_image_type
from galaxy.datatypes.metadata import MetadataElement
from galaxy.datatypes.sniff import get_headers
from galaxy.datatypes.util.image_util import check_image_type
from galaxy.util import nice_size
from . import data
lib/galaxy/datatypes/util/image_util.py
-78
View File
@@ -1,78 +0,0 @@
"""Provides utilities for working with image files."""
import logging
import imghdr
try:
import Image as PIL
except ImportError:
try:
from PIL import Image as PIL
except:
PIL = None
log = logging.getLogger(__name__)
def image_type( filename, image=None ):
format = ''
if PIL is not None:
if image is not None:
format = image.format
else:
try:
im = PIL.open( filename )
format = im.format
im.close()
except:
return False
else:
format = imghdr.what( filename )
if format is not None:
format = format.upper()
else:
return False
return format
def check_image_type( filename, types, image=None ):
format = image_type( filename, image )
# First check if we can use PIL
if format in types:
return True
return False
def get_image_ext( file_path, image ):
# determine ext
format = image_type( file_path, image )
if format in [ 'JPG', 'JPEG' ]:
return 'jpg'
if format == 'PNG':
return 'png'
if format == 'TIFF':
return 'tiff'
if format == 'BMP':
return 'bmp'
if format == 'GIF':
return 'gif'
if format == 'IM':
return 'im'
if format == 'PCD':
return 'pcd'
if format == 'PCX':
return 'pcx'
if format == 'PPM':
return 'ppm'
if format == 'PSD':
return 'psd'
if format == 'XBM':
return 'xbm'
if format == 'XPM':
return 'xpm'
if format == 'RGB':
return 'rgb'
if format == 'PBM':
return 'pbm'
if format == 'PGM':
return 'pgm'
return None
lib/galaxy/jobs/__init__.py
+4
View File
@@ -803,6 +803,10 @@ class JobWrapper( object ):
"""
return self.get_destination_configuration("cleanup_job", DEFAULT_CLEANUP_JOB)
@property
def requires_containerization(self):
return util.asbool(self.get_destination_configuration("require_container", "False"))
def can_split( self ):
# Should the job handler split this job up?
return self.app.config.use_tasked_jobs and self.tool.parallelism
lib/galaxy/jobs/runners/__init__.py
+2
View File
@@ -198,6 +198,8 @@ class BaseJobRunner( object ):
def build_command_line( self, job_wrapper, include_metadata=False, include_work_dir_outputs=True,
modify_command_for_container=True ):
container = self._find_container( job_wrapper )
if not container and job_wrapper.requires_containerization:
raise Exception("Failed to find a container when required, contact Galaxy admin.")
return build_command(
self,
job_wrapper,
lib/galaxy/managers/collections_util.py
+6 -7
View File
@@ -1,8 +1,7 @@
from galaxy import exceptions
from galaxy import web
from galaxy import model
import logging
from galaxy import exceptions, model, web
log = logging.getLogger( __name__ )
ERROR_MESSAGE_UNKNOWN_SRC = "Unknown dataset source (src) %s."
@@ -77,7 +76,7 @@ def dictify_dataset_collection_instance( dataset_collection_instance, parent, se
dict_value[ 'url' ] = web.url_for( 'library_content', library_id=encoded_library_id, id=encoded_id, folder_id=encoded_folder_id )
if view == "element":
collection = dataset_collection_instance.collection
dict_value[ 'elements' ] = map( dictify_element, collection.elements )
dict_value[ 'elements' ] = [ dictify_element(_) for _ in collection.elements ]
dict_value[ 'populated' ] = collection.populated
security.encode_all_ids( dict_value, recursive=True ) # TODO: Use Kyle's recursive formulation of this.
return dict_value
@@ -89,10 +88,10 @@ def dictify_element( element ):
if element.child_collection:
# Recursively yield elements for each nested collection...
child_collection = element.child_collection
object_detials[ "elements" ] = map( dictify_element, child_collection.elements )
object_detials[ "elements" ] = [ dictify_element(_) for _ in child_collection.elements ]
object_detials[ "populated" ] = child_collection.populated
dictified[ "object" ] = object_detials
return dictified
__all__ = [ api_payload_to_create_params, dictify_dataset_collection_instance ]
__all__ = ( 'api_payload_to_create_params', 'dictify_dataset_collection_instance' )
lib/galaxy/model/item_attrs.py
+5 -3
View File
@@ -1,7 +1,9 @@
import logging
from sqlalchemy.sql.expression import func
# Cannot import galaxy.model b/c it creates a circular import graph.
import galaxy
import logging
log = logging.getLogger( __name__ )
@@ -162,8 +164,8 @@ class UsesAnnotations:
return getattr( galaxy.model, class_name, None )
__all__ = [
__all__ = (
'UsesAnnotations',
'UsesItemRatings',
'RuntimeException',
]
)
lib/galaxy/model/metadata.py
+12 -11
View File
@@ -4,7 +4,6 @@ Galaxy Metadata
"""
import copy
import cPickle
import json
import logging
import os
@@ -15,6 +14,7 @@ import weakref
from os.path import abspath
from six import string_types
from six.moves import cPickle
from sqlalchemy.orm import object_session
import galaxy.model
@@ -93,13 +93,14 @@ class MetadataCollection( object ):
return default
def items(self):
return iter( [ ( k, self.get( k ) ) for k in self.spec.iterkeys() ] )
return iter( [ ( k, self.get( k ) ) for k in self.spec.keys() ] )
def __str__(self):
return dict( self.items() ).__str__()
def __nonzero__( self ):
def __bool__( self ):
return bool( self.parent._metadata )
__nonzero__ = __bool__
def __getattr__( self, name ):
if name in self.spec:
@@ -205,7 +206,7 @@ class MetadataSpecCollection( odict ):
self[item.name] = item
def iter( self ):
return self.itervalues()
return iter(self.values())
def __getattr__( self, name ):
return self.get( name )
@@ -452,7 +453,7 @@ class RangeParameter( SelectParameter ):
other_values = other_values or {}
if values is None:
values = zip( range( self.min, self.max, self.step ), range( self.min, self.max, self.step ))
values = list(zip( range( self.min, self.max, self.step ), range( self.min, self.max, self.step ) ))
return SelectParameter.get_html_field( self, value=value, context=context, other_values=other_values, values=values, **kwd )
def get_html( self, value, context=None, other_values=None, values=None, **kwd ):
@@ -460,7 +461,7 @@ class RangeParameter( SelectParameter ):
other_values = other_values or {}
if values is None:
values = zip( range( self.min, self.max, self.step ), range( self.min, self.max, self.step ))
values = list(zip( range( self.min, self.max, self.step ), range( self.min, self.max, self.step ) ))
return SelectParameter.get_html( self, value, context=context, other_values=other_values, values=values, **kwd )
@classmethod
@@ -478,7 +479,7 @@ class ColumnParameter( RangeParameter ):
if values is None and context:
column_range = range( 1, ( context.columns or 0 ) + 1, 1 )
values = zip( column_range, column_range )
values = list(zip( column_range, column_range ))
return RangeParameter.get_html_field( self, value=value, context=context, other_values=other_values, values=values, **kwd )
def get_html( self, value, context=None, other_values=None, values=None, **kwd ):
@@ -487,7 +488,7 @@ class ColumnParameter( RangeParameter ):
if values is None and context:
column_range = range( 1, ( context.columns or 0 ) + 1, 1 )
values = zip( column_range, column_range )
values = list(zip( column_range, column_range ))
return RangeParameter.get_html( self, value, context=context, other_values=other_values, values=values, **kwd )
@@ -815,7 +816,7 @@ class JobExternalOutputMetadataWrapper( object ):
metadata_files.filename_override_metadata = abspath( tempfile.NamedTemporaryFile( dir=tmp_dir, prefix="metadata_override_%s_" % key ).name )
open( metadata_files.filename_override_metadata, 'wb+' ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible)
override_metadata = []
for meta_key, spec_value in dataset.metadata.spec.iteritems():
for meta_key, spec_value in dataset.metadata.spec.items():
if isinstance( spec_value.param, FileParameter ) and dataset.metadata.get( meta_key, None ) is not None:
metadata_temp = MetadataTempFile()
shutil.copy( dataset.metadata.get( meta_key, None ).file_name, metadata_temp.file_name )
@@ -871,7 +872,7 @@ class JobExternalOutputMetadataWrapper( object ):
sa_session.add( metadata_files )
sa_session.flush()
__all__ = [
__all__ = (
"Statement",
"MetadataElement",
"MetadataCollection",
@@ -889,4 +890,4 @@ __all__ = [
"FileParameter",
"MetadataTempFile",
"JobExternalOutputMetadataWrapper",
]
)
lib/galaxy/model/orm/now.py
+1 -1
View File
@@ -10,4 +10,4 @@ from datetime import datetime
# Return the current time in UTC without any timezone information
now = datetime.utcnow
__all__ = [now]
__all__ = ('now', )
lib/galaxy/tools/__init__.py
+3 -2
View File
@@ -1538,9 +1538,10 @@ class Tool( object, Dictifiable ):
# Add input file tuples to the list.
for input in test.inputs:
for input_value in test.inputs[ input ]:
input_path = os.path.abspath( os.path.join( 'test-data', input_value ) )
input_filename = str( input_value )
input_path = os.path.abspath( os.path.join( 'test-data', input_filename ) )
if os.path.exists( input_path ):
td_tup = ( input_path, os.path.join( 'test-data', input_value ) )
td_tup = ( input_path, os.path.join( 'test-data', input_filename ) )
tarball_files.append( td_tup )
# And add output file tuples to the list.
for label, filename, _ in test.outputs:
lib/galaxy/tools/cwl/__init__.py
+3 -4
View File
@@ -2,13 +2,12 @@ from .cwltool_deps import (
needs_shell_quoting,
shellescape,
)
from .parser import tool_proxy
from .parser import workflow_proxy
from .parser import tool_proxy, workflow_proxy
from .representation import to_cwl_job, to_galaxy_parameters
from .runtime_actions import handle_outputs
__all__ = [
__all__ = (
'tool_proxy',
'workflow_proxy',
'handle_outputs',
@@ -16,4 +15,4 @@ __all__ = [
'to_galaxy_parameters',
'needs_shell_quoting',
'shellescape',
]
)
lib/galaxy/tools/cwl/cwltool_deps.py
+3 -4
View File
@@ -4,6 +4,7 @@ Use this as the import interface for cwltool and just call
:func:`ensure_cwltool_available` before using any of the imported
functionality at runtime.
"""
import re
try:
import requests
@@ -40,8 +41,6 @@ except (ImportError, SyntaxError):
# Drop SyntaxError once schema_salad supports Python 3
schema_salad = None
import re
needs_shell_quoting = re.compile(r"""(^$|[\s|&;()<>\'"$@])""").search
@@ -65,7 +64,7 @@ def ensure_cwltool_available():
raise ImportError(message)
__all__ = [
__all__ = (
'main',
'load_tool',
'workflow',
@@ -74,4 +73,4 @@ __all__ = [
'schema_salad',
'shellescape',
'needs_shell_quoting',
]
)
lib/galaxy/tools/cwl/parser.py
+9 -7
View File
@@ -10,6 +10,8 @@ import logging
import os
from abc import ABCMeta, abstractmethod
import six
from galaxy.util import safe_makedirs
from galaxy.util.bunch import Bunch
from galaxy.util.odict import odict
@@ -119,8 +121,8 @@ def check_requirements(rec, tool=True):
check_requirements(d, tool=tool)
@six.add_metaclass(ABCMeta)
class ToolProxy( object ):
__metaclass__ = ABCMeta
def __init__(self, tool, tool_path):
self._tool = tool
@@ -176,7 +178,7 @@ class CommandLineToolProxy(ToolProxy):
schema = self._tool.schemaDefs[schema_type]
if schema["type"] == "record":
return map(_simple_field_to_input, schema["fields"])
return [_simple_field_to_input(_) for _ in schema["fields"]]
def output_instances(self):
outputs_schema = self._tool.outputs_record_schema
@@ -235,12 +237,12 @@ class JobProxy(object):
def _ensure_cwl_job_initialized(self):
if self._cwl_job is None:
self._cwl_job = self._tool_proxy._tool.job(
self._cwl_job = next(self._tool_proxy._tool.job(
self._input_dict,
self._output_callback,
basedir=self._job_directory,
use_container=False
).next()
))
self._is_command_line_job = hasattr(self._cwl_job, "command_line")
@property
@@ -536,7 +538,7 @@ class ConditionalInstance(object):
when=odict(),
)
for value, block in self.whens:
as_dict["when"][value] = map(lambda i: i.to_dict(), block)
as_dict["when"][value] = [i.to_dict() for i in block]
return as_dict
@@ -616,7 +618,7 @@ class OutputInstance(object):
self.path = path
__all__ = [
__all__ = (
'tool_proxy',
'load_job_proxy',
]
)
lib/galaxy/tools/cwl/runtime_actions.py
+3 -3
View File
@@ -20,7 +20,7 @@ def handle_outputs(job_directory=None):
job_proxy = load_job_proxy(job_directory)
tool_working_directory = os.path.join(job_directory, "working")
outputs = job_proxy.collect_outputs(tool_working_directory)
for output_name, output in outputs.iteritems():
for output_name, output in outputs.items():
target_path = job_proxy.output_path( output_name )
if isinstance(output, dict) and "path" in output:
output_path = output["path"]
@@ -48,6 +48,6 @@ def handle_outputs(job_directory=None):
with open(target_path, "w") as f:
f.write(json.dumps(output))
__all__ = [
__all__ = (
'handle_outputs',
]
)
lib/galaxy/tools/deps/__init__.py
+5 -3
View File
@@ -5,10 +5,12 @@ Dependency management for tools.
import logging
import os.path
from collections import OrderedDict
from galaxy.util import plugin_config
from .resolvers import NullDependency
from .resolvers.conda import CondaDependencyResolver
from .resolvers.conda import CondaDependencyResolver, DEFAULT_ENSURE_CHANNELS
from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver
from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver
@@ -21,7 +23,7 @@ EXTRA_CONFIG_KWDS = {
'conda_prefix': None,
'conda_exec': None,
'conda_debug': None,
'conda_ensure_channels': 'r,bioconda,iuc',
'conda_ensure_channels': DEFAULT_ENSURE_CHANNELS,
'conda_auto_install': False,
'conda_auto_init': False,
'conda_copy_dependencies': False,
@@ -97,7 +99,7 @@ class DependencyManager( object ):
Takes a list of requirements and returns a dictionary
with requirements as key and dependencies as value.
"""
requirement_to_dependency = dict()
requirement_to_dependency = OrderedDict()
for requirement in requirements:
if requirement.type in [ 'package', 'set_environment' ]:
dependency = self.find_dep( name=requirement.name,
lib/galaxy/tools/deps/commands.py
+6 -10
View File
@@ -3,15 +3,12 @@ import os
import subprocess
import sys as _sys
from six.moves import shlex_quote
from galaxy.util import which
STDOUT_INDICATOR = "-"
try:
from shlex import quote as shell_quote
except ImportError:
from pipes import quote as shell_quote
def redirecting_io(sys=_sys):
"""Predicate to determine if we are redicting I/O in process."""
@@ -83,11 +80,11 @@ def argv_to_str(command_argv, quote=True):
If None appears in the command list it is simply excluded.
Arguments are quoted with shlex.quote. That said, this method is not meant to be
Arguments are quoted with shlex_quote. That said, this method is not meant to be
used in security critical paths of code and should not be used to sanitize
code.
"""
map_func = shell_quote if quote else lambda x: x
map_func = shlex_quote if quote else lambda x: x
return " ".join([map_func(c) for c in command_argv if c is not None])
@@ -144,7 +141,7 @@ class CommandLineException(Exception):
return self.message
__all__ = [
__all__ = (
'argv_to_str',
'CommandLineException',
'download_command',
@@ -153,6 +150,5 @@ __all__ = [
'redirecting_io',
'shell',
'shell_process',
'shell_quote',
'which',
]
)
lib/galaxy/tools/deps/conda_compat.py
+112
View File
@@ -0,0 +1,112 @@
"""Compat. layer with conda_build/verify if Galaxy/galaxy-lib not installed through conda.
In general there are utilities available for Conda building and parsing that are high-quality
and should be utilized when available but that are only available in conda channels and not in
PyPI. This module serves as a PyPI capable interface to these utilities.
"""
import collections
import os
import yaml
try:
from conda_build.metadata import MetaData
except ImportError:
MetaData = None
try:
from anaconda_verify.recipe import parse, render_jinja2
except ImportError:
render_jinja2 = None
parse = None
class _Memoized(object):
def __init__(self, func):
self.func = func
self.cache = {}
def __call__(self, *args):
if not isinstance(args, collections.Hashable):
# uncacheable. a list, for instance.
# better to not cache than blow up.
return self.func(*args)
if args in self.cache:
return self.cache[args]
else:
value = self.func(*args)
self.cache[args] = value
return value
def _parse(data, cfg):
"""Parse metadata YAML."""
assert cfg is None, "Conda utilities for evaluating cfg are not available."
return dict(yamlize(data))
def _render_jinja2(recipe_dir):
"""Evaluate Conda recipe as a jinja template."""
try:
import jinja2
except ImportError:
raise Exception("Failed to import jinja2 for evaluating Conda recipe templates.")
loaders = [jinja2.FileSystemLoader(recipe_dir)]
env = jinja2.Environment(loader=jinja2.ChoiceLoader(loaders))
template = env.get_or_select_template('meta.yaml')
return template.render(environment=env)
@_Memoized
def yamlize(data):
res = yaml.load(data)
# ensure the result is a dict
if res is None:
res = {}
return res
if render_jinja2 is None:
render_jinja2 = _render_jinja2
if parse is None:
parse = _parse
def raw_metadata(recipe_dir):
"""Evaluate Conda template if needed and return raw metadata for supplied recipe directory."""
meta_path = os.path.join(recipe_dir, 'meta.yaml')
with open(meta_path, 'rb') as fi:
data = fi.read()
if b'{{' in data:
data = render_jinja2(recipe_dir)
meta = parse(data, None)
return meta
class _MetaData(object):
def __init__(self, input_dir):
self.meta = raw_metadata(input_dir)
def get_value(self, field, default=None):
"""Get nested field value or supplied default is not present."""
section, key = field.split('/')
submeta = self.meta.get(section)
if submeta is None:
submeta = {}
res = submeta.get(key)
if res is None:
res = default
return res
if MetaData is None:
MetaData = _MetaData
__all__ = (
"MetaData",
"raw_metadata",
)
lib/galaxy/tools/deps/conda_util.py
+75 -17
View File
@@ -7,12 +7,14 @@ import re
import shutil
import tempfile
from distutils.version import LooseVersion
from sys import platform as _platform
import six
import yaml
from ..deps import commands
from ..deps import installable
log = logging.getLogger(__name__)
@@ -47,7 +49,8 @@ def find_conda_prefix(conda_prefix=None):
return conda_prefix
class CondaContext(object):
class CondaContext(installable.InstallableContext):
installable_description = "Conda"
def __init__(self, conda_prefix=None, conda_exec=None,
shell_exec=None, debug=False, ensure_channels='',
@@ -230,6 +233,16 @@ class CondaContext(object):
def activate(self):
return self._bin("activate")
def is_installed(self):
return self.is_conda_installed()
def can_install(self):
return self.can_install_conda()
@property
def parent_path(self):
return os.path.dirname(os.path.abspath(self.conda_prefix))
def _bin(self, name):
return os.path.join(self.conda_prefix, "bin", name)
@@ -273,6 +286,8 @@ class CondaTarget(object):
return "CondaTarget[%s]" % attributes
__repr__ = __str__
@property
def package_specifier(self):
""" Return a package specifier as consumed by conda install/create.
@@ -293,6 +308,17 @@ class CondaTarget(object):
else:
return "__%s@_uv_" % (self.package)
def __hash__(self):
return hash((self.package, self.version, self.channel))
def __eq__(self, other):
if isinstance(other, self.__class__):
return (self.package, self.version, self.channel) == (other.package, other.version, other.channel)
return False
def __ne__(self, other):
return not(self == other)
def hash_conda_packages(conda_packages, conda_target=None):
""" Produce a unique hash on supplied packages.
@@ -339,23 +365,55 @@ def cleanup_failed_install(conda_target, conda_context=None):
conda_context.exec_remove([conda_target.install_environment])
def is_target_available(conda_target, conda_context=None):
""" Checks if a specified target is available for installation.
If the package name exists return "True". If in addition the version matches exactly return "exact".
Otherwise return False.
def best_search_result(conda_target, conda_context=None, channels_override=None):
"""Find best "conda search" result for specified target.
Return ``None`` if no results match.
"""
conda_context = _ensure_conda_context(conda_context)
conda_context.ensure_channels_configured()
search_cmd = [conda_context.conda_exec, "search", "--full-name", "--json", conda_target.package]
if not channels_override:
conda_context.ensure_channels_configured()
search_cmd = [conda_context.conda_exec, "search", "--full-name", "--json"]
if channels_override:
search_cmd.append("--override-channels")
for channel in channels_override:
search_cmd.extend(["--channel", channel])
search_cmd.append(conda_target.package)
res = commands.execute(search_cmd)
hits = json.loads(res).get(conda_target.package, [])
hits = sorted(hits, key=lambda hit: LooseVersion(hit['version']), reverse=True)
if len(hits) > 0:
if conda_target.version:
for hit in hits:
if hit['version'] == conda_target.version:
return 'exact'
return True
if len(hits) == 0:
return (None, None)
best_result = (hits[0], False)
for hit in hits:
if is_search_hit_exact(conda_target, hit):
best_result = (hit, True)
break
return best_result
def is_search_hit_exact(conda_target, search_hit):
target_version = conda_target.version
# It'd be nice to make request verson of 1.0 match available
# version of 1.0.3 or something like that.
return not target_version or search_hit['version'] == target_version
def is_target_available(conda_target, conda_context=None, channels_override=None):
"""Check if a specified target is available for installation.
If the package name exists return ``True`` (the ``bool``). If in addition
the version matches exactly return "exact" (a string). Otherwise return
``False``.
"""
(best_hit, exact) = best_search_result(conda_target, conda_context, channels_override)
if best_hit:
return 'exact' if exact else True
else:
return False
@@ -387,7 +445,7 @@ def filter_installed_targets(conda_targets, conda_context=None, verbose_install_
installed = functools.partial(is_conda_target_installed,
conda_context=conda_context,
verbose_install_check=verbose_install_check)
return filter(installed, conda_targets)
return list(filter(installed, conda_targets))
def build_isolated_environment(
@@ -452,7 +510,7 @@ def requirement_to_conda_targets(requirement, conda_context=None):
def requirements_to_conda_targets(requirements, conda_context=None):
r_to_ct = functools.partial(requirement_to_conda_targets,
conda_context=conda_context)
conda_targets = map(r_to_ct, requirements)
conda_targets = (r_to_ct(_) for _ in requirements)
return [c for c in conda_targets if c is not None]
@@ -462,10 +520,10 @@ def _ensure_conda_context(conda_context):
return conda_context
__all__ = [
__all__ = (
'CondaContext',
'CondaTarget',
'install_conda',
'install_conda_target',
'requirements_to_conda_targets',
]
)
lib/galaxy/tools/deps/container_resolvers/__init__.py
+50
View File
@@ -0,0 +1,50 @@
"""The module defines the abstract interface for resolving container images for tool execution."""
from abc import (
ABCMeta,
abstractmethod,
abstractproperty,
)
from galaxy.util.dictifiable import Dictifiable
class ContainerResolver(Dictifiable, object):
"""Description of a technique for resolving container images for tool execution."""
# Keys for dictification.
dict_collection_visible_keys = ['resolver_type']
__metaclass__ = ABCMeta
def __init__(self, app_info=None, **kwds):
"""Default initializer for ``ContainerResolver`` subclasses."""
self.app_info = app_info
self.resolver_kwds = kwds
def _get_config_option(self, key, default=None, config_prefix=None, **kwds):
"""Look in resolver-specific settings for option and then fallback to
global settings.
"""
global_key = "%s_%s" % (config_prefix, key)
if key in kwds:
return kwds.get(key)
elif self.app_info and hasattr(self.app_info, global_key):
return getattr(self.app_info, global_key)
else:
return default
@abstractmethod
def resolve(self, tool_info):
"""Find a container matching all supplied requirements for tool.
The supplied argument is a :class:`galaxy.tools.deps.containers.ToolInfo` description
of the tool and its requirements.
"""
@abstractproperty
def resolver_type(self):
"""Short label for the type of container resolution."""
def _container_type_enabled(self, container_description, enabled_container_types):
"""Return a boolean indicating if the specified container type is enabled."""
return container_description.type in enabled_container_types
lib/galaxy/tools/deps/container_resolvers/explicit.py
+26
View File
@@ -0,0 +1,26 @@
"""This module describes the :class:`ExplicitContainerResolver` ContainerResolver plugin."""
import logging
from ..container_resolvers import (
ContainerResolver,
)
log = logging.getLogger(__name__)
class ExplicitContainerResolver(ContainerResolver):
"""Find explicit containers referenced in the tool description (e.g. tool XML file) if present."""
resolver_type = "explicit"
def resolve(self, enabled_container_types, tool_info):
"""Find a container explicitly mentioned in tool description.
This ignores the tool requirements and assumes the tool author crafted
a correct container.
"""
for container_description in tool_info.container_descriptions:
if self._container_type_enabled(container_description):
return True
return False
lib/galaxy/tools/deps/container_resolvers/mulled.py
+207
View File
@@ -0,0 +1,207 @@
"""This module describes the :class:`MulledContainerResolver` ContainerResolver plugin."""
import collections
import logging
from ..container_resolvers import (
ContainerResolver,
)
from ..docker_util import build_docker_images_command
from ..mulled.mulled_build import (
check_output,
DEFAULT_CHANNELS,
ensure_installed,
InvolucroContext,
mull_targets,
)
from ..mulled.mulled_build_tool import requirements_to_mulled_targets
from ..mulled.util import (
image_name,
mulled_tags_for,
split_tag,
)
from ..requirements import ContainerDescription
log = logging.getLogger(__name__)
CachedMulledImageSingleTarget = collections.namedtuple("CachedMulledImageSingleTarget", ["package_name", "version", "build", "image_identifier"])
CachedMulledImageMultiTarget = collections.namedtuple("CachedMulledImageMultiTarget", ["hash", "image_identifier"])
CachedMulledImageSingleTarget.multi_target = False
CachedMulledImageMultiTarget.multi_target = True
def list_cached_mulled_images(namespace=None):
command = build_docker_images_command(truncate=True, sudo_docker=False)
command = "%s | tail -n +2 | tr -s ' ' | cut -d' ' -f1,2" % command
images_and_versions = check_output(command)
name_filter = get_filter(namespace)
def output_line_to_image(line):
image_name, version = line.split(" ", 1)
identifier = "%s:%s" % (image_name, version)
url, namespace, package_description = image_name.split("/")
if package_description.startswith("mulled-v1-"):
hash = package_description
image = CachedMulledImageMultiTarget(hash, identifier)
else:
build = None
if not version or version == "latest":
version = None
if version and "--" in version:
version, build = split_tag(version)
image = CachedMulledImageSingleTarget(image_name, version, build, identifier)
return image
return [output_line_to_image(_) for _ in filter(name_filter, images_and_versions.splitlines())]
def get_filter(namespace):
prefix = "quay.io/" if namespace is None else "quay.io/%s" % namespace
return lambda name: name.startswith(prefix) and name.count("/") == 2
def cached_container_description(targets, namespace):
if len(targets) == 0:
return None
cached_images = list_cached_mulled_images(namespace)
image = None
if len(targets) == 1:
target = targets[0]
for cached_image in cached_images:
if cached_image.multi_target:
continue
if not cached_image.package_name == target.package_name:
continue
if not target.version or target.version == cached_image.version:
image = cached_image
break
else:
name = image_name(targets)
for cached_image in cached_images:
if not cached_image.multi_target:
continue
if name == cached_image.hash:
image = cached_image
break
container = None
if image:
container = ContainerDescription(
image.image_identifier,
type="docker",
)
return container
class CachedMulledContainerResolver(ContainerResolver):
resolver_type = "cached_mulled"
def __init__(self, app_info=None, namespace=None):
super(CachedMulledContainerResolver, self).__init__(app_info)
self.namespace = namespace
def resolve(self, enabled_container_types, tool_info):
targets = mulled_targets(tool_info)
return cached_container_description(targets, self.namespace)
class MulledContainerResolver(ContainerResolver):
"""Look for mulled images matching tool dependencies."""
resolver_type = "mulled"
def __init__(self, app_info=None, namespace="mulled"):
super(MulledContainerResolver, self).__init__(app_info)
self.namespace = namespace
def resolve(self, enabled_container_types, tool_info):
targets = mulled_targets(tool_info)
if len(targets) == 0:
return None
name = None
if len(targets) == 1:
target = targets[0]
target_version = target.version
tags = mulled_tags_for(self.namespace, target.package_name)
if not tags:
return None
if target_version:
for tag in tags:
version, build = split_tag(tag)
if version == target_version:
name = "%s:%s--%s" % (target.package_name, version, build)
break
else:
version, build = split_tag(tags[0])
name = "%s:%s--%s" % (target.package_name, version, build)
else:
base_image_name = image_name(targets)
tags = mulled_tags_for(self.namespace, base_image_name)
if tags:
name = "%s:%s" % (base_image_name, tags[0])
if name:
return ContainerDescription(
"quay.io/%s/%s" % (self.namespace, name),
type="docker",
)
class BuildMulledContainerResolver(ContainerResolver):
"""Look for mulled images matching tool dependencies."""
resolver_type = "build_mulled"
def __init__(self, app_info=None, namespace="local", **kwds):
super(BuildMulledContainerResolver, self).__init__(app_info)
self._involucro_context_kwds = {
'involucro_bin': self._get_config_option("involucro_path", None)
}
self.namespace = namespace
self._mulled_kwds = {
'namespace': namespace,
'channels': self._get_config_option("channels", DEFAULT_CHANNELS, prefix="mulled"),
}
self.auto_init = self._get_config_option("auto_init", DEFAULT_CHANNELS, prefix="involucro")
def resolve(self, enabled_container_types, tool_info):
targets = mulled_targets(tool_info)
if len(targets) == 0:
return None
mull_targets(
targets,
involucro_context=self._get_involucro_context(),
**self._mulled_kwds
)
return cached_container_description(targets, self.namespace)
def _get_involucro_context(self):
involucro_context = InvolucroContext(**self._involucro_context_kwds)
self.enabled = ensure_installed(involucro_context, self.auto_init)
return involucro_context
def mulled_targets(tool_info):
return requirements_to_mulled_targets(tool_info.requirements)
__all__ = (
"CachedMulledContainerResolver",
"MulledContainerResolver",
"BuildMulledContainerResolver",
)
lib/galaxy/tools/deps/containers.py
+80 -26
View File
@@ -10,14 +10,23 @@ from abc import (
import six
from galaxy.util import asbool
from galaxy.util import plugin_config
from .container_resolvers.explicit import ExplicitContainerResolver
from .container_resolvers.mulled import (
BuildMulledContainerResolver,
CachedMulledContainerResolver,
MulledContainerResolver,
)
from .requirements import ContainerDescription
from .requirements import DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES, DEFAULT_CONTAINER_SHELL
from ..deps import docker_util
log = logging.getLogger(__name__)
DEFAULT_CONTAINER_TYPE = "docker"
DOCKER_CONTAINER_TYPE = "docker"
DEFAULT_CONTAINER_TYPE = DOCKER_CONTAINER_TYPE
ALL_CONTAINER_TYPES = [DOCKER_CONTAINER_TYPE]
LOAD_CACHED_IMAGE_COMMAND_TEMPLATE = '''
python << EOF
@@ -47,9 +56,18 @@ class ContainerFinder(object):
def __init__(self, app_info):
self.app_info = app_info
self.container_registry = ContainerRegistry()
self.container_registry = ContainerRegistry(app_info)
def __enabled_container_types(self, destination_info):
return [t for t in ALL_CONTAINER_TYPES if self.__container_type_enabled(t, destination_info)]
def find_container(self, tool_info, destination_info, job_info):
enabled_container_types = self.__enabled_container_types(destination_info)
# Short-cut everything else and just skip checks if no container type is enabled.
if not enabled_container_types:
return NULL_CONTAINER
def __destination_container(container_description=None, container_id=None, container_type=None):
if container_description:
container_id = container_description.identifier
@@ -82,23 +100,10 @@ class ContainerFinder(object):
return container
# Otherwise lets see if we can find container for the tool.
# Exact matches first from explicitly listed containers in tools...
for container_description in tool_info.container_descriptions:
container = __destination_container(container_description)
if container:
return container
# Implement vague concept of looping through all containers
# matching requirements. Exact details need to be worked through
# but hopefully the idea that it sits below find_container somewhere
# external components to this module don't need to worry about it
# is good enough.
container_descriptions = self.container_registry.container_descriptions_for_requirements(tool_info.requirements)
for container_description in container_descriptions:
container = __destination_container(container_description)
if container:
return container
container_description = self.container_registry.find_best_container_description(enabled_container_types, tool_info)
container = __destination_container(container_description)
if container:
return container
# If we still don't have a container, check to see if any container
# types define a default container id and use that.
@@ -175,14 +180,55 @@ class NullContainerFinder(object):
return []
class ContainerRegistry():
class ContainerRegistry(object):
"""Loop through enabled ContainerResolver plugins and find first match."""
def __init__(self):
pass
def __init__(self, app_info):
self.resolver_classes = self.__resolvers_dict()
self.enable_beta_mulled_containers = app_info.enable_beta_mulled_containers
self.app_info = app_info
self.container_resolvers = self.__build_container_resolvers(app_info)
def container_descriptions_for_requirements(self, requirements):
# Return lists of containers that would match requirements...
return []
def __build_container_resolvers( self, app_info ):
conf_file = getattr(app_info, 'containers_resolvers_config_file', None)
if not conf_file:
return self.__default_containers_resolvers()
if not os.path.exists( conf_file ):
log.debug( "Unable to find config file '%s'", conf_file)
return self.__default_containers_resolvers()
plugin_source = plugin_config.plugin_source_from_path( conf_file )
return self.__parse_resolver_conf_xml( plugin_source )
def __parse_resolver_conf_xml(self, plugin_source):
extra_kwds = {}
return plugin_config.load_plugins(self.resolver_classes, plugin_source, extra_kwds)
def __default_containers_resolvers(self):
default_resolvers = [
ExplicitContainerResolver(self.app_info),
]
if self.enable_beta_mulled_containers:
default_resolvers.extend([
CachedMulledContainerResolver(self.app_info),
MulledContainerResolver(self.app_info, namespace="mulled"),
BuildMulledContainerResolver(self.app_info),
])
return default_resolvers
def __resolvers_dict( self ):
import galaxy.tools.deps.container_resolvers
return plugin_config.plugins_dict( galaxy.tools.deps.container_resolvers, 'resolver_type' )
def find_best_container_description(self, enabled_container_types, tool_info):
"""Yield best container description of supplied types matching tool info."""
for container_resolver in self.container_resolvers:
container_description = container_resolver.resolve(enabled_container_types, tool_info)
log.info("Checking with container resolver [%s] found description [%s]" % (container_resolver, container_description))
if container_description:
assert container_description.type in enabled_container_types
return container_description
return None
class AppInfo(object):
@@ -193,7 +239,11 @@ class AppInfo(object):
default_file_path=None,
outputs_to_working_directory=False,
container_image_cache_path=None,
library_import_dir=None
library_import_dir=None,
enable_beta_mulled_containers=False,
containers_resolvers_config_file=None,
involucro_path=None,
involucro_auto_init=True,
):
self.galaxy_root_dir = galaxy_root_dir
self.default_file_path = default_file_path
@@ -201,6 +251,10 @@ class AppInfo(object):
self.outputs_to_working_directory = outputs_to_working_directory
self.container_image_cache_path = container_image_cache_path
self.library_import_dir = library_import_dir
self.enable_beta_mulled_containers = enable_beta_mulled_containers
self.containers_resolvers_config_file = containers_resolvers_config_file
self.involucro_path = involucro_path
self.involucro_auto_init = involucro_auto_init
class ToolInfo(object):
lib/galaxy/tools/deps/docker_util.py
+13 -10
View File
@@ -3,7 +3,10 @@
...using common defaults and configuration mechanisms.
"""
import os
from .commands import argv_to_str, shell_quote
from six.moves import shlex_quote
from .commands import argv_to_str
DEFAULT_DOCKER_COMMAND = "docker"
DEFAULT_SUDO = True
@@ -33,7 +36,7 @@ class DockerVolume(object):
if not volumes_as_str:
return []
volume_strs = [v.strip() for v in volumes_as_str.split(",")]
return map(DockerVolume.volume_from_str, volume_strs)
return [DockerVolume.volume_from_str(_) for _ in volume_strs]
@staticmethod
def volume_from_str(as_str):
@@ -152,19 +155,19 @@ def build_docker_run_command(
if terminal:
command_parts.append("-t")
for env_directive in env_directives:
command_parts.extend(["-e", shell_quote(env_directive)])
command_parts.extend(["-e", shlex_quote(env_directive)])
for volume in volumes:
command_parts.extend(["-v", shell_quote(str(volume))])
command_parts.extend(["-v", shlex_quote(str(volume))])
if volumes_from:
command_parts.extend(["--volumes-from", shell_quote(str(volumes_from))])
command_parts.extend(["--volumes-from", shlex_quote(str(volumes_from))])
if memory:
command_parts.extend(["-m", shell_quote(memory)])
command_parts.extend(["-m", shlex_quote(memory)])
if name:
command_parts.extend(["--name", shell_quote(name)])
command_parts.extend(["--name", shlex_quote(name)])
if working_directory:
command_parts.extend(["-w", shell_quote(working_directory)])
command_parts.extend(["-w", shlex_quote(working_directory)])
if net:
command_parts.extend(["--net", shell_quote(net)])
command_parts.extend(["--net", shlex_quote(net)])
if auto_rm:
command_parts.append("--rm")
if run_extra_arguments:
@@ -177,7 +180,7 @@ def build_docker_run_command(
full_image = image
if tag:
full_image = "%s:%s" % (full_image, tag)
command_parts.append(shell_quote(full_image))
command_parts.append(shlex_quote(full_image))
command_parts.append(container_command)
return " ".join(command_parts)
lib/galaxy/tools/deps/installable.py
+77
View File
@@ -0,0 +1,77 @@
"""Abstractions for installing local software managed and required by Galaxy/galaxy-lib."""
import logging
import os
from abc import (
ABCMeta,
abstractmethod,
abstractproperty,
)
from galaxy.util.filelock import (
FileLock,
FileLockException
)
log = logging.getLogger(__name__)
class InstallableContext(object):
"""Represent a directory/configuration of something that can be installed."""
__metaclass__ = ABCMeta
@abstractmethod
def is_installed(self):
"""Return bool indicating if the configured software is installed."""
@abstractmethod
def can_install(self):
"""Check preconditions for installation."""
@abstractproperty
def installable_description(self):
"""Short description of thing being installed for log statements."""
@abstractproperty
def parent_path(self):
"""Return parent path of the location the installable will be created within."""
def ensure_installed(installable_context, install_func, auto_init):
"""Make sure target is installed - handle multiple processes potentially attempting installation."""
parent_path = installable_context.parent_path
desc = installable_context.installable_description
def _check():
if not installable_context.is_installed():
if auto_init:
if installable_context.can_install():
if install_func(installable_context):
installed = False
log.warning("%s installation requested and failed." % desc)
else:
installed = installable_context.is_installed()
if not installed:
log.warning("%s installation requested, seemed to succeed, but not found." % desc)
else:
installed = False
else:
installed = False
log.warning("%s not installed and auto-installation disabled.", desc)
else:
installed = True
return installed
if not os.path.exists(parent_path):
os.mkdir(parent_path)
try:
if auto_init and os.access(parent_path, os.W_OK):
with FileLock(os.path.join(parent_path, desc.lower())):
return _check()
else:
return _check()
except FileLockException:
return ensure_installed(installable_context, auto_init)
lib/galaxy/tools/deps/mulled/__init__.py
View File
lib/galaxy/tools/deps/mulled/_cli.py
+19
View File
@@ -0,0 +1,19 @@
"""CLI helpers for mulled command-line tools."""
import argparse
def arg_parser(argv, globals):
"""Build an argparser for this CLI tool."""
doc = globals["__doc__"]
description, epilog = doc.split("\n", 1)
parser = argparse.ArgumentParser(
description=description,
epilog=epilog,
formatter_class=argparse.RawTextHelpFormatter,
)
return parser
__all__ = (
"arg_parser",
)
lib/galaxy/tools/deps/mulled/invfile.lua
+67
View File
@@ -0,0 +1,67 @@
-- http://stackoverflow.com/questions/19262761/lua-need-to-split-at-comma/19263313#19263313
function string:split( inSplitPattern, outResults )
if not outResults then
outResults = { }
end
local theStart = 1
local theSplitStart, theSplitEnd = string.find( self, inSplitPattern, theStart )
while theSplitStart do
table.insert( outResults, string.sub( self, theStart, theSplitStart-1 ) )
theStart = theSplitEnd + 1
theSplitStart, theSplitEnd = string.find( self, inSplitPattern, theStart )
end
table.insert( outResults, string.sub( self, theStart ) )
return outResults
end
local repo = VAR.REPO
local channel_args = ''
local channels = VAR.CHANNELS:split(",")
for i = 1, #channels do
channel_args = channel_args .. " -c " .. channels[i]
end
local target_args = ''
local targets = VAR.TARGETS:split(",")
for i = 1, #targets do
target_args = target_args .. " " .. targets[i]
end
local bind_args = {}
local binds_table = VAR.BINDS:split(",")
for i = 1, #binds_table do
table.insert(bind_args, binds_table[i])
end
inv.task('build')
.using('continuumio/miniconda:latest')
.withHostConfig({binds = {"build:/data"}})
.run('rm', '-rf', '/data/dist')
.using('continuumio/miniconda:latest')
.withHostConfig({binds = bind_args})
.run('/bin/sh', '-c', 'conda install '
.. channel_args .. ' '
.. target_args
.. ' -p /usr/local --copy --yes')
.wrap('build/dist')
.at('/usr/local')
.inImage('bgruening/busybox-bash:0.1')
.as(repo)
inv.task('test')
.using(repo)
.withConfig({entrypoint = {'/bin/sh', '-c'}})
.run(VAR.TEST)
inv.task('push')
.push(repo)
inv.task('build-and-test')
.runTask('build')
.runTask('test')
inv.task('all')
.runTask('build')
.runTask('test')
.runTask('push')
lib/galaxy/tools/deps/mulled/mulled_build.py
+286
View File
@@ -0,0 +1,286 @@
#!/usr/bin/env python
"""Build a mulled image for specified conda targets.
Examples
Build a mulled image with:
mulled-build build 'samtools=1.3.1--4,bedtools=2.22'
"""
from __future__ import print_function
import json
import os
import string
import subprocess
from sys import platform as _platform
try:
import yaml
except ImportError:
yaml = None
from galaxy.tools.deps import commands, installable
from ._cli import arg_parser
from .util import build_target, conda_build_target_str, image_name
from ..conda_compat import MetaData
DIRNAME = os.path.dirname(__file__)
DEFAULT_CHANNEL = "bioconda"
DEFAULT_EXTRA_CHANNELS = ["conda-forge", "r"]
DEFAULT_CHANNELS = [DEFAULT_CHANNEL] + DEFAULT_EXTRA_CHANNELS
DEFAULT_REPOSITORY_TEMPLATE = "quay.io/${namespace}/${image}"
DEFAULT_BINDS = ["build/dist:/usr/local/"]
IS_OS_X = _platform == "darwin"
INVOLUCRO_VERSION = "1.1.2"
def involucro_link():
if IS_OS_X:
url = "https://github.com/involucro/involucro/releases/download/v%s/involucro.darwin" % INVOLUCRO_VERSION
else:
url = "https://github.com/involucro/involucro/releases/download/v%s/involucro" % INVOLUCRO_VERSION
return url
def get_tests(args, pkg_path):
"""Extract test cases given a recipe's meta.yaml file."""
recipes_dir = args.recipes_dir
tests = ""
input_dir = os.path.dirname(os.path.join(recipes_dir, pkg_path))
recipe_meta = MetaData(input_dir)
tests_commands = recipe_meta.get_value('test/commands')
tests_imports = recipe_meta.get_value('test/imports')
requirements = recipe_meta.get_value('requirements/run')
if tests_imports or tests_commands:
if tests_commands:
tests = ' && '.join(tests_commands)
elif tests_imports and 'python' in requirements:
tests = ' && '.join('python -c "import %s"' % imp for imp in tests_imports)
elif tests_imports and ('perl' in requirements or 'perl-threaded' in requirements):
tests = ' && '.join('''perl -e "use %s;"''' % imp for imp in tests_imports)
tests = tests.replace('$R ', 'Rscript ')
else:
pass
return tests
def get_pkg_name(args, pkg_path):
"""Extract the package name from a given meta.yaml file."""
recipes_dir = args.recipes_dir
input_dir = os.path.dirname(os.path.join(recipes_dir, pkg_path))
recipe_meta = MetaData(input_dir)
return recipe_meta.get_value('package/name')
def get_affected_packages(args):
"""Return a list of all meta.yaml file that where modified/created recently.
Length of time to check for indicated by the ``hours`` parameter.
"""
recipes_dir = args.recipes_dir
hours = args.diff_hours
cmd = """cd '%s' && git log --diff-filter=ACMRTUXB --name-only --pretty="" --since="%s hours ago" | grep -E '^recipes/.*/meta.yaml' | sort | uniq""" % (recipes_dir, hours)
pkg_list = check_output(cmd, shell=True)
ret = list()
for pkg in pkg_list.strip().split('\n'):
if pkg and os.path.exists(os.path.join( recipes_dir, pkg )):
ret.append( (get_pkg_name(args, pkg), get_tests(args, pkg)) )
return ret
def check_output(cmd, shell=True):
return subprocess.check_output(cmd, shell=shell)
def conda_versions(pkg_name, file_name):
"""Return all conda version strings for a specified package name."""
j = json.load(open(file_name))
ret = list()
for pkg in j['packages'].values():
if pkg['name'] == pkg_name:
ret.append('%s--%s' % (pkg['version'], pkg['build']))
return ret
def mull_targets(
targets, involucro_context=None,
command="build", channels=DEFAULT_CHANNELS, namespace="mulled",
test='true', image_build=None, name_override=None,
repository_template=DEFAULT_REPOSITORY_TEMPLATE, dry_run=False,
binds=DEFAULT_BINDS
):
targets = list(targets)
if involucro_context is None:
involucro_context = InvolucroContext()
repo_template_kwds = {
"namespace": namespace,
"image": image_name(targets, image_build=image_build, name_override=name_override)
}
repo = string.Template(repository_template).safe_substitute(repo_template_kwds)
for channel in channels:
if channel.startswith('file://'):
bind_path = channel.lstrip('file://')
binds.append('/%s:/%s' % (bind_path, bind_path))
channels = ",".join(channels)
target_str = ",".join(map(conda_build_target_str, targets))
bind_str = ",".join(binds)
involucro_args = [
'-f', '%s/invfile.lua' % DIRNAME,
'-set', "CHANNELS='%s'" % channels,
'-set', "TEST='%s'" % test,
'-set', "TARGETS='%s'" % target_str,
'-set', "REPO='%s'" % repo,
'-set', "BINDS='%s'" % bind_str,
command,
]
print(" ".join(involucro_context.build_command(involucro_args)))
if not dry_run:
ensure_installed(involucro_context, True)
involucro_context.exec_command(involucro_args)
def context_from_args(args):
return InvolucroContext(involucro_bin=args.involucro_path)
class InvolucroContext(installable.InstallableContext):
installable_description = "Involucro"
def __init__(self, involucro_bin=None, shell_exec=None, verbose="3"):
if involucro_bin is None:
if os.path.exists("./involucro"):
self.involucro_bin = "./involucro"
else:
self.involucro_bin = "involucro"
else:
self.involucro_bin = involucro_bin
self.shell_exec = shell_exec or commands.shell
self.verbose = verbose
def build_command(self, involucro_args):
return [self.involucro_bin, "-v=%s" % self.verbose] + involucro_args
def exec_command(self, involucro_args):
cmd = self.build_command(involucro_args)
return self.shell_exec(" ".join(cmd))
def is_installed(self):
return os.path.exists(self.involucro_bin)
def can_install(self):
return True
@property
def parent_path(self):
return os.path.dirname(os.path.abspath(self.involucro_bin))
def ensure_installed(involucro_context, auto_init):
return installable.ensure_installed(involucro_context, install_involucro, auto_init)
def install_involucro(involucro_context=None, to_path=None):
to_path = involucro_context.involucro_bin
download_cmd = " ".join(commands.download_command(involucro_link(), to=to_path, quote_url=True))
full_cmd = "%s && chmod +x %s" % (download_cmd, to_path)
return involucro_context.shell_exec(full_cmd)
def add_build_arguments(parser):
"""Base arguments describing how to 'mull'."""
parser.add_argument('--involucro-path', dest="involucro_path", default=None,
help="Path to involucro (if not set will look in working directory and on PATH).")
parser.add_argument('--force-rebuild', dest="force_rebuild", action="store_true",
help="Rebuild package even if already published.")
parser.add_argument('--dry-run', dest='dry_run', action="store_true",
help='Just print commands instead of executing them.')
parser.add_argument('-n', '--namespace', dest='namespace', default="mulled",
help='quay.io namespace.')
parser.add_argument('-r', '--repository_template', dest='repository_template', default=DEFAULT_REPOSITORY_TEMPLATE,
help='Docker repository target for publication (only quay.io or compat. API is currently supported).')
parser.add_argument('-c', '--channel', dest='channel', default=DEFAULT_CHANNEL,
help='Target conda channel')
parser.add_argument('--extra-channels', dest='extra_channels', default=",".join(DEFAULT_EXTRA_CHANNELS),
help='Dependent conda channels.')
def add_single_image_arguments(parser):
parser.add_argument("--name-override", dest="name_override", default=None,
help="Override mulled image name - this is not recommended since metadata will not be detectable from the name of resulting images")
parser.add_argument("--image-build", dest="image_build", default=None,
help="Build a versioned variant of this image.")
def target_str_to_targets(targets_raw):
def parse_target(target_str):
if "=" in target_str:
package_name, version = target_str.split("=", 1)
build = None
if "--" in version:
version, build = version.split('--')
target = build_target(package_name, version, build)
else:
target = build_target(target_str)
return target
targets = [parse_target(_) for _ in targets_raw.split(",")]
return targets
def args_to_mull_targets_kwds(args):
kwds = {}
if hasattr(args, "image_build"):
kwds["image_build"] = args.image_build
if hasattr(args, "name_override"):
kwds["name_override"] = args.name_override
if hasattr(args, "namespace"):
kwds["namespace"] = args.namespace
if hasattr(args, "dry_run"):
kwds["dry_run"] = args.dry_run
if hasattr(args, "test"):
kwds["test"] = args.test
if hasattr(args, "channel"):
channels = [args.channel]
if hasattr(args, "extra_channels"):
channels += args.extra_channels.split(",")
kwds["channels"] = channels
if hasattr(args, "command"):
kwds["command"] = args.command
if hasattr(args, "repository_template"):
kwds["repository_template"] = args.repository_template
kwds["involucro_context"] = context_from_args(args)
return kwds
def main(argv=None):
"""Main entry-point for the CLI tool."""
parser = arg_parser(argv, globals())
add_build_arguments(parser)
add_single_image_arguments(parser)
parser.add_argument('command', metavar='COMMAND', help='Command (build-and-test, build, all)')
parser.add_argument('targets', metavar="TARGETS", default=None, help="Build a single container with specific package(s).")
parser.add_argument('--repository-name', dest="repository_name", default=None, help="Name of mulled container (leave blank to auto-generate based on packages - recommended).")
parser.add_argument('--test', help='Provide a test command for the container.')
args = parser.parse_args()
targets = target_str_to_targets(args.targets)
mull_targets(targets, **args_to_mull_targets_kwds(args))
__all__ = ("main", )
if __name__ == '__main__':
main()
lib/galaxy/tools/deps/mulled/mulled_build_channel.py
+109
View File
@@ -0,0 +1,109 @@
#!/usr/bin/env python
"""Build a mulled images for all recent conda recipe updates that don't have existing images.
Examples:
Build mulled images for recent bioconda changes with:
mulled-build-channel build
Build, test, and publish images with the follow command:
mulled-build-channel all
See recent changes that would be built with:
mulled-build-channel list
"""
from __future__ import print_function
import os
import time
from ._cli import arg_parser
from .mulled_build import (
add_build_arguments,
args_to_mull_targets_kwds,
build_target,
check_output,
conda_versions,
get_affected_packages,
mull_targets,
)
from .util import quay_versions, version_sorted
def _fetch_repo_data(args):
repo_data = args.repo_data
channel = args.channel
if repo_data is None:
repo_data = "%s-repodata.json" % channel
if not os.path.exists(repo_data):
check_output("wget --quiet https://conda.anaconda.org/%s/linux-64/repodata.json.bz2 -O '%s.bz2' && bzip2 -d '%s.bz2'" % (channel, repo_data, repo_data))
return repo_data
def _new_versions(quay, conda):
"""Calculate the versions that are in conda but not on quay.io."""
sconda = set(conda)
squay = set(quay) if quay else set()
return sconda - squay # sconda.symmetric_difference(squay)
def run_channel(args, build_last_n_versions=1):
"""Build list of involucro commands (as shell snippet) to run."""
pkgs = get_affected_packages(args)
for pkg_name, pkg_tests in pkgs:
repo_data = _fetch_repo_data(args)
c = conda_versions(pkg_name, repo_data)
# only package the most recent N versions
c = version_sorted(c)[:build_last_n_versions]
if not args.force_rebuild:
time.sleep(1)
q = quay_versions(args.namespace, pkg_name)
versions = _new_versions(q, c)
else:
versions = c
for tag in versions:
target = build_target(pkg_name, tag=tag)
targets = [target]
mull_targets(targets, test=pkg_tests, **args_to_mull_targets_kwds(args))
def get_pkg_names(args):
"""Print package names that would be affected."""
print('\n'.join(pkg_name for pkg_name, pkg_tests in get_affected_packages(args)))
def add_channel_arguments(parser):
"""Add arguments only used if running mulled over a whole conda channel."""
parser.add_argument('--repo-data', dest='repo_data', default=None,
help='Published repository data (will be fetched from --channel if not available and written). Defaults to [channel_name]-repodata.json.')
parser.add_argument('--diff-hours', dest='diff_hours', default="25",
help='If finding all recently changed recipes, use this number of hours.')
parser.add_argument('--recipes-dir', dest="recipes_dir", default="./bioconda-recipes")
def main(argv=None):
"""Main entry-point for the CLI tool."""
parser = arg_parser(argv, globals())
add_channel_arguments(parser)
add_build_arguments(parser)
parser.add_argument('command', metavar='COMMAND', help='Command (list, build-and-test, build, all)')
parser.add_argument('--targets', dest="targets", default=None, help="Build a single container with specific package(s).")
parser.add_argument('--repository-name', dest="repository_name", default=None, help="Name of a single container (leave blank to auto-generate based on packages).")
args = parser.parse_args()
if args.command == "list":
get_pkg_names(args)
else:
run_channel(args)
__all__ = ("main", )
if __name__ == '__main__':
main()
lib/galaxy/tools/deps/mulled/mulled_build_files.py
+79
View File
@@ -0,0 +1,79 @@
"""Build all composite mulled recipes discovered in TSV files.
Use mulled-build-channel to build images for single recipes for a whole conda
channel. This script instead builds images for combinations of recipes. This
script can be given a single TSV file or a directory of TSV files to process.
Examples:
Build all recipes discovered in tsv files in a single directory.
mulled-build-files build
"""
import collections
import glob
import os
from ._cli import arg_parser
from .mulled_build import (
add_build_arguments,
args_to_mull_targets_kwds,
mull_targets,
target_str_to_targets,
)
def main(argv=None):
"""Main entry-point for the CLI tool."""
parser = arg_parser(argv, globals())
add_build_arguments(parser)
parser.add_argument('command', metavar='COMMAND', help='Command (build-and-test, build, all)')
parser.add_argument('files', metavar="FILES", default=".",
help="Path to directory (or single file) of TSV files describing composite recipes.")
args = parser.parse_args()
for targets in generate_targets(args.files):
mull_targets(targets, **args_to_mull_targets_kwds(args))
def generate_targets(target_source):
"""Generate all targets from TSV files in specified file or directory."""
target_source = os.path.abspath(target_source)
if os.path.isdir(target_source):
target_source_files = glob.glob(target_source + "/*.tsv")
else:
target_source_files = [target_source]
for target_source_file in target_source_files:
with open(target_source_file, "r") as f:
for line in f.readlines():
if line:
line = line.strip()
if not line or line.startswith("#"):
continue
yield line_to_targets(line)
def line_to_targets(line_str):
line = _parse_line(line_str)
return target_str_to_targets(line)
_Line = collections.namedtuple("_Line", ["targets", "image_build", "name_override"])
def _parse_line(line_str):
line_parts = line_str.split(" ")
assert len(line_parts) < 3, "Too many fields in line [%s], expect at most 3 - targets, image build number, and name override." % line_str
line_parts += [None] * (3 - len(line_parts))
return _Line(*line_parts)
__all__ = ("main", )
if __name__ == '__main__':
main()
lib/galaxy/tools/deps/mulled/mulled_build_tool.py
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env python
"""Build a mulled images for a tool source (Galaxy or CWL tool).
Examples:
Build mulled images for requirements defined in a tool:
mulled-build-tool build path/to/tool_file.xml
"""
from galaxy.tools.parser import get_tool_source
from ._cli import arg_parser
from .mulled_build import (
add_build_arguments,
add_single_image_arguments,
args_to_mull_targets_kwds,
build_target,
mull_targets,
)
def main(argv=None):
"""Main entry-point for the CLI tool."""
parser = arg_parser(argv, globals())
add_build_arguments(parser)
add_single_image_arguments(parser)
parser.add_argument('command', metavar='COMMAND', help='Command (build-and-test, build, all)')
parser.add_argument('tool', metavar="TOOL", default=None, help="Path to tool to build mulled image for.")
args = parser.parse_args()
tool_source = get_tool_source(args.tool)
requirements, _ = tool_source.parse_requirements_and_containers()
targets = requirements_to_mulled_targets(requirements)
mull_targets(targets, **args_to_mull_targets_kwds(args))
def requirements_to_mulled_targets(requirements):
"""Convert Galaxy's representation of requirements into mulled Target objects.
Only package requirements are retained.
"""
package_requirements = [r for r in requirements if r.type == "package"]
targets = [build_target(r.name, r.version) for r in package_requirements]
return targets
__all__ = ("main", "requirements_to_mulled_targets")
if __name__ == '__main__':
main()
lib/galaxy/tools/deps/mulled/mulled_search.py
+131
View File
@@ -0,0 +1,131 @@
#!/usr/bin/env python
import argparse
import json
import sys
import tempfile
try:
import requests
except ImportError:
requests = None
try:
from whoosh.fields import Schema
from whoosh.fields import TEXT
from whoosh.fields import STORED
from whoosh.index import create_in
from whoosh.qparser import QueryParser
except ImportError:
Schema = TEXT = STORED = create_in = QueryParser = None
QUAY_API_URL = 'https://quay.io/api/v1/repository'
class QuaySearch():
"""
Tool to search within a quay organization for a given software name.
"""
def __init__(self, organization):
self.index = None
self.organization = organization
def build_index(self):
"""
Create an index to quickly examine the repositories of a given quay.io organization.
"""
# download all information about the repositories from the
# given organization in self.organization
parameters = {'public': 'true', 'namespace': self.organization}
r = requests.get(QUAY_API_URL, headers={'Accept-encoding': 'gzip'}, params=parameters,
timeout=12)
tmp_dir = tempfile.mkdtemp()
schema = Schema(title=TEXT(stored=True), content=STORED)
self.index = create_in(tmp_dir, schema)
json_decoder = json.JSONDecoder()
decoded_request = json_decoder.decode(r.text)
writer = self.index.writer()
for repository in decoded_request['repositories']:
writer.add_document(title=repository['name'], content=repository['description'])
writer.commit()
def search_repository(self, search_string, non_strict):
"""
Search Docker containers on quay.io.
Results are displayed with all available versions,
including the complete image name.
"""
# with statement closes searcher after usage.
with self.index.searcher() as searcher:
search_string = "*%s*" % search_string
query = QueryParser("title", self.index.schema).parse(search_string)
results = searcher.search(query)
if non_strict:
# look for spelling errors and use suggestions as a search term too
corrector = searcher.corrector("title")
suggestions = corrector.suggest(search_string, limit=2)
# get all repositories with suggested keywords
for suggestion in suggestions:
search_string = "*%s*" % suggestion
query = QueryParser("title", self.index.schema).parse(search_string)
results_tmp = searcher.search(query)
results.extend(results_tmp)
sys.stdout.write("The query \033[1m %s \033[0m resulted in %s result(s).\n" % (search_string, len(results)))
if non_strict:
sys.stdout.write('The search was relaxed and the following search terms were searched: ')
sys.stdout.write('\033[1m %s \033[0m\n' % ', '.join(suggestions))
out = list()
for result in results:
title = result['title']
for version in self.get_additional_repository_information(title):
row = [title]
row.append(version)
out.append(row)
if out:
col_width = max(len(word) for row in out for word in row) + 2 # padding
for row in out:
name = row[0]
version = row[1]
sys.stdout.write("".join(word.ljust(col_width) for word in row) + "docker pull quay.io/%s/%s:%s\n" % (self.organization, name, version))
else:
sys.stdout.write("No results found for %s in quay.io/%s.\n" % (search_string, self.organization))
def get_additional_repository_information(self, repository_string):
"""
Function downloads additional information from quay.io to
get the tag-field which includes the version number.
"""
url = "%s/%s/%s" % (QUAY_API_URL, self.organization, repository_string)
r = requests.get(url, headers={'Accept-encoding': 'gzip'}, timeout=12)
json_decoder = json.JSONDecoder()
decoded_request = json_decoder.decode(r.text)
return decoded_request['tags']
def main(argv=None):
parser = argparse.ArgumentParser(description='Searches in a given quay organization for a repository')
parser.add_argument('-o', '--organization', dest='organization_string', default="mulled",
help='Change organization. Default is mulled.')
parser.add_argument('--non-strict', dest='non_strict', action="store_true",
help='Autocorrection of typos activated. Lists more results but can be confusing.\
For too many queries quay.io blocks the request and the results can be incomplete.')
parser.add_argument('-s', '--search', required=True,
help='The name of the tool you want to search for.')
args = parser.parse_args()
quay = QuaySearch(args.organization_string)
quay.build_index()
quay.search_repository(args.search, args.non_strict)
if __name__ == "__main__":
main()
lib/galaxy/tools/deps/mulled/util.py
+117
View File
@@ -0,0 +1,117 @@
"""Utilities for working with mulled abstractions outside the mulled package."""
from __future__ import print_function
import collections
import hashlib
from distutils.version import LooseVersion
try:
import requests
except ImportError:
requests = None
def quay_versions(namespace, pkg_name):
"""Get all version tags for a Docker image stored on quay.io for supplied package name."""
if requests is None:
raise Exception("requets library is unavailable, functionality not available.")
assert namespace is not None
assert pkg_name is not None
url = 'https://quay.io/api/v1/repository/%s/%s' % (namespace, pkg_name)
response = requests.get(url, timeout=None)
data = response.json()
if 'error_type' in data and data['error_type'] == "invalid_token":
return []
if 'tags' not in data:
raise Exception("Unexpected response from quay.io - not tags description found [%s]" % data)
return [tag for tag in data['tags'] if tag != 'latest']
def mulled_tags_for(namespace, image):
"""Fetch remote tags available for supplied image name.
The result will be sorted so newest tags are first.
"""
tags = quay_versions(namespace, image)
tags = version_sorted(tags)
return tags
def split_tag(tag):
"""Split mulled image name into conda version and conda build."""
version = tag.split('--', 1)[0]
build = tag.split('--', 1)[1]
return version, build
def version_sorted(elements):
"""Sort iterable based on loose description of "version" from newest to oldest."""
return sorted(elements, key=LooseVersion, reverse=True)
Target = collections.namedtuple("Target", ["package_name", "version", "build"])
def build_target(package_name, version=None, build=None, tag=None):
"""Use supplied arguments to build a :class:`Target` object."""
if tag is not None:
assert version is None
assert build is None
version, build = split_tag(tag)
return Target(package_name, version, build)
def conda_build_target_str(target):
rval = target.package_name
if target.version:
rval += "=%s" % target.version
if target.build:
rval += "=%s" % target.build
return rval
def image_name(targets, image_build=None, name_override=None):
if name_override is not None:
print("WARNING: Overriding mulled image name, auto-detection of 'mulled' package attributes will fail to detect result.")
return name_override
targets = list(targets)
if len(targets) == 1:
target = targets[0]
suffix = ""
if target.version is not None:
if image_build is not None:
print("WARNING: Hard-coding image build instead of using Conda build - this is not recommended.")
suffix = image_build
else:
suffix += ":%s" % target.version
build = target.build
if build is not None:
suffix += "--%s" % build
return "%s%s" % (target.package_name, suffix)
else:
targets_order = sorted(targets, key=lambda t: t.package_name)
requirements_buffer = "\n".join(map(conda_build_target_str, targets_order))
m = hashlib.sha1()
m.update(requirements_buffer.encode())
suffix = "" if not image_build else ":%s" % image_build
return "mulled-v1-%s%s" % (m.hexdigest(), suffix)
__all__ = (
"build_target",
"conda_build_target_str",
"image_name",
"mulled_tags_for",
"quay_versions",
"split_tag",
"Target",
"version_sorted",
)
lib/galaxy/tools/deps/resolvers/__init__.py
+12 -7
View File
@@ -1,3 +1,4 @@
"""The module defines the abstract interface for dealing tool dependency resolution plugins."""
from abc import (
ABCMeta,
abstractmethod,
@@ -10,6 +11,8 @@ from ..requirements import ToolRequirement
class DependencyResolver(Dictifiable, object):
"""Abstract description of a technique for resolving container images for tool execution."""
# Keys for dictification.
dict_collection_visible_keys = ['resolver_type', 'resolves_simple_dependencies']
# A "simple" dependency is one that does not depend on the the tool
@@ -23,13 +26,15 @@ class DependencyResolver(Dictifiable, object):
@abstractmethod
def resolve( self, name, version, type, **kwds ):
"""
Given inputs describing dependency in the abstract, yield tuple of
(script, bin, version). Here script is the env.sh file to source
before running a job, if that is not found the bin directory will be
appended to the path (if it is not None). Finally, version is the
resolved tool dependency version (which may differ from requested
version for instance if the request version is 'default'.)
"""Given inputs describing dependency in the abstract yield a Dependency object.
The Dependency object describes various attributes (script, bin,
version) used to build scripts with the dependency availble. Here
script is the env.sh file to source before running a job, if that is
not found the bin directory will be appended to the path (if it is
not ``None``). Finally, version is the resolved tool dependency
version (which may differ from requested version for instance if the
request version is 'default'.)
"""
def _get_config_option(self, key, dependency_resolver, default=None, config_prefix=None, **kwds):
lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py
+1 -2
View File
@@ -5,7 +5,6 @@ via shed2tap (e.g. https://github.com/jmchilton/homebrew-toolshed).
"""
import logging
import os
from xml.etree import ElementTree as ET
from .resolver_mixins import (
@@ -147,4 +146,4 @@ def build_recipe_name(package_name, package_version, repository_owner, repositor
return base
__all__ = ['HomebrewToolShedDependencyResolver']
__all__ = ('HomebrewToolShedDependencyResolver', )
lib/galaxy/tools/deps/resolvers/conda.py
+5 -40
View File
@@ -6,10 +6,8 @@ incompatible changes coming.
import logging
import os
from galaxy.util.filelock import (
FileLock,
FileLockException
)
import galaxy.tools.deps.installable
from ..conda_util import (
build_isolated_environment,
cleanup_failed_install,
@@ -32,7 +30,7 @@ from ..resolvers import (
DEFAULT_BASE_PATH_DIRECTORY = "_conda"
DEFAULT_CONDARC_OVERRIDE = "_condarc"
DEFAULT_ENSURE_CHANNELS = "r,bioconda,iuc"
DEFAULT_ENSURE_CHANNELS = "conda-forge,r,bioconda,iuc"
log = logging.getLogger(__name__)
@@ -95,44 +93,11 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
copy_dependencies = _string_as_bool(get_option("copy_dependencies"))
self.auto_init = _string_as_bool(get_option("auto_init"))
self.conda_context = conda_context
self.ensure_conda_installed()
self.disabled = not galaxy.tools.deps.installable.ensure_installed(conda_context, install_conda, self.auto_init)
self.auto_install = auto_install
self.copy_dependencies = copy_dependencies
self.verbose_install_check = verbose_install_check
def ensure_conda_installed(self):
"""
Make sure that conda is installed, and if conda can't be installed, mark resolver as disabled.
We acquire a lock, so that multiple handlers do not attempt to install conda simultaneously.
"""
target_path = self.conda_prefix_parent
def _check():
if not self.conda_context.is_conda_installed():
if self.auto_init:
if self.conda_context.can_install_conda():
if install_conda(self.conda_context):
self.disabled = True
log.warning("Conda installation requested and failed.")
else:
self.disabled = True
else:
self.disabled = True
log.warning("Conda not installed and auto-installation disabled.")
else:
self.disabled = False
if not os.path.exists(target_path):
os.mkdir(target_path)
try:
if self.auto_init and os.access(target_path, os.W_OK):
with FileLock(os.path.join(target_path, 'conda')):
_check()
else:
_check()
except FileLockException:
self.ensure_conda_installed()
def resolve(self, name, version, type, **kwds):
# Check for conda just not being there, this way we can enable
# conda by default and just do nothing in not configured.
@@ -279,4 +244,4 @@ def _string_as_bool( value ):
return str( value ).lower() == "true"
__all__ = ['CondaDependencyResolver']
__all__ = ('CondaDependencyResolver', )
lib/galaxy/tools/deps/resolvers/galaxy_packages.py
+5 -1
View File
@@ -122,4 +122,8 @@ def _is_dependency_directory(directory):
return exists(join(directory, 'env.sh')) or exists(join(directory, 'bin'))
__all__ = ['GalaxyPackageDependencyResolver', 'GalaxyPackageDependency', 'ToolShedDependency']
__all__ = (
'GalaxyPackageDependency',
'GalaxyPackageDependencyResolver',
'ToolShedDependency'
)
lib/galaxy/tools/deps/resolvers/homebrew.py
+1 -1
View File
@@ -53,4 +53,4 @@ def _string_as_bool( value ):
return str( value ).lower() == "true"
__all__ = ['HomebrewDependencyResolver']
__all__ = ('HomebrewDependencyResolver', )
lib/galaxy/tools/deps/resolvers/modules.py
+1 -2
View File
@@ -7,7 +7,6 @@ it, hence support for it will be minimal. The Galaxy team eagerly welcomes
community contribution and maintenance however.
"""
import logging
from os import environ, pathsep
from os.path import exists, isdir, join
from subprocess import PIPE, Popen
@@ -180,4 +179,4 @@ class ModuleDependency(Dependency):
def _string_as_bool( value ):
return str( value ).lower() == "true"
__all__ = ['ModuleDependencyResolver']
__all__ = ('ModuleDependencyResolver', )
lib/galaxy/tools/deps/resolvers/tool_shed_packages.py
+1 -2
View File
@@ -2,7 +2,6 @@ from os.path import abspath, exists, join
from .galaxy_packages import BaseGalaxyPackageDependencyResolver, ToolShedDependency
from .resolver_mixins import UsesInstalledRepositoriesMixin
from ..resolvers import NullDependency
@@ -64,4 +63,4 @@ class ToolShedPackageDependencyResolver(BaseGalaxyPackageDependencyResolver, Use
return NullDependency(version=None, name=name)
__all__ = ['ToolShedPackageDependencyResolver']
__all__ = ('ToolShedPackageDependencyResolver', )
lib/galaxy/tools/deps/resolvers/unlinked_tool_shed_packages.py
+1 -2
View File
@@ -19,7 +19,6 @@ See bottom for instructions on how to add this resolver.
"""
import logging
from os import listdir
from os.path import exists, getmtime, join
@@ -153,7 +152,7 @@ class CandidateDependency(Dependency):
return self.dependency.shell_commands( requirement )
__all__ = ['UnlinkedToolShedPackageDependencyResolver']
__all__ = ('UnlinkedToolShedPackageDependencyResolver', )
"""
At the time of writing July 3 2015 this resolver has to be plugged in.
lib/galaxy/tools/exception_handling.py
+2 -2
View File
@@ -1,6 +1,6 @@
# We put a tool that references this package into the tool shed
# so we have to provide this legacy location for import indefinitely
# it seems.
from galaxy.util.ucsc import UCSCOutWrapper, UCSCLimitException
from galaxy.util.ucsc import UCSCLimitException, UCSCOutWrapper
__all__ = ['UCSCOutWrapper', 'UCSCLimitException']
__all__ = ('UCSCOutWrapper', 'UCSCLimitException')
lib/galaxy/tools/execute.py
+10 -9
View File
@@ -4,13 +4,15 @@ from various states, tracking results, and building implicit dataset
collections from matched collections.
"""
import collections
import logging
from threading import Thread
from six.moves.queue import Queue
from galaxy.tools.actions import on_text_for_names, ToolExecutionCache
from galaxy.tools.parser import ToolOutputCollectionPart
from galaxy.util import ExecutionTimer
from galaxy.tools.actions import on_text_for_names, ToolExecutionCache
from threading import Thread
from Queue import Queue
import logging
log = logging.getLogger( __name__ )
EXECUTION_SUCCESS_MESSAGE = "Tool [%s] created job [%s] %s"
@@ -135,7 +137,6 @@ class ToolExecutionTracker( object ):
return []
structure = self.collection_info.structure
collections = self.collection_info.collections.values()
# params is just one sample tool param execution with parallelized
# collection replaced with a specific dataset. Need to replace this
@@ -143,13 +144,13 @@ class ToolExecutionTracker( object ):
# label.
params.update( self.collection_info.collections ) # Replace datasets with source collections for labelling outputs.
collection_names = map( lambda c: "collection %d" % c.hid, collections )
collection_names = ["collection %d" % c.hid for c in self.collection_info.collections.values()]
on_text = on_text_for_names( collection_names )
collections = {}
implicit_inputs = list(self.collection_info.collections.iteritems())
for output_name, outputs in self.outputs_by_output_name.iteritems():
implicit_inputs = list(self.collection_info.collections.items())
for output_name, outputs in self.outputs_by_output_name.items():
if not len( structure ) == len( outputs ):
# Output does not have the same structure, if all jobs were
# successfully submitted this shouldn't have happened.
@@ -202,4 +203,4 @@ class ToolExecutionTracker( object ):
trans.sa_session.flush()
self.implicit_collections = collections
__all__ = [ execute ]
__all__ = ( 'execute', )
lib/galaxy/tools/loader.py
+1 -1
View File
@@ -7,4 +7,4 @@ from galaxy.util.xml_macros import (
load_tool = load
__all__ = ["load_tool", "raw_tool_xml_tree", "imported_macro_paths", "template_macro_params"]
__all__ = ("load_tool", "raw_tool_xml_tree", "imported_macro_paths", "template_macro_params")
lib/galaxy/tools/loader_directory.py
+4 -4
View File
@@ -241,11 +241,11 @@ def _find_tool_files(path, recursive, enable_beta_formats):
files = glob.glob(path + "/*.xml")
else:
files = _find_files(path, "*.xml")
return map(os.path.abspath, files)
return [os.path.abspath(_) for _ in files]
def _has_extension(path, extensions):
return any(map(lambda e: path.endswith(e), extensions))
return any(path.endswith(e) for e in extensions)
def _find_files(directory, pattern='*'):
@@ -266,7 +266,7 @@ BETA_TOOL_CHECKERS = {
'cwl': looks_like_a_tool_cwl,
}
__all__ = [
__all__ = (
"find_possible_tools_from_path",
"is_a_yaml_with_class",
"is_tool_load_error",
@@ -276,4 +276,4 @@ __all__ = [
"looks_like_a_tool_cwl",
"looks_like_a_tool_xml",
"looks_like_a_tool_yaml",
]
)
lib/galaxy/tools/parameters/__init__.py
+9 -8
View File
@@ -4,15 +4,16 @@ Classes encapsulating Galaxy tool parameters.
import re
from json import dumps, loads
from basic import RuntimeValue, DataCollectionToolParameter, DataToolParameter, SelectToolParameter
from grouping import Conditional, Repeat, Section, UploadDataset
from galaxy.util.expressions import ExpressionContext
from galaxy.util.json import json_fix
from .basic import DataCollectionToolParameter, DataToolParameter, RuntimeValue, SelectToolParameter
from .grouping import Conditional, Repeat, Section, UploadDataset
REPLACE_ON_TRUTHY = object()
# Some tools use the code tag and access the code base, expecting certain tool parameters to be available here.
__all__ = [ DataCollectionToolParameter, DataToolParameter, SelectToolParameter ]
__all__ = ( 'DataCollectionToolParameter', 'DataToolParameter', 'SelectToolParameter' )
def visit_input_values( inputs, input_values, callback, name_prefix='', label_prefix='', parent_prefix='', context=None, no_replacement_value=REPLACE_ON_TRUTHY ):
@@ -82,7 +83,7 @@ def visit_input_values( inputs, input_values, callback, name_prefix='', label_pr
context = ExpressionContext( input_values, context )
payload = { 'context': context, 'no_replacement_value': no_replacement_value }
for input in inputs.itervalues():
for input in inputs.values():
if isinstance( input, Repeat ) or isinstance( input, UploadDataset ):
values = input_values[ input.name ] = input_values.get( input.name, [] )
for i, d in enumerate( values ):
@@ -143,7 +144,7 @@ def params_to_strings( params, param_values, app ):
such).
"""
rval = dict()
for key, value in param_values.iteritems():
for key, value in param_values.items():
if key in params:
value = params[ key ].value_to_basic( value, app )
rval[ key ] = str( dumps( value ) )
@@ -158,7 +159,7 @@ def params_from_strings( params, param_values, app, ignore_errors=False ):
preferred form).
"""
rval = dict()
for key, value in param_values.iteritems():
for key, value in param_values.items():
value = json_fix( loads( value ) )
if key in params:
value = params[ key ].value_from_basic( value, app, ignore_errors )
@@ -173,7 +174,7 @@ def params_to_incoming( incoming, inputs, input_values, app, name_prefix="" ):
Useful for e.g. the rerun function.
"""
for input in inputs.itervalues():
for input in inputs.values():
if isinstance( input, Repeat ) or isinstance( input, UploadDataset ):
for d in input_values[ input.name ]:
index = d[ '__index__' ]
@@ -201,7 +202,7 @@ def update_param( prefixed_name, input_values, new_value ):
"""
for key in input_values:
match = re.match( '^' + key + '_(\d+)\|(.+)', prefixed_name )
if match:
if match and not key.endswith( "|__identifier__" ):
index = int( match.group( 1 ) )
if isinstance( input_values[ key ], list ) and len( input_values[ key ] ) > index:
update_param( match.group( 2 ), input_values[ key ][ index ], new_value )
lib/galaxy/tools/parameters/basic.py
+1 -1
View File
@@ -151,7 +151,7 @@ class ToolParameter( object, Dictifiable ):
Convert a value to a text representation suitable for displaying to
the user
"""
if value:
if value is not None:
return unicodify( value )
return "Not available."
lib/galaxy/tools/parameters/dataset_matcher.py
+3 -2
View File
@@ -1,6 +1,7 @@
from logging import getLogger
import galaxy.model
from logging import getLogger
log = getLogger( __name__ )
ROLES_UNSET = object()
@@ -179,4 +180,4 @@ class DatasetCollectionMatcher( object ):
break
return valid
__all__ = [ DatasetMatcher, DatasetCollectionMatcher ]
__all__ = ( 'DatasetMatcher', 'DatasetCollectionMatcher' )
lib/galaxy/tools/parameters/wrapped.py
+12 -11
View File
@@ -1,20 +1,21 @@
from galaxy.tools.parameters.basic import (
DataToolParameter,
DataCollectionToolParameter,
DataToolParameter,
SelectToolParameter
)
from galaxy.tools.wrappers import (
InputValueWrapper,
SelectToolParameterWrapper,
DatasetFilenameWrapper,
DatasetListWrapper,
DatasetCollectionWrapper
)
from galaxy.tools.parameters.grouping import (
Repeat,
Conditional,
Repeat,
Section
)
from galaxy.tools.wrappers import (
DatasetCollectionWrapper,
DatasetFilenameWrapper,
DatasetListWrapper,
InputValueWrapper,
SelectToolParameterWrapper
)
PARAMS_UNWRAPPED = object()
@@ -40,7 +41,7 @@ class WrappedParameters( object ):
incoming = self.incoming
# Wrap tool inputs as necessary
for input in inputs.itervalues():
for input in inputs.values():
if input.name not in input_values and skip_missing_values:
continue
value = input_values[ input.name ]
@@ -112,4 +113,4 @@ def make_list_copy( from_list ):
return new_list
__all__ = [ 'WrappedParameters', 'make_dict_copy' ]
__all__ = ( 'WrappedParameters', 'make_dict_copy' )
lib/galaxy/tools/parameters/wrapped_json.py
+4 -4
View File
@@ -1,6 +1,6 @@
import logging
log = logging.getLogger(__name__)
log = logging.getLogger(__name__)
SKIP_INPUT = object()
@@ -9,7 +9,7 @@ def json_wrap(inputs, input_values, as_dict=None, handle_files="SKIP"):
if as_dict is None:
as_dict = {}
for input in inputs.itervalues():
for input in inputs.values():
input_name = input.name
value = input_values[input_name]
json_value = _json_wrap_input(input, value, handle_files=handle_files)
@@ -65,7 +65,7 @@ def _json_wrap_input(input, value, handle_files="SKIP"):
json_value = _cast_if_not_none(value, bool)
elif input_type == "data_column":
# value is a SelectToolParameterWrapper()
json_value = map(int, _cast_if_not_none(value.value, list))
json_value = [int(_) for _ in _cast_if_not_none(value.value, list)]
else:
raise NotImplementedError("input_type [%s] not implemented" % input_type)
@@ -80,4 +80,4 @@ def _cast_if_not_none(value, cast_to, empty_to_none=False):
return cast_to(value)
__all__ = ['json_wrap']
__all__ = ('json_wrap', )
lib/galaxy/tools/parser/__init__.py
+3 -4
View File
@@ -1,15 +1,14 @@
""" Package responsible for parsing tools from files/abstract tool sources.
"""
from .factory import get_input_source
from .factory import get_tool_source
from .factory import get_input_source, get_tool_source
from .interface import ToolSource
from .output_objects import (
ToolOutputCollectionPart,
)
__all__ = [
__all__ = (
"get_input_source",
"get_tool_source",
"ToolOutputCollectionPart",
"ToolSource",
]
)
lib/galaxy/tools/parser/factory.py
+2 -3
View File
@@ -10,8 +10,7 @@ from galaxy.util.odict import odict
from .cwl import CwlToolSource
from .interface import InputSource
from .xml import XmlInputSource
from .xml import XmlToolSource
from .xml import XmlInputSource, XmlToolSource
from .yaml import YamlToolSource
log = logging.getLogger(__name__)
@@ -72,4 +71,4 @@ def get_input_source(content):
return content
__all__ = ["get_tool_source", "get_input_source"]
__all__ = ("get_tool_source", "get_input_source")
lib/galaxy/tools/toolbox/__init__.py
+4 -7
View File
@@ -1,16 +1,13 @@
"""API for this module containing functionality related to the toolbox."""
from .base import AbstractToolBox
from .base import BaseGalaxyToolBox
from .base import AbstractToolBox, BaseGalaxyToolBox
from .panel import panel_item_types
from .panel import ToolSection
from .panel import ToolSectionLabel
from .panel import panel_item_types, ToolSection, ToolSectionLabel
__all__ = [
__all__ = (
"AbstractToolBox",
"BaseGalaxyToolBox",
"panel_item_types",
"ToolSection",
"ToolSectionLabel",
]
)
lib/galaxy/tools/toolbox/base.py
+8 -3
View File
@@ -520,15 +520,20 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
tool = self.load_tool_from_cache(os.path.join(tool_path, path))
from_cache = tool
if from_cache:
log.debug("Loading tool %s from cache", str(tool.id))
elif guid: # tool was not in cache and is a tool shed tool
if guid and tool.id != guid:
# In rare cases a tool shed tool is loaded into the cache without guid.
# In that case recreating the tool will correct the cached version.
from_cache = False
else:
log.debug("Loading tool %s from cache", str(tool.id))
if guid and not from_cache: # tool was not in cache and is a tool shed tool
tool_shed_repository = self.get_tool_repository_from_xml_item(item, path)
if tool_shed_repository:
# Only load tools if the repository is not deactivated or uninstalled.
can_load_into_panel_dict = not tool_shed_repository.deleted
repository_id = self.app.security.encode_id(tool_shed_repository.id)
tool = self.load_tool(os.path.join( tool_path, path ), guid=guid, repository_id=repository_id, use_cached=False)
else: # tool was not in cache and is not a tool shed tool.
if not tool: # tool was not in cache and is not a tool shed tool.
tool = self.load_tool(os.path.join(tool_path, path), use_cached=False)
if string_as_bool(item.get( 'hidden', False )):
tool.hidden = True
lib/galaxy/tools/toolbox/lineages/__init__.py
+1 -1
View File
@@ -3,4 +3,4 @@ from .interface import ToolLineage
from .tool_shed import ToolVersionCache
__all__ = ["LineageMap", "ToolLineage", "ToolVersionCache"]
__all__ = ("LineageMap", "ToolLineage", "ToolVersionCache")
lib/galaxy/tools/toolbox/lineages/factory.py
+1 -1
View File
@@ -46,4 +46,4 @@ class LineageMap(object):
return self.lineage_map.get(versionless_tool_id, None)
__all__ = ["LineageMap"]
__all__ = ("LineageMap", )
lib/galaxy/tools/toolbox/lineages/tool_shed.py
+4 -5
View File
@@ -1,5 +1,4 @@
from .interface import ToolLineage
from .interface import ToolLineageVersion
from .interface import ToolLineage, ToolLineageVersion
try:
from galaxy.model.tool_shed_install import ToolVersion
@@ -78,13 +77,13 @@ class ToolShedLineage(ToolLineage):
return result
def get_versions( self, reverse=False ):
return map( ToolLineageVersion.from_guid, self.get_version_ids( reverse=reverse ) )
return [ ToolLineageVersion.from_guid(_) for _ in self.get_version_ids( reverse=reverse ) ]
def to_dict(self):
tool_shed_repository = self._tool_shed_repository
rval = dict(
tool_version_id=self.tool_version_id,
tool_versions=map(lambda v: v.to_dict(), self.get_versions()),
tool_versions=[v.to_dict() for v in self.get_versions()],
tool_shed_repository=tool_shed_repository if tool_shed_repository is not None else None,
lineage_type='tool_shed',
)
@@ -94,4 +93,4 @@ class ToolShedLineage(ToolLineage):
def get_installed_tool_version( app, tool_id ):
return app.tool_version_cache.tool_version_by_tool_id.get(tool_id, None)
__all__ = [ "ToolShedLineage" ]
__all__ = ( "ToolShedLineage", )
lib/galaxy/tools/toolbox/parser.py
+10 -11
View File
@@ -3,9 +3,9 @@
These files define tool lists, sections, labels, etc... the elements of the
Galaxy tool panel.
"""
from abc import ABCMeta
from abc import abstractmethod
from abc import ABCMeta, abstractmethod
import six
import yaml
from galaxy.util import parse_xml, string_as_bool
@@ -13,11 +13,10 @@ from galaxy.util import parse_xml, string_as_bool
DEFAULT_MONITOR = False
@six.add_metaclass(ABCMeta)
class ToolConfSource(object):
"""Interface represents a container of tool references."""
__metaclass__ = ABCMeta
@abstractmethod
def parse_items(self):
"""Return a list of ToolConfItem describing source."""
@@ -45,7 +44,7 @@ class XmlToolConfSource(ToolConfSource):
return self.root.get('tool_path')
def parse_items(self):
return map(ensure_tool_conf_item, self.root.getchildren())
return [ensure_tool_conf_item(_) for _ in self.root.getchildren()]
def is_shed_tool_conf(self):
has_tool_path = self.parse_tool_path() is not None
@@ -67,7 +66,7 @@ class YamlToolConfSource(ToolConfSource):
return self.as_dict.get('tool_path')
def parse_items(self):
return map(ToolConfItem.from_dict, self.as_dict.get('items'))
return [ToolConfItem.from_dict(_) for _ in self.as_dict.get('items')]
def parse_monitor(self):
return self.as_dict.get('monitor', DEFAULT_MONITOR)
@@ -94,7 +93,7 @@ class ToolConfItem(object):
del as_dict['type']
attributes = as_dict
if type == 'section':
items = map(cls.from_dict, as_dict['items'])
items = [cls.from_dict(_) for _ in as_dict['items']]
del as_dict['items']
item = ToolConfSection(attributes, items)
else:
@@ -141,18 +140,18 @@ def ensure_tool_conf_item(xml_or_item):
if type != "section":
return ToolConfItem(type, attributes, elem)
else:
items = map(ensure_tool_conf_item, elem.getchildren())
items = [ensure_tool_conf_item(_) for _ in elem.getchildren()]
return ToolConfSection(attributes, items, elem=elem)
def get_toolbox_parser(config_filename):
is_yaml = any(map(lambda e: config_filename.endswith(e), [".yml", ".yaml", ".json"]))
is_yaml = any(config_filename.endswith(e) for e in [".yml", ".yaml", ".json"])
if is_yaml:
return YamlToolConfSource(config_filename)
else:
return XmlToolConfSource(config_filename)
__all__ = [
__all__ = (
"get_toolbox_parser",
"ensure_tool_conf_item",
]
)
lib/galaxy/tools/xsd/galaxy.xsd
+28 -1
View File
@@ -1990,6 +1990,33 @@ multiple formats.
help="The set of aligned reads." />
```
Perhaps counter-intuitively, a ``multiple="true"`` data parameter requires at least one
data input. If ``optional="true"`` is specified, this condition is relaxed and the user
is allowed to select 0 datasets. Unfortunately, if 0 datasets are selected the resulting
value for the parameter during Cheetah templating (such as in a ``command`` block) will
effectively be a list with one ``None``-like entity in it.
The following idiom can be used to iterate over such a list and build a hypothetical ``-B``
parameter for each file - the ``if`` block is used to handle the case where a ``None``-like
entity appears in the list because no files were selected:
```
#for $input in $input1
#if $input
-B "$input"
#end if
#end for
```
Some example tools using ``multiple="true"`` data parameters include:
- [multi_data_param.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/multi_data_param.xml)
- [multi_data_optional.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/multi_data_optional.xml)
Additionally, a detailed discussion of handling multiple homogenous files can be found in the
the [Planemo Documentation](http://planemo.readthedocs.io/en/latest/writing_advanced.html#consuming-collections)
on this topic.
$attribute_list:format,multiple
#### ``select``
@@ -3661,7 +3688,7 @@ conditionals are accessed using a hash named after the conditional.
</inputs>
<outputs>
<data format="txt" label="Alignment report on ${on_string}" name="output_txt" />
<data format="vcf" label="Variant summary on ${on_string}" name="output_vcf>
<data format="vcf" label="Variant summary on ${on_string}" name="output_vcf">
<filter>options['selection_mode'] == 'advanced' and options['vcf_output']</filter>
</data>
</outputs>
lib/galaxy/util/checkers.py
+8 -24
View File
@@ -1,38 +1,15 @@
import bz2
import gzip
import imghdr
import re
import zipfile
from six import StringIO
from galaxy import util
from galaxy.util.image_util import image_type
HTML_CHECK_LINES = 100
try:
import Image as PIL
except ImportError:
try:
from PIL import Image as PIL
except:
PIL = None
def check_image( file_path ):
if PIL is not None:
try:
im = PIL.open( file_path )
except:
return False
if im:
return im
return False
else:
if imghdr.what( file_path ) is not None:
return True
return False
def check_html( file_path, chunk=None ):
if chunk is None:
@@ -143,6 +120,13 @@ def is_gzip( file_path ):
return is_gzipped
def check_image( file_path ):
""" Simple wrapper around image_type to yield a True/False verdict """
if image_type( file_path ):
return True
return False
__all__ = [
'check_binary',
'check_bz2',
lib/galaxy/util/image_util.py
+75
View File
@@ -0,0 +1,75 @@
"""Provides utilities for working with image files."""
import imghdr
import logging
try:
import Image as PIL
except ImportError:
try:
from PIL import Image as PIL
except:
PIL = None
log = logging.getLogger(__name__)
def image_type( filename ):
fmt = None
if PIL is not None:
try:
im = PIL.open( filename )
fmt = im.format
im.close()
except:
# We continue to try with imghdr, so this is a rare case of an
# exception we expect to happen frequently, so we're not logging
pass
if not fmt:
fmt = imghdr.what( filename )
if fmt:
return fmt.upper()
else:
return False
def check_image_type( filename, types ):
fmt = image_type( filename )
if fmt in types:
return True
return False
def get_image_ext( file_path ):
# determine ext
fmt = image_type( file_path )
if fmt in [ 'JPG', 'JPEG' ]:
return 'jpg'
if fmt == 'PNG':
return 'png'
if fmt == 'TIFF':
return 'tiff'
if fmt == 'BMP':
return 'bmp'
if fmt == 'GIF':
return 'gif'
if fmt == 'IM':
return 'im'
if fmt == 'PCD':
return 'pcd'
if fmt == 'PCX':
return 'pcx'
if fmt == 'PPM':
return 'ppm'
if fmt == 'PSD':
return 'psd'
if fmt == 'XBM':
return 'xbm'
if fmt == 'XPM':
return 'xpm'
if fmt == 'RGB':
return 'rgb'
if fmt == 'PBM':
return 'pbm'
if fmt == 'PGM':
return 'pgm'
return None
lib/galaxy/workflow/extract.py
+51 -49
View File
@@ -1,25 +1,26 @@
""" This module contains functionality to aid in extracting workflows from
histories.
"""
from galaxy.util.odict import odict
from galaxy import exceptions
from galaxy import model
import logging
from galaxy import exceptions, model
from galaxy.tools.parameters.basic import (
DataToolParameter,
DataCollectionToolParameter
DataCollectionToolParameter,
DataToolParameter
)
from galaxy.tools.parser import ToolOutputCollectionPart
from galaxy.tools.parameters.grouping import (
Conditional,
Repeat,
Section
)
from galaxy.tools.parser import ToolOutputCollectionPart
from galaxy.util.odict import odict
from .steps import (
attach_ordered_steps,
order_workflow_steps_with_levels
)
import logging
log = logging.getLogger( __name__ )
WARNING_SOME_DATASETS_NOT_READY = "Some datasets still queued or running were ignored"
@@ -67,16 +68,14 @@ def extract_steps( trans, history=None, job_ids=None, dataset_ids=None, dataset_
elif type( dataset_collection_ids) is not list:
dataset_collection_ids = [ dataset_collection_ids ]
# Convert both sets of ids to integers
job_ids = [ int( id ) for id in job_ids ]
dataset_ids = [ int( id ) for id in dataset_ids ]
dataset_collection_ids = [ int( id ) for id in dataset_collection_ids ]
# Find each job, for security we (implicately) check that they are
# associated witha job in the current history.
job_ids = [ int( _ ) for _ in job_ids ]
dataset_ids = [ int( _ ) for _ in dataset_ids ]
dataset_collection_ids = [ int( _ ) for _ in dataset_collection_ids ]
# Find each job, for security we (implicitly) check that they are
# associated with a job in the current history.
summary = WorkflowSummary( trans, history )
jobs = summary.jobs
jobs_by_id = dict( ( job.id, job ) for job in jobs.keys() )
steps = []
steps_by_job_id = {}
hid_to_output_pair = {}
# Input dataset steps
for i, hid in enumerate( dataset_ids ):
@@ -104,10 +103,10 @@ def extract_steps( trans, history=None, job_ids=None, dataset_ids=None, dataset_
steps.append( step )
# Tool steps
for job_id in job_ids:
if job_id not in jobs_by_id:
log.warning( "job_id %s not found in jobs_by_id %s" % ( job_id, jobs_by_id ) )
if job_id not in summary.job_id2representative_job:
log.warning( "job_id %s not found in job_id2representative_job %s" % ( job_id, summary.job_id2representative_job ) )
raise AssertionError( "Attempt to create workflow with job not connected to current history" )
job = jobs_by_id[ job_id ]
job = summary.job_id2representative_job[job_id]
tool_inputs, associations = step_inputs( trans, job )
step = model.WorkflowStep()
step.type = 'tool'
@@ -134,7 +133,6 @@ def extract_steps( trans, history=None, job_ids=None, dataset_ids=None, dataset_
conn.output_step = other_step
conn.output_name = other_name
steps.append( step )
steps_by_job_id[ job_id ] = step
# Store created dataset hids
for assoc in (job.output_datasets + job.output_dataset_collection_instances):
assoc_name = assoc.name
@@ -147,8 +145,8 @@ def extract_steps( trans, history=None, job_ids=None, dataset_ids=None, dataset_
if query_assoc_name == assoc_name:
hid = dataset_collection.hid
if hid is None:
template = "Failed to find matching implicit job - job is %s, jobs are %s, assoc_name is %s."
message = template % ( job.id, jobs, assoc.name )
template = "Failed to find matching implicit job - job id is %s, implicit pairs are %s, assoc_name is %s."
message = template % ( job.id, jobs[job], assoc_name )
log.warning( message )
raise Exception( "Failed to extract job." )
else:
@@ -202,6 +200,7 @@ class WorkflowSummary( object ):
self.history = history
self.warnings = set()
self.jobs = odict()
self.job_id2representative_job = {} # map a non-fake job id to its representative job
self.implicit_map_jobs = []
self.collection_types = {}
@@ -223,25 +222,27 @@ class WorkflowSummary( object ):
else:
self.__summarize_dataset( content )
def __summarize_dataset_collection( self, content ):
content = self.__original_hdca( content )
dataset_collection = content
hid = content.hid
self.collection_types[ hid ] = content.collection.collection_type
cja = content.creating_job_associations
def __summarize_dataset_collection( self, dataset_collection ):
dataset_collection = self.__original_hdca( dataset_collection )
hid = dataset_collection.hid
self.collection_types[ hid ] = dataset_collection.collection.collection_type
cja = dataset_collection.creating_job_associations
if cja:
# Use the first job to represent all mapped jobs.
representive_job_assoc = content.creating_job_associations[0]
job = representive_job_assoc.job
if job not in self.jobs or self.jobs[ job ][ 0 ][ 1 ].history_content_type == "dataset":
self.jobs[ job ] = [ ( representive_job_assoc.name, dataset_collection ) ]
if content.implicit_output_name:
self.implicit_map_jobs.append( job )
# Use the "first" job to represent all mapped jobs.
representative_assoc = cja[0]
representative_job = representative_assoc.job
if representative_job not in self.jobs or self.jobs[ representative_job ][ 0 ][ 1 ].history_content_type == "dataset":
self.jobs[ representative_job ] = [ ( representative_assoc.name, dataset_collection ) ]
if dataset_collection.implicit_output_name:
self.implicit_map_jobs.append( representative_job )
else:
self.jobs[ job ].append( ( representive_job_assoc.name, dataset_collection ) )
self.jobs[ representative_job ].append( ( representative_assoc.name, dataset_collection ) )
for assoc in cja:
job = assoc.job
self.job_id2representative_job[job.id] = representative_job
# This whole elif condition may no longer be needed do to additional
# tracking with creating_job_associations. Will delete at some point.
elif content.implicit_output_name:
elif dataset_collection.implicit_output_name:
# TODO: Optimize db call
dataset_instance = dataset_collection.collection.dataset_instances[ 0 ]
if not self.__check_state( dataset_instance ):
@@ -249,38 +250,40 @@ class WorkflowSummary( object ):
# makes me wonder if even need this check at all?
return
job_hda = self.__original_hda( dataset_instance )
if not job_hda.creating_job_associations:
original_hda = self.__original_hda( dataset_instance )
if not original_hda.creating_job_associations:
log.warning( "An implicitly create output dataset collection doesn't have a creating_job_association, should not happen!" )
job = DatasetCollectionCreationJob( dataset_collection )
self.jobs[ job ] = [ ( None, dataset_collection ) ]
for assoc in job_hda.creating_job_associations:
for assoc in original_hda.creating_job_associations:
job = assoc.job
if job not in self.jobs or self.jobs[ job ][ 0 ][ 1 ].history_content_type == "dataset":
self.jobs[ job ] = [ ( assoc.name, dataset_collection ) ]
self.job_id2representative_job[job.id] = job
self.implicit_map_jobs.append( job )
else:
self.jobs[ job ].append( ( assoc.name, dataset_collection ) )
else:
job = DatasetCollectionCreationJob( content )
self.jobs[ job ] = [ ( None, content ) ]
job = DatasetCollectionCreationJob( dataset_collection )
self.jobs[ job ] = [ ( None, dataset_collection ) ]
def __summarize_dataset( self, dataset ):
if not self.__check_state( dataset ):
return
job_hda = self.__original_hda( dataset )
original_hda = self.__original_hda( dataset )
if not job_hda.creating_job_associations:
if not original_hda.creating_job_associations:
self.jobs[ FakeJob( dataset ) ] = [ ( None, dataset ) ]
for assoc in job_hda.creating_job_associations:
for assoc in original_hda.creating_job_associations:
job = assoc.job
if job in self.jobs:
self.jobs[ job ].append( ( assoc.name, dataset ) )
else:
self.jobs[ job ] = [ ( assoc.name, dataset ) ]
self.job_id2representative_job[job.id] = job
def __original_hdca( self, hdca ):
while hdca.copied_from_history_dataset_collection_association:
@@ -288,11 +291,10 @@ class WorkflowSummary( object ):
return hdca
def __original_hda( self, hda ):
# if this hda was copied from another, we need to find the job that created the origial hda
job_hda = hda
while job_hda.copied_from_history_dataset_association:
job_hda = job_hda.copied_from_history_dataset_association
return job_hda
# if this hda was copied from another, we need to find the job that created the original hda
while hda.copied_from_history_dataset_association:
hda = hda.copied_from_history_dataset_association
return hda
def __check_state( self, hda ):
# FIXME: Create "Dataset.is_finished"
@@ -369,4 +371,4 @@ def __cleanup_param_values( inputs, values ):
cleanup( "", inputs, values )
return associations
__all__ = [ summarize, extract_workflow ]
__all__ = ( 'summarize', 'extract_workflow' )
lib/galaxy/workflow/run.py
+7 -9
View File
@@ -1,15 +1,13 @@
import logging
import uuid
from galaxy import model
from galaxy import util
from galaxy.util.odict import odict
from galaxy import model, util
from galaxy.util import ExecutionTimer
from galaxy.util.odict import odict
from galaxy.workflow import modules
from galaxy.workflow.run_request import WorkflowRunConfig
from galaxy.workflow.run_request import workflow_run_config_to_request
from galaxy.workflow.run_request import (workflow_run_config_to_request,
WorkflowRunConfig)
import logging
log = logging.getLogger( __name__ )
@@ -253,7 +251,7 @@ class WorkflowProgress( object ):
self.module_injector.inject( step )
if step_id not in step_states:
template = "Workflow invocation [%s] has no step state for step id [%s]. States ids are %s."
message = template % (self.workflow_invocation.id, step_id, step_states.keys())
message = template % (self.workflow_invocation.id, step_id, list(step_states.keys()))
raise Exception(message)
runtime_state = step_states[ step_id ].value
step.state = step.module.recover_runtime_state( runtime_state )
@@ -399,4 +397,4 @@ class WorkflowProgress( object ):
except modules.DelayedWorkflowEvaluation:
self.mark_step_outputs_delayed( step )
__all__ = [ invoke, WorkflowRunConfig ]
__all__ = ( 'invoke', 'WorkflowRunConfig' )
lib/galaxy/workflow/schedulers/core.py
+4 -5
View File
@@ -1,14 +1,13 @@
""" The class defines the stock Galaxy workflow scheduling plugin - currently
it simply schedules the whole workflow up front when offered.
"""
from ..schedulers import ActiveWorkflowSchedulingPlugin
import logging
from galaxy.work import context
from galaxy.workflow import run, run_request
from galaxy.workflow import run
from galaxy.workflow import run_request
from ..schedulers import ActiveWorkflowSchedulingPlugin
import logging
log = logging.getLogger( __name__ )
@@ -43,4 +42,4 @@ class CoreWorkflowSchedulingPlugin( ActiveWorkflowSchedulingPlugin ):
workflow_invocation=workflow_invocation,
)
__all__ = [ 'CoreWorkflowSchedulingPlugin' ]
__all__ = ( 'CoreWorkflowSchedulingPlugin', )
static/maps/mvc/tool/tools.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/analysis.bundled.js
+2 -2
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/analysis.bundled.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/libs.bundled.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
static/scripts/mvc/tool/tools.js
+1 -1
View File
File diff suppressed because one or more lines are too long
test/api/helpers.py
+5 -4
View File
@@ -1,10 +1,11 @@
from base import api_asserts
import json
import time
from operator import itemgetter
import time
import json
from six import StringIO
from pkg_resources import resource_string
from six import StringIO
from base import api_asserts
# Simple workflow that takes an input and call cat wrapper on it.
workflow_str = resource_string( __name__, "test_workflow_1.ga" )
test/api/test_api_batch.py
+3 -1
View File
@@ -1,7 +1,9 @@
import json
import pprint
import logging
import pprint
from requests import post
from base import api
log = logging.getLogger( "functional_tests.py" )
test/api/test_authenticate.py
+1 -1
View File
@@ -1,8 +1,8 @@
import base64
from requests import get
from base import api
from base import api
TEST_USER_EMAIL = "auth_user_test@bx.psu.edu"
TEST_USER_PASSWORD = "testpassword1"
test/api/test_dataset_collections.py
+4 -3
View File
@@ -1,7 +1,8 @@
from base import api
import json
from .helpers import DatasetPopulator
from .helpers import DatasetCollectionPopulator
from base import api
from .helpers import DatasetCollectionPopulator, DatasetPopulator
class DatasetCollectionApiTestCase( api.ApiTestCase ):
test/api/test_history_contents.py
+5 -6
View File
@@ -1,11 +1,10 @@
from base import api
import json
from .helpers import TestsDatasets
from .helpers import LibraryPopulator
from .helpers import DatasetCollectionPopulator
from requests import delete
from requests import put
from requests import delete, put
from base import api
from .helpers import DatasetCollectionPopulator, LibraryPopulator, TestsDatasets
# TODO: Test anonymous access.
test/api/test_page_revisions.py
+2 -1
View File
@@ -1,5 +1,6 @@
from galaxy.exceptions import error_codes
from api.test_pages import BasePageApiTestCase
from .test_pages import BasePageApiTestCase
class PageRevisionsApiTestCase( BasePageApiTestCase ):
test/api/test_pages.py
+3 -4
View File
@@ -1,8 +1,7 @@
from galaxy.exceptions import error_codes
from base import api
from requests import delete
from operator import itemgetter
from base import api
from galaxy.exceptions import error_codes
class BasePageApiTestCase( api.ApiTestCase ):
@@ -102,4 +101,4 @@ class PageApiTestCase( BasePageApiTestCase ):
index_response = self._get( "pages" )
self._assert_status_code_is( index_response, 200 )
pages = index_response.json()
return id in map( itemgetter( "id" ), pages )
return id in ( _["id"] for _ in pages )
test/api/test_tool_data.py
+3 -1
View File
@@ -1,9 +1,11 @@
""" Tests for the tool data API.
"""
from __future__ import print_function
from base import api
import operator
from base import api
class ToolDataApiTestCase( api.ApiTestCase ):
test/api/test_tools.py
+10 -11
View File
@@ -1,13 +1,12 @@
# Test tools API.
from base import api
import json
from operator import itemgetter
from .helpers import DatasetPopulator
from .helpers import DatasetCollectionPopulator
from .helpers import LibraryPopulator
from .helpers import skip_without_tool
from base import api
from galaxy.tools.verify.test_data import TestDataResolver
from .helpers import (DatasetCollectionPopulator, DatasetPopulator,
LibraryPopulator, skip_without_tool)
class ToolsTestCase( api.ApiTestCase ):
@@ -25,7 +24,7 @@ class ToolsTestCase( api.ApiTestCase ):
tools_index = index.json()
# No need to flatten out sections, with in_panel=False, only tools are
# returned.
tool_ids = map( itemgetter( "id" ), tools_index )
tool_ids = [_["id"] for _ in tools_index]
assert "upload1" in tool_ids
@skip_without_tool( "cat1" )
@@ -195,7 +194,7 @@ class ToolsTestCase( api.ApiTestCase ):
def get_state(dce):
return dce["object"]["state"]
mixed_states = map(get_state, mixed_hdca["elements"])
mixed_states = [get_state(_) for _ in mixed_hdca["elements"]]
assert mixed_states == [u"ok", u"error", u"ok", u"error"], mixed_states
inputs = {
"input": { "src": "hdca", "id": mixed_hdca["id"] },
@@ -206,7 +205,7 @@ class ToolsTestCase( api.ApiTestCase ):
self.assertEquals( len(filter_output_collections), 1 )
filtered_hid = filter_output_collections[0]["hid"]
filtered_hdca = self.dataset_populator.get_history_collection_details(history_id, hid=filtered_hid, wait=False)
filtered_states = map(get_state, filtered_hdca["elements"])
filtered_states = [get_state(_) for _ in filtered_hdca["elements"]]
assert filtered_states == [u"ok", u"ok"], filtered_states
@skip_without_tool( "multi_select" )
@@ -525,7 +524,7 @@ class ToolsTestCase( api.ApiTestCase ):
# Assert we have three outputs with 1, 2, and 3 lines respectively.
assert len( outputs ) == 3
outputs_contents = [ self.dataset_populator.get_history_dataset_content( history_id, dataset=o ).strip() for o in outputs ]
assert sorted( map( lambda c: len( c.split( "\n" ) ), outputs_contents ) ) == [ 1, 2, 3 ]
assert sorted( len( c.split( "\n" ) ) for c in outputs_contents ) == [ 1, 2, 3 ]
@skip_without_tool( "cat1" )
def test_multirun_in_repeat( self ):
@@ -1230,7 +1229,7 @@ class ToolsTestCase( api.ApiTestCase ):
else:
tools.append( tool_or_section )
tool_ids = map( itemgetter( "id" ), tools )
tool_ids = [_["id"] for _ in tools]
return tool_ids
def __build_nested_list( self, history_id ):
test/api/test_users.py
+2
View File
@@ -1,5 +1,7 @@
import json
from requests import put
from base import api
TEST_USER_EMAIL = "user_for_users_index_test@bx.psu.edu"
test/api/test_workflow_extraction.py
+5 -4
View File
@@ -1,8 +1,9 @@
from __future__ import print_function
from collections import namedtuple
import functools
from json import dumps, loads
import operator
from collections import namedtuple
from json import dumps, loads
from .helpers import skip_without_tool
from .test_workflows import BaseWorkflowsApiTestCase
@@ -215,7 +216,7 @@ test_data:
text_input2: "samp1\t30.0\nsamp2\t40.0\n"
""")
tool_ids = [ "cat1", "collection_split_on_column", "cat_list" ]
job_ids = list(map( functools.partial(self._job_id_for_tool, jobs_summary.jobs ), tool_ids ))
job_ids = [ functools.partial(self._job_id_for_tool, jobs_summary.jobs )(_) for _ in tool_ids ]
downloaded_workflow = self._extract_and_download_workflow(
dataset_ids=[ "1", "2" ],
job_ids=job_ids,
@@ -265,7 +266,7 @@ test_data:
content: "samp1\t30.0\nsamp2\t40.0\n"
""")
tool_ids = [ "cat1", "collection_creates_pair", "cat_collection", "cat_list" ]
job_ids = list(map( functools.partial(self._job_id_for_tool, jobs_summary.jobs ), tool_ids ))
job_ids = [ functools.partial(self._job_id_for_tool, jobs_summary.jobs )(_) for _ in tool_ids ]
downloaded_workflow = self._extract_and_download_workflow(
dataset_collection_ids=[ "3" ],
job_ids=job_ids,
test/api/test_workflows.py
+12 -14
View File
@@ -1,26 +1,24 @@
from __future__ import print_function
import time
import yaml
from json import dumps
from collections import namedtuple
from json import dumps
from uuid import uuid4
import yaml
from requests import delete, put
from base import api
from galaxy.tools.verify.test_data import TestDataResolver
from galaxy.exceptions import error_codes
from .helpers import WorkflowPopulator
from .helpers import DatasetPopulator
from .helpers import DatasetCollectionPopulator
from .helpers import skip_without_tool
from galaxy.tools.verify.test_data import TestDataResolver
from .helpers import (DatasetCollectionPopulator, DatasetPopulator,
skip_without_tool, WorkflowPopulator)
from .workflows_format_2 import (
convert_and_import_workflow,
ImporterGalaxyInterface,
)
from requests import delete
from requests import put
SIMPLE_NESTED_WORKFLOW_YAML = """
class: GalaxyWorkflow
inputs:
@@ -277,7 +275,7 @@ class WorkflowsApiTestCase( BaseWorkflowsApiTestCase ):
workflow = show_response.json()
self._assert_looks_like_instance_workflow_representation( workflow )
assert len(workflow["steps"]) == 3
self.assertEqual(sorted([step["id"] for step in workflow["steps"].values()]), [0, 1, 2])
self.assertEqual(sorted(step["id"] for step in workflow["steps"].values()), [0, 1, 2])
show_response = self._get( "workflows/%s" % workflow_id, {"legacy": True} )
workflow = show_response.json()
@@ -285,7 +283,7 @@ class WorkflowsApiTestCase( BaseWorkflowsApiTestCase ):
assert len(workflow["steps"]) == 3
# Can't reay say what the legacy IDs are but must be greater than 3 because dummy
# workflow was created first in this instance.
self.assertNotEqual(sorted([step["id"] for step in workflow["steps"].values()]), [0, 1, 2])
self.assertNotEqual(sorted(step["id"] for step in workflow["steps"].values()), [0, 1, 2])
def test_show_invalid_key_is_400( self ):
show_response = self._get( "workflows/%s" % self._random_key() )
@@ -457,7 +455,7 @@ class WorkflowsApiTestCase( BaseWorkflowsApiTestCase ):
def get_subworkflow_content_id(workflow_id):
workflow_contents = self._download_workflow(workflow_id, style="editor")
steps = workflow_contents['steps']
subworkflow_step = filter(lambda s: s["type"] == "subworkflow", steps.values())[0]
subworkflow_step = next(s for s in steps.values() if s["type"] == "subworkflow")
return subworkflow_step['content_id']
workflow_id = self._upload_yaml_workflow(SIMPLE_NESTED_WORKFLOW_YAML, publish=True)
@@ -1636,7 +1634,7 @@ steps:
contents_url = "histories/%s/contents" % history
history_contents_response = self._get( contents_url )
self._assert_status_code_is( history_contents_response, 200 )
hda_summary = filter( lambda hc: hc[ "hid" ] == hid, history_contents_response.json() )[ 0 ]
hda_summary = next(hc for hc in history_contents_response.json() if hc[ "hid" ] == hid)
hda_info_response = self._get( "%s/%s" % ( contents_url, hda_summary[ "id" ] ) )
self._assert_status_code_is( hda_info_response, 200 )
self.assertEqual( hda_info_response.json()[ "metadata_data_lines" ], lines )
test/api/workflows_format_2/__init__.py
+3 -3
View File
@@ -1,11 +1,11 @@
""" This module defines the public interface or entry point for the
Format 2 workflow code.
"""
from .main import convert_and_import_workflow
from .interface import ImporterGalaxyInterface
from .main import convert_and_import_workflow
__all__ = [
__all__ = (
'convert_and_import_workflow',
'ImporterGalaxyInterface',
]
)
test/api/workflows_format_2/converter.py
+8 -8
View File
@@ -1,11 +1,11 @@
"""Functionality for converting a Format 2 workflow into a standard Galaxy workflow."""
from __future__ import print_function
from collections import OrderedDict
import json
import os
import sys
import uuid
from collections import OrderedDict
import yaml
@@ -96,7 +96,7 @@ def _python_to_workflow(as_python, conversion_context):
as_python["steps"] = steps_as_dict
steps = steps_as_dict
for step in steps.itervalues():
for step in steps.values():
step_type = step.get("type", None)
if "run" in step:
if step_type is not None:
@@ -118,7 +118,7 @@ def _python_to_workflow(as_python, conversion_context):
run_to_step_function(conversion_context, step, run_action)
del step["run"]
for step in steps.itervalues():
for step in steps.values():
step_type = step.get("type", "tool")
step_type = STEP_TYPE_ALIASES.get(step_type, step_type)
if step_type not in STEP_TYPES:
@@ -326,7 +326,7 @@ def transform_tool(context, step):
return {"__class__": "RuntimeValue"}
if isinstance(value, dict):
new_values = {}
for k, v in value.iteritems():
for k, v in value.items():
new_key = _join_prefix(key, k)
new_values[k] = replace_links(v, new_key)
return new_values
@@ -349,7 +349,7 @@ def transform_tool(context, step):
step_state = step["state"]
step_state = replace_links(step_state)
for key, value in step_state.iteritems():
for key, value in step_state.items():
tool_state[key] = json.dumps(value)
del step["state"]
@@ -470,7 +470,7 @@ def _populate_input_connections(context, step, connect):
input_connections = step["input_connections"]
is_subworkflow_step = step.get("type") == "subworkflow"
for key, values in connect.iteritems():
for key, values in connect.items():
input_connection_value = []
if not isinstance(values, list):
values = [values]
@@ -512,7 +512,7 @@ def main(argv):
if __name__ == "__main__":
main(sys.argv)
__all__ = [
__all__ = (
'yaml_to_workflow',
'python_to_workflow',
]
)
test/api/workflows_format_2/main.py
+2 -2
View File
@@ -36,6 +36,6 @@ def convert_and_import_workflow(has_workflow, **kwds):
import_kwds["publish"] = True
return galaxy_interface.import_workflow(workflow, **import_kwds)
__all__ = [
__all__ = (
'convert_and_import_workflow',
]
)
test/base/api.py
+10 -14
View File
@@ -1,23 +1,19 @@
from contextlib import contextmanager
from six.moves.urllib.parse import urlencode
from .api_asserts import (
assert_error_code_is,
assert_has_keys,
assert_not_has_keys,
assert_status_code_is,
)
from .api_util import get_master_api_key, get_user_api_key
from .interactor import GalaxyInteractorApi as BaseInteractor
# TODO: We don't need all of TwillTestCase, strip down to a common super class
# shared by API and Twill test cases.
from .twilltestcase import TwillTestCase
from base.interactor import GalaxyInteractorApi as BaseInteractor
from .api_util import get_master_api_key
from .api_util import get_user_api_key
from .api_asserts import (
assert_status_code_is,
assert_has_keys,
assert_not_has_keys,
assert_error_code_is,
)
from urllib import urlencode
TEST_USER = "user@bx.psu.edu"
DEFAULT_OTHER_USER = "otheruser@bx.psu.edu" # A second user for API testing.
test/base/driver_util.py
+10 -13
View File
@@ -17,24 +17,21 @@ import nose.config
import nose.core
import nose.loader
import nose.plugins.manager
from paste import httpserver
import requests
from .api_util import get_master_api_key, get_user_api_key
from .tool_shed_util import parse_tool_panel_config
from .nose_util import run
from .instrument import StructuredTestDataPlugin
from paste import httpserver
from functional import database_contexts
from galaxy.app import UniverseApplication as GalaxyUniverseApplication
from galaxy.web import buildapp
from galaxy.webapps.tool_shed.app import UniverseApplication as ToolshedUniverseApplication
from galaxy.util import asbool
from galaxy.util.properties import load_app_properties
from galaxy.web import buildapp
from galaxy.webapps.tool_shed.app import UniverseApplication as ToolshedUniverseApplication
from base.test_logging import logging_config_file
from .api_util import get_master_api_key, get_user_api_key
from .instrument import StructuredTestDataPlugin
from .nose_util import run
from .test_logging import logging_config_file
from .tool_shed_util import parse_tool_panel_config
galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir))
DEFAULT_WEB_HOST = "localhost"
@@ -695,7 +692,7 @@ def drive_test(test_driver_class):
sys.exit(test_driver_class().run())
__all__ = [
__all__ = (
"copy_database_template",
"build_logger",
"drive_test",
@@ -708,4 +705,4 @@ __all__ = [
"setup_galaxy_config",
"TestDriver",
"wait_for_http_server",
]
)

Some files were not shown because too many files have changed in this diff Show More