A bunch more sphinx fixes.

This commit is contained in:
John Chilton
2021-01-12 09:34:28 -05:00
parent ee3280ad52
commit e3ffa759f3
43 changed files with 331 additions and 316 deletions
+18 -17
View File
@@ -26,12 +26,14 @@ class IdentityProvider:
:type config: lxml.etree.ElementTree._Element
:param config: Is the configuration element of the provider
from the configuration file (e.g., oidc_config.xml).
This element contains the all the provider-specific
configuration elements.
from the configuration file (e.g., oidc_config.xml).
This element contains the all the provider-specific
configuration elements.
:type backend_config: lxml.etree.ElementTree._Element
:param backend_config: Is the configuration element of the backend of
:param backend_config:
Is the configuration element of the backend of
the provider from the configuration file (e.g.,
oidc_backends_config.xml). This element contains all the
backend-specific configuration elements.
@@ -47,24 +49,23 @@ class IdentityProvider:
:type trans: GalaxyWebTransaction
:param trans: Galaxy web transaction.
:return: a redirect URI to the provider's authentication
endpoint.
:returns: a redirect URI to the provider's authentication endpoint
"""
raise NotImplementedError()
def callback(self, state_token, authz_code, trans, login_redirect_url):
"""
Handles authentication call-backs from identity providers.
This process maps `state-token` to a user
:type state_token: string
def callback(self, state_token: str, authz_code: str, trans, login_redirect_url):
"""Handles authentication call-backs from identity providers.
This process maps `state-token` to a user.
:param state_token: is an anti-forgery token which identifies
a Galaxy user to whom the given authorization code belongs to.
:type authz_code: string
a Galaxy user to whom the given authorization code belongs to.
:param authz_code: a very short-lived, single-use token to
request a refresh token.
request a refresh token.
:type trans: GalaxyWebTransaction
:param trans: Galaxy web transaction.
:return tuple: a tuple of redirect_url and user.
:rtype: tuple
:returns: a tuple of redirect_url and user.
"""
raise NotImplementedError()
@@ -79,7 +80,7 @@ class IdentityProvider:
:type trans: GalaxyWebTransaction
:param trans: Galaxy web transaction.
:type trans: string
:param trans: Optional URL to redirect to after logging out of IDP.
:type post_logout_redirect_url: string
:param post_logout_redirect_url: Optional URL to redirect to after logging out of IDP.
"""
raise NotImplementedError()
+5 -3
View File
@@ -282,9 +282,10 @@ def contains_required_data(response=None, is_new=False, **kwargs):
and returns void if otherwise.
:type response: dict
:param response: a dictionary containing decoded response from
OIDC backend that contain the following keys
among others:
:param response: a dictionary containing decoded response from
OIDC backend that contain the following keys
among others:
- id_token; see: http://openid.net/specs/openid-connect-core-1_0.html#IDToken
- access_token; see: https://tools.ietf.org/html/rfc6749#section-1.4
- refresh_token; see: https://tools.ietf.org/html/rfc6749#section-1.5
@@ -297,6 +298,7 @@ def contains_required_data(response=None, is_new=False, **kwargs):
:param is_new: has the user been authenticated?
:param kwargs: may contain the following keys among others:
- uid: user ID
- user: Galaxy user; if user is already authenticated
- backend: the backend that is used for user authentication.
+5 -2
View File
@@ -73,12 +73,15 @@ class ContainerVolume(metaclass=ABCMeta):
class Container(metaclass=ABCMeta):
def __init__(self, interface, id, name=None, **kwargs):
""":param interface: Container interface for the given container type
"""
:param interface: Container interface for the given container type
:type interface: :class:`ContainerInterface` subclass instance
:param id: Container identifier
:type id: str
:param name: Container name
:type name: str
"""
self._interface = interface
self._id = id
@@ -111,7 +114,7 @@ class Container(metaclass=ABCMeta):
"""Attribute for accessing details of ports published by the container.
:returns: Port details
:rtype: list of :class:`ContainerPort`s
:rtype: list of :class:`ContainerPort` s
"""
@abstractproperty
+10 -7
View File
@@ -66,10 +66,11 @@ class HasSettings(type):
class DataProvider(metaclass=HasSettings):
"""
Base class for all data providers. Data providers:
(a) have a source (which must be another file-like object)
(b) implement both the iterator and context manager interfaces
(c) do not allow write methods
(but otherwise implement the other file object interface methods)
- have a source (which must be another file-like object)
- implement both the iterator and context manager interfaces
- do not allow write methods (but otherwise implement the other file object interface methods)
"""
# a definition of expected types for keyword arguments sent to __init__
# useful for controlling how query string dictionaries can be parsed into correct types for __init__
@@ -77,10 +78,12 @@ class DataProvider(metaclass=HasSettings):
settings: Dict[str, str] = {}
def __init__(self, source, **kwargs):
"""
"""Sets up a data provider, validates supplied source.
:param source: the source that this iterator will loop over.
(Should implement the iterable interface and ideally have the
context manager interface as well)
(Should implement the iterable interface and ideally have the
context manager interface as well)
"""
self.source = self.validate_source(source)
+17 -18
View File
@@ -44,41 +44,40 @@ class ColumnarDataProvider(line.RegexLineDataProvider):
column_count=None, column_types=None, parsers=None, parse_columns=True,
deliminator='\t', filters=None, **kwargs):
"""
:param indeces: a list of indeces of columns to gather from each row
Optional: will default to `None`.
If `None`, this provider will return all rows (even when a
particular row contains more/less than others).
If a row/line does not contain an element at a given index, the
provider will-return/fill-with a `None` value as the element.
Optional: will default to `None`.
If `None`, this provider will return all rows (even when a
particular row contains more/less than others).
If a row/line does not contain an element at a given index, the
provider will-return/fill-with a `None` value as the element.
:type indeces: list or None
:param column_count: an alternate means of defining indeces, use an int
here to effectively provide the first N columns.
Optional: will default to `None`.
here to effectively provide the first N columns.
Optional: will default to `None`.
:type column_count: int
:param column_types: a list of string names of types that the
provider will use to look up an appropriate parser for the column.
(e.g. 'int', 'float', 'str', 'bool')
Optional: will default to parsing all columns as strings.
provider will use to look up an appropriate parser for the column.
(e.g. 'int', 'float', 'str', 'bool')
Optional: will default to parsing all columns as strings.
:type column_types: list of strings
:param parsers: a dictionary keyed with column type strings
and with values that are functions to use when parsing those
types.
Optional: will default to using the function `_get_default_parsers`.
and with values that are functions to use when parsing those
types.
Optional: will default to using the function `_get_default_parsers`.
:type parsers: dictionary
:param parse_columns: attempt to parse columns?
Optional: defaults to `True`.
:param parse_columns: attempt to parse columns? Optional: defaults to `True`.
:type parse_columns: bool
:param deliminator: character(s) used to split each row/line of the source.
Optional: defaults to the tab character.
:param deliminator: character(s) used to split each row/line of the source. Optional: defaults to the tab character.
:type deliminator: str
.. note:: that the subclass constructors are passed kwargs - so they're
params (limit, offset, etc.) are also applicable here.
params (limit, offset, etc.) are also applicable here.
"""
# TODO: other columnar formats: csv, etc.
super().__init__(source, **kwargs)
+7 -31
View File
@@ -531,6 +531,7 @@ class BigWigDataProvider(base.LimitedOffsetDataProvider):
def __init__(self, source, chrom, start, end, named_columns=False, column_names=None, **kwargs):
"""
:param chrom: which chromosome within the bigbed file to extract data for
:type chrom: str
:param start: the start of the region from which to extract data
@@ -539,15 +540,16 @@ class BigWigDataProvider(base.LimitedOffsetDataProvider):
:type end: int
:param named_columns: optionally return dictionaries keying each column
with 'chrom', 'start', 'end', 'strand', or 'name'.
Optional: defaults to False
with 'chrom', 'start', 'end', 'strand', or 'name'.
Optional: defaults to False
:type named_columns: bool
:param column_names: an ordered list of strings that will be used as the keys
for each column in the returned dictionaries.
The number of key, value pairs each returned dictionary has will
be as short as the number of column names provided.
for each column in the returned dictionaries.
The number of key, value pairs each returned dictionary has will
be as short as the number of column names provided.
:type column_names:
"""
raise NotImplementedError('Work in progress')
# TODO: validate is a wig
@@ -688,32 +690,6 @@ class SamtoolsDataProvider(line.RegexLineDataProvider):
return options_dict, new_kwargs
class BcftoolsDataProvider(line.RegexLineDataProvider):
"""
Data provider that uses an bcftools on a bcf (or vcf?) file as its source.
This can be piped through other providers (column, map, genome region, etc.).
"""
def __init__(self, dataset, **kwargs):
# TODO: as samtools
raise NotImplementedError()
# super(BcftoolsDataProvider, self).__init__(dataset, **kwargs)
class BGzipTabixDataProvider(base.DataProvider):
"""
Data provider that uses an g(un)zip on a file as its source.
This can be piped through other providers (column, map, genome region, etc.).
"""
def __init__(self, dataset, **kwargs):
# TODO: as samtools - need more info on output format
raise NotImplementedError()
# super(BGzipTabixDataProvider, self).__init__(dataset, **kwargs)
class SQliteDataProvider(base.DataProvider):
"""
Data provider that uses a sqlite database file as its source.
+50 -24
View File
@@ -357,6 +357,7 @@ class LowerTriangleDistanceMatrix(DistanceMatrix):
Determines whether the file is a lower-triangle distance matrix (phylip) format
The first line has the number of sequences in the matrix.
The remaining lines have the sequence name followed by a list of distances from all preceeding sequences
5 # possibly but not always preceded by a tab :/
U68589
U68590 0.3371
@@ -422,6 +423,7 @@ class SquareDistanceMatrix(DistanceMatrix):
The first line has the number of sequences in the matrix.
The following lines have the sequence name in the first column plus a column for the distance to each sequence
in the row order in which they appear in the matrix.
3
U68589 0.0000 0.3371 0.3610
U68590 0.3371 0.0000 0.3783
@@ -623,11 +625,14 @@ class Frequency(Tabular):
def sniff_prefix(self, file_prefix):
"""
Determines whether the file is a frequency tabular format for chimera analysis
#1.14.0
0 0.000
1 0.000
...
155 0.975
.. code-block::
#1.14.0
0 0.000
1 0.000
...
155 0.975
>>> from galaxy.datatypes.sniff import get_test_fname
>>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.freq' )
@@ -685,10 +690,13 @@ class Quantile(Tabular):
def sniff_prefix(self, file_prefix):
"""
Determines whether the file is a quantiles tabular format for chimera analysis
1 0 0 0 0 0 0
2 0.309198 0.309198 0.37161 0.37161 0.37161 0.37161
3 0.510982 0.563213 0.693529 0.858939 1.07442 1.20608
...
.. code-block::
1 0 0 0 0 0 0
2 0.309198 0.309198 0.37161 0.37161 0.37161 0.37161
3 0.510982 0.563213 0.693529 0.858939 1.07442 1.20608
...
>>> from galaxy.datatypes.sniff import get_test_fname
>>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.quan' )
@@ -805,17 +813,26 @@ class RefTaxonomy(Tabular):
http://www.mothur.org/wiki/Taxonomy_outline
A table with 2 or 3 columns:
- SequenceName
- Taxonomy (semicolon-separated taxonomy in descending order)
- integer ?
Example: 2-column (http://www.mothur.org/wiki/Taxonomy_outline)
X56533.1 Eukaryota;Alveolata;Ciliophora;Intramacronucleata;Oligohymenophorea;Hymenostomatida;Tetrahymenina;Glaucomidae;Glaucoma;
X97975.1 Eukaryota;Parabasalidea;Trichomonada;Trichomonadida;unclassified_Trichomonadida;
AF052717.1 Eukaryota;Parabasalidea;
.. code-block::
X56533.1 Eukaryota;Alveolata;Ciliophora;Intramacronucleata;Oligohymenophorea;Hymenostomatida;Tetrahymenina;Glaucomidae;Glaucoma;
X97975.1 Eukaryota;Parabasalidea;Trichomonada;Trichomonadida;unclassified_Trichomonadida;
AF052717.1 Eukaryota;Parabasalidea;
Example: 3-column (http://vamps.mbl.edu/resources/databases.php)
v3_AA008 Bacteria;Firmicutes;Bacilli;Lactobacillales;Streptococcaceae;Streptococcus 5
v3_AA016 Bacteria 120
v3_AA019 Archaea;Crenarchaeota;Marine_Group_I 1
.. code-block::
v3_AA008 Bacteria;Firmicutes;Bacilli;Lactobacillales;Streptococcaceae;Streptococcus 5
v3_AA016 Bacteria 120
v3_AA019 Archaea;Crenarchaeota;Marine_Group_I 1
>>> from galaxy.datatypes.sniff import get_test_fname
>>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.ref.taxonomy' )
@@ -882,16 +899,19 @@ class Axes(Tabular):
Determines whether the file is an axes format
The first line may have column headings.
The following lines have the name in the first column plus float columns for each axis.
==> 98_sq_phylip_amazon.fn.unique.pca.axes <==
group axis1 axis2
forest 0.000000 0.145743
pasture 0.145743 0.000000
==> 98_sq_phylip_amazon.nmds.axes <==
axis1 axis2
U68589 0.262608 -0.077498
U68590 0.027118 0.195197
U68591 0.329854 0.014395
.. code-block::
group axis1 axis2
forest 0.000000 0.145743
pasture 0.145743 0.000000
.. code-block::
axis1 axis2
U68589 0.262608 -0.077498
U68590 0.027118 0.195197
U68591 0.329854 0.014395
>>> from galaxy.datatypes.sniff import get_test_fname
>>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.axes' )
@@ -943,14 +963,20 @@ class SffFlow(Tabular):
https://mothur.org/wiki/flow_file/
The first line is the total number of flow values - 800 for Titanium data. For GS FLX it would be 400.
Following lines contain:
- SequenceName
- the number of useable flows as defined by 454's software
- the flow intensity for each base going in the order of TACG.
Example:
.. code-block::
800
GQY1XT001CQL4K 85 1.04 0.00 1.00 0.02 0.03 1.02 0.05 ...
GQY1XT001CQIRF 84 1.02 0.06 0.98 0.06 0.09 1.05 0.07 ...
GQY1XT001CF5YW 88 1.02 0.02 1.01 0.04 0.06 1.02 0.03 ...
"""
def __init__(self, **kwd):
+13 -11
View File
@@ -1238,18 +1238,20 @@ class MatrixMarket(TabularData):
suitable for representing sparse matrices. Only nonzero entries need
be encoded, and the coordinates of each are given explicitly.
The tabular file format is defined as follows::
The tabular file format is defined as follows:
%%MatrixMarket matrix coordinate real general <--- header line
% <--+
% comments |-- 0 or more comment lines
% <--+
M N L <--- rows, columns, entries
I1 J1 A(I1, J1) <--+
I2 J2 A(I2, J2) |
I3 J3 A(I3, J3) |-- L lines
. . . |
IL JL A(IL, JL) <--+
.. code-block::
%%MatrixMarket matrix coordinate real general <--- header line
% <--+
% comments |-- 0 or more comment lines
% <--+
M N L <--- rows, columns, entries
I1 J1 A(I1, J1) <--+
I2 J2 A(I2, J2) |
I3 J3 A(I3, J3) |-- L lines
. . . |
IL JL A(IL, JL) <--+
Indices are 1-based, i.e. A(1,1) is the first element.
+65 -99
View File
@@ -83,21 +83,17 @@ class RuleValidator:
"""
@classmethod
def validate_rule(cls, rule_type, app, return_bool=False, *args, **kwargs):
def validate_rule(cls, rule_type: str, app, return_bool: bool = False, *args, **kwargs):
"""
This function is responsible for passing each rule to its relevant
function.
@type rule_type: str
@param rule_type: the current rule's type
@type return_bool: bool
@param return_bool: True when we are only interested in the result of
:param rule_type: the current rule's type
:param return_bool: True when we are only interested in the result of
the validation, and not the validated rule itself.
@rtype: bool, dict (depending on return_bool)
@return: validated rule or result of validation (depending on
return_bool)
:rtype: bool, dict (depending on return_bool)
:returns: validated rule or result of validation (depending on
return_bool)
"""
if rule_type == 'file_size':
return cls.__validate_file_size_rule(app, return_bool, *args, **kwargs)
@@ -380,30 +376,20 @@ class RuleValidator:
return valid_rule, rule
@classmethod
def __validate_destination(cls, valid_rule, app, return_bool, rule, tool, counter):
def __validate_destination(cls, valid_rule: bool, app, return_bool: bool, rule: dict, tool: str, counter: int):
"""
This function is responsible for validating destination.
@type valid_rule: bool
@param valid_rule: returns True if everything is valid. False if it
:param valid_rule: returns True if everything is valid. False if it
encounters any abnormalities in the config.
@type return_bool: bool
@param return_bool: True when we are only interested in the result of
:param return_bool: True when we are only interested in the result of
the validation, and not the validated rule itself.
@type rule: dict
@param rule: contains the original received rule
@type tool: str
@param tool: the name of the current tool. Necessary for log output.
@type counter: int
@param counter: this counter is used to identify what rule # is
:param rule: contains the original received rule
:param tool: the name of the current tool. Necessary for log output.
:param counter: this counter is used to identify what rule # is
currently being validated. Necessary for log output.
@rtype: bool, dict (tuple)
@return: validated rule and result of validation
:rtype: bool, dict (tuple)
:returns: validated rule and result of validation
"""
if "fail_message" in rule:
@@ -698,27 +684,20 @@ class RuleValidator:
return valid_rule, rule
def parse_yaml(path="/config/tool_destinations.yml",
job_conf_path="/config/job_conf.xml", app=None, test=False,
return_bool=False):
def parse_yaml(path: str = "/config/tool_destinations.yml",
job_conf_path: str = "/config/job_conf.xml", app=None, test: bool = False,
return_bool: bool = False):
"""
Get a yaml file from path and send it to validate_config for validation.
@type path: str
@param path: the path to the tool destinations config file
@type job_conf_path: str
@param job_conf_path: the path to the job config file
@type test: bool
@param test: indicates whether to run in test mode or production mode
@type return_bool: bool
@param return_bool: True when we are only interested in the result of the
:param path: the path to the tool destinations config file
:param job_conf_path: the path to the job config file
:param test: indicates whether to run in test mode or production mode
:type return_bool: bool
:param return_bool: True when we are only interested in the result of the
validation, and not the validated rule itself.
@rtype: bool, dict (depending on return_bool)
@return: validated rule or result of validation (depending on return_bool)
:rtype: bool, dict (depending on return_bool)
:returns: validated rule or result of validation (depending on return_bool)
"""
@@ -754,7 +733,7 @@ def parse_yaml(path="/config/tool_destinations.yml",
config = validate_config(config, app)
except MalformedYMLException:
if verbose:
log.error(str(sys.exc_value))
log.exception("Failed to parse YAML for dynamic job destination")
raise
except ScannerError:
if verbose:
@@ -768,34 +747,26 @@ def parse_yaml(path="/config/tool_destinations.yml",
return config
def validate_destination(app, destination, err_message, err_message_contents,
return_bool=True):
def validate_destination(app, destination: str, err_message: str, err_message_contents,
return_bool: bool = True):
"""
Validate received destination id.
@type app:
@param app: Current app
@type destination: str
@param destination: string containing the destination id that is being
:param app: Current app
:param destination: string containing the destination id that is being
validated
@type err_message: str
@param err_message: Error message to be formatted with the contents of
:param err_message: Error message to be formatted with the contents of
`err_message_contents` upon the event of invalid
destination
@type err_message_contents: tuple
@param err_message_contents: A tuple of strings to be placed in
`err_message`
@type return_bool: bool
@param return_bool: Whether or not the calling function has been told to
:type err_message_contents: tuple
:param err_message_contents: A tuple of strings to be placed in
``err_message``
:param return_bool: Whether or not the calling function has been told to
return a boolean value or not. Determines whether or
not to print 'Ignoring...' after error messages.
@rtype: bool
@return: True if the destination is valid and False otherwise.
:rtype: bool
:returns: True if the destination is valid and False otherwise.
"""
valid_destination = False
@@ -824,19 +795,15 @@ def validate_destination(app, destination, err_message, err_message_contents,
return valid_destination
def validate_config(obj, app=None, return_bool=False,):
def validate_config(obj: dict, app=None, return_bool: bool = False):
"""
Validate received config.
@type obj: dict
@param obj: the entire contents of the config
@type return_bool: bool
@param return_bool: True when we are only interested in the result of the
validation, and not the validated rule itself.
@rtype: bool, dict (depending on return_bool)
@return: validated rule or result of validation (depending on return_bool)
:param obj: the entire contents of the config
:param return_bool: True when we are only interested in the result of the
validation, and not the validated rule itself.
:rtype: bool, dict (depending on return_bool)
:returns: validated rule or result of validation (depending on return_bool)
"""
global priority_list
@@ -1625,21 +1592,22 @@ def map_tool_to_destination(
return destination
def get_destination_list_from_job_config(job_config_location):
def get_destination_list_from_job_config(job_config_location) -> set:
"""
returns A list of all destination IDs declared in the job configuration
@type job_config_location: str
@param job_config_location: The location of the job config file relative
to the galaxy root directory. If NoneType, defaults to
galaxy/config/job_conf.xml,
galaxy/config/job_conf.xml.sample_advanced, or
galaxy/config/job_conf.xml.sample_basic
(first one that exists)
:type job_config_location: str
:param job_config_location:
@rtype: list
@return: A list of all of the destination IDs declared in the job
configuration file.
The location of the job config file relative
to the galaxy root directory. If NoneType, defaults to
galaxy/config/job_conf.xml,
galaxy/config/job_conf.xml.sample_advanced, or
galaxy/config/job_conf.xml.sample_basic
(first one that exists)
:return: A list of all of the destination IDs declared in the job
configuration file.
"""
global destination_list
@@ -1751,20 +1719,18 @@ def get_typo_correction(typo_str, word_set, max_dist):
the same when case is not considered. If there are no
appropriate matches, nothing is returned instead.
@type typo_str: str
@param typo_str: The string to be compared
:type typo_str: str
:param typo_str: The string to be compared
:type word_set: set of str
:param word_set: The set of strings to compare to
:type max_dist: int
:param max_dist: the largest allowed edit distance between
the word and the result. If nothing is
within this range, nothing is returned
@type word_set: set of str
@param word_set: The set of strings to compare to
@type max_dist: int
@param max_dist: the largest allowed edit distance between
the word and the result. If nothing is
within this range, nothing is returned
@rtype: str or NoneType
@return: The closest matching string, or None, if no strings
being compared to are within max_dist edit distance.
:rtype: str or NoneType
:returns: The closest matching string, or None, if no strings
being compared to are within max_dist edit distance.
"""
# Start curr_best out as the largest
+2 -2
View File
@@ -54,7 +54,7 @@ class JobManager:
Due to the nature of some handler assignment methods which are wholly DB-based, the enqueue method will flush
the job. Callers who create the job typically should not flush the job before handing it off to ``enqueue()``.
If a job handler cannot be assigned, :exception:`ToolExecutionError` is raised.
If a job handler cannot be assigned, py:class:`ToolExecutionError` is raised.
:param job: Job to enqueue.
:type job: Instance of :class:`galaxy.model.Job`.
@@ -62,7 +62,7 @@ class JobManager:
:type tool: Instance of :class:`galaxy.tools.Tool`.
:raises ToolExecutionError: if a handler was unable to be assigned.
returns: str or None -- Handler ID, tag, or pool assigned to the job.
:returns: str or None -- Handler ID, tag, or pool assigned to the job.
"""
tool_id = None
configured_handler = None
+4
View File
@@ -261,13 +261,17 @@ class DRMAAJobRunner(AsynchronousJobRunner):
that could happen in this process. to be called from check_watched_items()
returns the state or None if exceptions occurred
in the latter case the job is appended to new_watched if a
1 drmaa.InternalException,
2 drmaa.InvalidJobExceptionnot, or
3 drmaa.DrmCommunicationException occurred
(which causes the job to be tested again in the next iteration of check_watched_items)
- the job is finished as errored if any other exception occurs
- the job is finished OK or errored after the maximum number of retries
depending on the exception
Note that None is returned in all cases where the loop in check_watched_items
is to be continued
"""
+5 -1
View File
@@ -4,9 +4,12 @@ Job control via the DRMAA API / qstat and qacct.
Known to work on the UNIVA grid engine.
known bugs/problems:
- if a job runs longer than the time limits of the queue two things happen
1. at the soft limit (s_rt) SIGUSR1 is sent to the job
2. at the hard limit (h_rt) SIGKILL is sent to the job
The second case is covered by the runner -- it's the same mechanism that
kills a job when the job time limit is reached. The first case is currently
not covered. The good thing is that most programs ignore SIGUSR1.
@@ -14,11 +17,12 @@ known bugs/problems:
at random.
Possible solutions:
- configure the job destinations such that the queue limits are never reached.
the time limits can be determined with
`for i in `qconf -sql`; do qconf -sq $i | grep -E '(h|s)_rt'; done`
- extend the job runner to cover s_rt cases (some ideas):
* I'm unsure if the programs reaction to SIGUSR1 can be determined easily
because it depends on the program. It seems that exit code is in most
cases 10.
+2 -2
View File
@@ -61,11 +61,11 @@ class CloudManager(sharable.SharableModelManager):
:type provider: string
:param provider: the name of cloud-based resource provided. A list of supported providers is given in
`SUPPORTED_PROVIDERS` variable.
`SUPPORTED_PROVIDERS` variable.
:type credentials: dict
:param credentials: a dictionary containing all the credentials required to authenticated to the
specified provider.
specified provider.
:rtype: provider specific, e.g., `cloudbridge.cloud.providers.aws.provider.AWSCloudProvider` for AWS.
:return: a cloudbridge connection to the specified provider.
+1 -1
View File
@@ -16,7 +16,7 @@ class RBACPermissionFailedException(galaxy.exceptions.InsufficientPermissionsExc
class RBACPermission:
"""
Base class for wrangling/controlling the permissions ORM models (*Permissions, Roles)
Base class for wrangling/controlling the permissions ORM models (Permissions, Roles)
that control which users can perform certain actions on their associated models
(Libraries, Datasets).
"""
+4 -3
View File
@@ -132,12 +132,13 @@ class MetadataCollection(Mapping):
def element_is_set(self, name):
"""
check if the meta data with the given name is set, i.e.
- if the such a metadata actually exists and
- if its value differs from no_value
param name the name of the metadata element
return True if the value differes from the no_value
False if its equal of if no metadata with the name is specified
:param name: the name of the metadata element
:returns: True if the value differes from the no_value
False if its equal of if no metadata with the name is specified
"""
try:
meta_val = self.parent._metadata[name]
+2 -2
View File
@@ -112,14 +112,14 @@ class NavigatesGalaxy(HasDriver):
In most cases, methods for interacting with Galaxy components that appear in
multiple tests or applications should be refactored into this class for now.
Keep in mind that this class is used outside the context of ``TestCase``s as
Keep in mind that this class is used outside the context of ``TestCase`` s as
well - so some methods more explicitly related to test data or assertion checking
may make more sense in SeleniumTestCase for instance.
Some day this class will likely be split up into smaller mixins for particular
components of Galaxy, but until that day the best practice is to prefix methods
for driving or querying the interface with the name of the component or page
the method operates on. These serve as psedu-namespaces until we decompose this
the method operates on. These serve as pseudo-namespaces until we decompose this
class. For instance, the method for clicking an option in the workflow editor is
workflow_editor_click_option instead of click_workflow_editor_option.
"""
+6 -10
View File
@@ -62,13 +62,11 @@ def sizzle_presence_of_selector(selector):
return ec
def find_element_by_sizzle(driver, sizzle_selector):
def find_element_by_sizzle(driver, sizzle_selector: str):
"""
Finds an element by sizzle selector.
:Args:
- sizzle_selector: The sizzle selector to use when finding element.
:Usage:
driver.find_element_by_sizzle('#foo')
:param sizzle_selector: The sizzle selector to use when finding element.
"""
elements = driver.find_elements_by_sizzle(sizzle_selector)
if elements:
@@ -79,13 +77,11 @@ def find_element_by_sizzle(driver, sizzle_selector):
)
def find_elements_by_sizzle(driver, sizzle_selector):
def find_elements_by_sizzle(driver, sizzle_selector: str):
"""
Finds elements by sizzle selector.
:Args:
- sizzle_selector: The sizzle selector to use when finding elements.
:Usage:
driver.find_elements_by_sizzle('.foo')
:param sizzle_selector: The sizzle selector to use when finding elements.
"""
if not _is_sizzle_loaded(driver):
_inject_sizzle(driver, SIZZLE_URL, SIZZLE_LOAD_TIMEOUT)
@@ -168,11 +168,15 @@ class ToolPanelManager:
The intent is to call this method for every tool config in a repository and
append each of these as entries to a tool panel dictionary for the repository.
This enables each tool to be loaded into a different section in the tool panel.
{<Tool guid> :
[{ tool_config : <tool_config_file>,
id: <ToolSection id>,
version : <ToolSection version>,
name : <TooSection name>}]}
.. code-block::
{<Tool guid> :
[{ tool_config : <tool_config_file>,
id: <ToolSection id>,
version : <ToolSection version>,
name : <TooSection name>}]}
"""
tool_panel_dict = {}
file_name = strip_path(tool_config)
@@ -162,7 +162,7 @@ def open_recipe_file(file, recipes_path=None, github_repo='bioconda/bioconda-rec
def get_alternative_versions(filepath, filename, recipes_path=None, github_repo='bioconda/bioconda-recipes'):
"""
Return files that match 'filepath/*/filename' in the bioconda-recipes repository
Return files that match ``filepath/*/filename`` in the bioconda-recipes repository
"""
if recipes_path:
return [n.replace('%s/' % recipes_path, '') for n in glob(f'{recipes_path}/{filepath}/*/{filename}')]
@@ -3,13 +3,15 @@ Backup resolvers for when dependencies can not be loaded from the database.
Mainly suited for testing stage.
Ideally all dependencies will be stored in the database
when a tool is added from a Tool Shed.
when a tool is added from a Tool Shed.
That should remain the preferred way of locating dependencies.
In cases where that is not possible
for example during testing this resolver can act as a backup.
for example during testing this resolver can act as a backup.
This resolver looks not just for manually added dependencies
but also ones added from a Tool Shed.
but also ones added from a Tool Shed.
This tool is still under development so the default behaviour could change.
It has been tested when placed in the same directory as galaxy_packages.py
+12 -10
View File
@@ -25,7 +25,7 @@ class ToolOutput(ToolOutputBase):
Represents an output datasets produced by a tool. For backward
compatibility this behaves as if it were the tuple::
(format, metadata_source, parent)
(format, metadata_source, parent)
"""
dict_collection_visible_keys = ['name', 'format', 'label', 'hidden', 'output_type', 'format_source',
@@ -123,15 +123,17 @@ class ToolOutputCollection(ToolOutputBase):
Represents a HistoryDatasetCollectionAssociation of output datasets produced
by a tool.
<outputs>
<collection type="list" label="${tool.name} on ${on_string} fasta">
<discover_datasets pattern="__name__" ext="fasta" visible="True" directory="outputFiles" />
</collection>
<collection type="paired" label="${tool.name} on ${on_string} paired reads">
<data name="forward" format="fastqsanger" />
<data name="reverse" format="fastqsanger"/>
</collection>
<outputs>
.. code-block::
<outputs>
<collection type="list" label="${tool.name} on ${on_string} fasta">
<discover_datasets pattern="__name__" ext="fasta" visible="True" directory="outputFiles" />
</collection>
<collection type="paired" label="${tool.name} on ${on_string} paired reads">
<data name="forward" format="fastqsanger" />
<data name="reverse" format="fastqsanger"/>
</collection>
<outputs>
"""
dict_collection_visible_keys = ['name', 'format', 'label', 'hidden', 'output_type', 'default_format',
'default_format_source', 'default_metadata_source', 'inherit_format', 'inherit_metadata']
+2
View File
@@ -443,6 +443,7 @@ class XmlToolSource(ToolSource):
parse error handling from command and stdio tag
returns list of exit codes, list of regexes
- exit_codes contain all non-zero exit codes (:-1 and 1:) if
detect_errors is default (if not legacy), exit_code, or aggressive
- the oom_exit_code if given and detect_errors is exit_code
@@ -451,6 +452,7 @@ class XmlToolSource(ToolSource):
and exit codes derived from the properties of the command tag.
thus more specific regexes of the same or more severe error level
are triggered first.
"""
command_el = self._command_el
+8 -7
View File
@@ -1117,7 +1117,7 @@ class Tool(Dictifiable):
def parse_inputs(self, tool_source):
"""
Parse the "<inputs>" element and create appropriate `ToolParameter`s.
Parse the "<inputs>" element and create appropriate `ToolParameter` s.
This implementation supports multiple pages and grouping constructs.
"""
# Load parameters (optional)
@@ -2728,17 +2728,18 @@ class DataManagerTool(OutputParameterJSONTool):
history = None
return history
def allow_user_access(self, user, attempting_access=True):
"""
def allow_user_access(self, user, attempting_access=True) -> bool:
"""Check user access to this tool.
:param user: model object representing user.
:type user: galaxy.model.User
:param attempting_access: is the user attempting to do something with the
the tool (set false for incidental checks like toolbox
listing)
the tool (set false for incidental checks like toolbox
listing)
:type attempting_access: bool
:returns: bool -- Whether the user is allowed to access the tool.
Data Manager tools are only accessible to admins.
:returns: Whether the user is allowed to access the tool.
Data Manager tools are only accessible to admins.
"""
if super().allow_user_access(user) and self.app.config.is_admin_user(user):
return True
+7 -5
View File
@@ -785,11 +785,13 @@ class DefaultToolAction:
"""
This allows to map names of input files to metadata default values. Example:
<data format="tabular" name="output" label="Tabular output, aggregates data from individual_inputs" >
<actions>
<action name="column_names" type="metadata" default="${','.join(input.name for input in $individual_inputs)}" />
</actions>
</data>
.. code-block::
<data format="tabular" name="output" label="Tabular output, aggregates data from individual_inputs" >
<actions>
<action name="column_names" type="metadata" default="${','.join(input.name for input in $individual_inputs)}" />
</actions>
</data>
"""
if output.actions:
for action in output.actions.actions:
+13 -8
View File
@@ -315,7 +315,9 @@ class ToolDataTable:
class TabularToolDataTable(ToolDataTable, Dictifiable):
"""
Data stored in a tabular / separated value format on disk, allows multiple
files to be merged but all must have the same column definitions::
files to be merged but all must have the same column definitions:
.. code-block:: xml
<table type="tabular" name="test">
<column name='...' index = '...' />
@@ -798,13 +800,16 @@ class TabularToolDataField(Dictifiable):
class RefgenieToolDataTable(TabularToolDataTable):
"""
Data stored in refgenie
<table name="all_fasta" type="refgenie" asset="fasta" >
<file path="refgenie.yml" />
<field name="value" template="true">${__REFGENIE_UUID__}</field>
<field name="dbkey" template="true">${__REFGENIE_GENOME__}</field>
<field name="name" template="true">${__REFGENIE_DISPLAY_NAME__}</field>
<field name="path" template="true">${__REFGENIE_ASSET__}</field>
</table>
.. code-block:: xml
<table name="all_fasta" type="refgenie" asset="fasta" >
<file path="refgenie.yml" />
<field name="value" template="true">${__REFGENIE_UUID__}</field>
<field name="dbkey" template="true">${__REFGENIE_GENOME__}</field>
<field name="name" template="true">${__REFGENIE_DISPLAY_NAME__}</field>
<field name="path" template="true">${__REFGENIE_ASSET__}</field>
</table>
"""
dict_collection_visible_keys = ['name']
+1
View File
@@ -142,6 +142,7 @@ class NicerConfigParser(ConfigParser):
self._interpolation = self.InterpolateWrapper(self._interpolation)
read_file = getattr(ConfigParser, 'read_file', ConfigParser.readfp)
read_file.__doc__ = ""
def defaults(self):
"""Return the defaults, with their values interpolated (with the
@@ -6,8 +6,11 @@ from galaxy.datatypes.tabular import Tabular
class BaseDataProvider:
"""
Base class for data providers. Data providers (a) read and package data from datasets;
and (b) write subsets of data to new datasets.
Base class for data providers. Data providers both:
- read and package data from datasets
- write subsets of data to new datasets
"""
def __init__(self, converted_dataset=None, original_dataset=None, dependencies=None,
@@ -586,6 +586,7 @@ class VcfDataProvider(GenomeDataProvider):
Payload format: An array of entries for each locus in the file. Each array
has the following entries:
1. GUID (unused)
2. location (0-based)
3. reference base(s)
@@ -594,7 +595,8 @@ class VcfDataProvider(GenomeDataProvider):
6. whether variant passed filter
7. sample genotypes -- a single string with samples separated by commas; empty string
denotes the reference genotype
8-end: allele counts for each alternative
8. allele counts for each alternative
"""
col_name_data_attr_mapping: Dict[Union[str, int], Dict] = {'Qual': {'index': 6, 'name': 'Qual'}}
@@ -164,7 +164,7 @@ class VisualizationsConfigParser:
def parse_entry_point(self, xml_tree):
"""
Parse the config file for an appropriate entry point: a mako template, a script tag,
or an html file, returning as dictionary with: `type`, `file`, and `attr`ibutes of
or an html file, returning as dictionary with: ``type``, ``file``, and ``attr`` (-ibutes) of
the element.
"""
# (older) mako-only syntax: the template to use in rendering the visualization
@@ -510,7 +510,7 @@ class InteractiveEnvironmentRequest:
:type env_override: dict
:param env_override: dictionary of environment variables to add.
:type volumes: list of :class:`galaxy.containers.docker_model.DockerVolume`s
:type volumes: list of py:class:`galaxy.containers.docker_model.DockerVolume` s
:param volumes: dictionary of docker volume mounts
"""
+2 -2
View File
@@ -25,10 +25,10 @@ from .framework.decorators import (
require_login,
)
__all__ = ('FormBuilder', 'do_not_cache', 'error', 'expose', 'expose_api',
__all__ = ('do_not_cache', 'error', 'expose', 'expose_api',
'expose_api_anonymous', 'expose_api_anonymous_and_sessionless',
'expose_api_raw', 'expose_api_raw_anonymous',
'expose_api_raw_anonymous_and_sessionless', 'form',
'expose_api_raw_anonymous_and_sessionless',
'format_return_as_json', 'httpexceptions', 'json', 'json_pretty',
'legacy_expose_api', 'legacy_expose_api_anonymous',
'legacy_expose_api_raw', 'legacy_expose_api_raw_anonymous',
+12 -8
View File
@@ -41,16 +41,20 @@ class BatchMiddleware:
"""
Adds a URL endpoint for processing batch API calls formatted as a JSON
array of JSON dictionaries. These dictionaries are in the form:
[
{
"url": "/api/histories",
"type": "POST",
"body": "{ \"name\": \"New History Name\" }"
},
...
]
.. code-block: json
[
{
"url": "/api/histories",
"type": "POST",
"body": "{ \"name\": \"New History Name\" }"
},
...
]
where:
* `url` is the url for the API call to be made including any query string
* `type` is the HTTP method used (e.g. 'POST', 'PUT') - defaults to 'GET'
* `body` is the text body of the request (optional)
+1 -1
View File
@@ -434,7 +434,7 @@ class ConfiguresHandlers:
def assign_handler(self, obj, configured=None, **kwargs):
"""Set a job handler, flush obj
Called assignment methods should raise :exception:`HandlerAssignmentSkip` to indicate that the next method
Called assignment methods should raise py:class:`HandlerAssignmentSkip` to indicate that the next method
should be tried.
:param obj: Object to assign a handler to (must be a model object with ``handler`` attribute and
+9 -10
View File
@@ -2,6 +2,7 @@
Contains functionality needed in every web interface
"""
import logging
from typing import Optional
from sqlalchemy import true
from webob.exc import (
@@ -338,19 +339,17 @@ class JSAppLauncher(BaseUIController):
log.exception(exc)
return {}
def template(self, trans, app_name, entry_fn='app', options=None, bootstrapped_data=None, masthead=True, **additional_options):
def template(self, trans, app_name: str, entry_fn: str = 'app', options=None, bootstrapped_data: Optional[dict] = None, masthead: Optional[bool] = True, **additional_options):
"""
Render and return the single page mako template that starts the app.
`app_name` (string): the first portion of the webpack bundle to as the app.
`entry_fn` (string): the name of the window-scope function that starts the
app. Defaults to 'app'.
`bootstrapped_data` (dict): (optional) update containing any more data
the app may need.
`masthead` (boolean): (optional, default=True) include masthead elements in
the initial page dom.
`additional_options` (kwargs): update to the options sent to the app.
:param app_name: the first portion of the webpack bundle to as the app.
:param entry_fn: the name of the window-scope function that starts the
app. Defaults to 'app'.
:param bootstrapped_data: update containing any more data
the app may need.
:param masthead: include masthead elements in the initial page dom.
:param additional_options: update to the options sent to the app.
"""
options = options or self._get_js_options(trans)
options.update(additional_options)
+4 -3
View File
@@ -475,9 +475,10 @@ class HistoriesController(BaseAPIController, ExportsHistoryMixin, ImportsHistory
@expose_api
def index_exports(self, trans, id):
"""
index_exports(self, trans, id)
* GET /api/histories/{id}/exports:
Get history exports.
GET /api/histories/{id}/exports
Get previous history exports (to links). Effectively returns serialized
JEHA objects.
"""
return self.history_export_view.get_exports(trans, id)
+1 -1
View File
@@ -36,7 +36,7 @@ def wrap_if_allowed_or_fail(app, stack, wrap, name=None, args=None, kwargs=None)
Arguments are the same as for :func:`wrap_if_allowed`.
Raises :exception:`MiddlewareWrapUnsupported` if the stack does not allow the middleware.
Raises py:class:`MiddlewareWrapUnsupported` if the stack does not allow the middleware.
"""
name = name or wrap.__name__
if not stack.allowed_middleware(wrap):
+2 -2
View File
@@ -44,8 +44,8 @@ def order_workflow_steps(steps):
def edgelist_for_workflow_steps(steps):
"""
Create a list of tuples representing edges between ``WorkflowSteps`` based
on associated ``WorkflowStepConnection``s
Create a list of tuples representing edges between ``WorkflowStep`` s based
on associated ``WorkflowStepConnection`` s
"""
edges = []
steps_to_index = {step: i for i, step in enumerate(steps)}
+2 -2
View File
@@ -109,7 +109,7 @@ def get_latest_downloadable_repository_metadata(trans, repository):
"""
Return the latest downloadable repository_metadata record for the received repository. This will
return repositories of type unrestricted as well as types repository_suite_definition and
tool_dependency_definition.
tool_dependency_definition.
"""
encoded_repository_id = trans.security.encode_id(repository.id)
repo = repository.hg_repo
@@ -148,7 +148,7 @@ def get_latest_repository_metadata(trans, repository):
"""
Return the latest repository_metadata record for the received repository if it exists. This will
return repositories of type unrestricted as well as types repository_suite_definition and
tool_dependency_definition.
tool_dependency_definition.
"""
encoded_repository_id = trans.security.encode_id(repository.id)
repo = repository.hg_repo
@@ -22,6 +22,7 @@ bwa_tool_dependency_repository_long_description = "NT space mapping with BWA"
'''
Simple repository dependencies:
1. Create and populate column_maker_0440 so that it has an installable revision 0.
2. Create and populate convert_chars_0440 so that it has an installable revision 0.
3. Add a valid simple repository_dependencies.xml to convert_chars_0440 that points to the installable revision of column_maker_0440.
@@ -29,6 +30,7 @@ Simple repository dependencies:
5. Delete repository_dependencies.xml from convert_chars_0440, and make sure convert_chars_0440 now has two installable revisions: 1 and 2
Complex repository dependencies:
1. Create and populate bwa_package_0440 so that it has a valid tool dependency definition and an installable revision 0.
2. Create and populate bwa_base_0440 so that it has an installable revision 0.
3. Add a valid complex repository dependency tool_dependencies.xml to bwa_base_0440 that points to the installable revision 0 of bwa_package_0440.
@@ -36,11 +38,13 @@ Complex repository dependencies:
5. Delete tool_dependencies.xml from bwa_base_0440, and make sure bwa_base_0440 now has two installable revisions: 1 and 2
Tool dependencies:
1. Create and populate bwa_tool_dependency_0440 so that it has a valid tool dependency definition and an installable revision 0.
2. Delete tool_dependencies.xml from bwa_tool_dependency_0440, and make sure that bwa_tool_dependency_0440 still has
a single installable revision 0.
3. Add the same tool_dependencies.xml file to bwa_tool_dependency_0440, and make sure that bwa_tool_dependency_0440
still has a single installable revision 0.
'''
@@ -11,13 +11,10 @@ repository_long_description = "Convert bed to gff"
category_name = 'Test 0140 Tool Help Images'
category_description = 'Test 0140 Tool Help Images'
'''
1) Create and populate the htseq_count_0140 repository.
2) Visit the manage_repository page, then the tool page, and look for the image string
similar to the following string where the encoded repository_id is previously determined:
src="/repository/static/images/<id>/count_modes.png"
'''
# 1) Create and populate the htseq_count_0140 repository.
# 2) Visit the manage_repository page, then the tool page, and look for the image string
# similar to the following string where the encoded repository_id is previously determined:
# src="/repository/static/images/<id>/count_modes.png"
class TestToolHelpImages(ShedTwillTestCase):
@@ -24,8 +24,7 @@ category_0060_name = 'Test 0060 Workflow Features'
'''
This script will run in one of two possible ways:
1. Directly, by calling sh run_functional_tests.sh -toolshed test/tool_shed/functional/test_1300_reset_all_metadata.py.
-OR-
1. Directly, by calling sh run_functional_tests.sh -toolshed test/tool_shed/functional/test_1300_reset_all_metadata.py. -or-
2. After the previous test scripts have completed.
In the first case, it is desirable to have the Galaxy database in a state that is as close as possible to the state it would
+4 -1
View File
@@ -303,6 +303,7 @@ def get_tool_path_by_shed_tool_conf_filename(app, shed_tool_conf):
def handle_email_alerts(app, host, repository, content_alert_str='', new_repo_alert=False, admin_only=False):
"""
There are 2 complementary features that enable a tool shed user to receive email notification:
1. Within User Preferences, they can elect to receive email when the first (or first valid)
change set is produced for a new repository.
2. When viewing or managing a repository, they can check the box labeled "Receive email alerts"
@@ -310,18 +311,20 @@ def handle_email_alerts(app, host, repository, content_alert_str='', new_repo_al
is available on a per-repository basis on the repository grid within the tool shed.
There are currently 4 scenarios for sending email notification when a change is made to a repository:
1. An admin user elects to receive email when the first change set is produced for a new repository
from User Preferences. The change set does not have to include any valid content. This allows for
the capture of inappropriate content being uploaded to new repositories.
2. A regular user elects to receive email when the first valid change set is produced for a new repository
from User Preferences. This differs from 1 above in that the user will not receive email until a
change set tha tincludes valid content is produced.
change set that includes valid content is produced.
3. An admin user checks the "Receive email alerts" check box on the manage repository page. Since the
user is an admin user, the email will include information about both HTML and image content that was
included in the change set.
4. A regular user checks the "Receive email alerts" check box on the manage repository page. Since the
user is not an admin user, the email will not include any information about both HTML and image content
that was included in the change set.
"""
sa_session = app.model.context.current
repo = repository.hg_repo
@@ -389,6 +389,7 @@ class RepositoryReviewController(BaseUIController, ratings_util.ItemRatings):
def manage_repositories_ready_for_review(self, trans, **kwd):
"""
A repository is ready to be reviewed if one of the following conditions is met:
1) It contains no tools
2) It contains tools the tools_functionally_correct flag is set to True. This implies that the repository metadata revision was installed and tested
by the Tool Shed's install and test framework.