opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-01 15:37:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into remove_old_library

Browse Source
This commit is contained in:
Martin Cech
2017-11-13 11:21:42 -05:00
parent fc9059a4dd 7d8dd63957
commit 15852fcbf8
42 changed files with 288 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.ci/jenkins/selenium/run_tests.sh
+15 -3
View File
@@ -2,10 +2,22 @@
# Enable retries on tests to reduce chances of transient failures.
: ${GALAXY_TEST_SELENIUM_RETRIES:=1}
: ${GALAXY_TEST_ERRORS_DIRECTORY:=database/test-errors}
# If in Jenkins environment, use it for artifacts.
if [ -n "$BUILD_NUMBER" ];
then
: ${GALAXY_TEST_ERRORS_DIRECTORY:=${BUILD_NUMBER}-test-errors}
: ${GALAXY_TEST_SCREENSHOTS_DIRECTORY:=${BUILD_NUMBER}-test-screenshots}
else
: ${GALAXY_TEST_ERRORS_DIRECTORY:=database/test-errors}
: ${GALAXY_TEST_SCREENSHOTS_DIRECTORY:=database/test-screenshots}
fi
mkdir -p "$GALAXY_TEST_ERRORS_DIRECTORY"
mkdir -p "$GALAXY_TEST_SCREENSHOTS_DIRECTORY"
# Start Selenium server in the test Docker container.
DOCKER_RUN_EXTRA_ARGS="-e USE_SELENIUM=1 -e GALAXY_TEST_SELENIUM_RETRIES=${GALAXY_TEST_SELENIUM_RETRIES} -e GALAXY_TEST_ERRORS_DIRECTORY=${GALAXY_TEST_ERRORS_DIRECTORY} ${DOCKER_RUN_EXTRA_ARGS}"
DOCKER_RUN_EXTRA_ARGS="-e USE_SELENIUM=1 -e GALAXY_TEST_SELENIUM_RETRIES=${GALAXY_TEST_SELENIUM_RETRIES} -e GALAXY_TEST_ERRORS_DIRECTORY=${GALAXY_TEST_ERRORS_DIRECTORY} -e GALAXY_TEST_SCREENSHOTS_DIRECTORY=${GALAXY_TEST_SCREENSHOTS_DIRECTORY} ${DOCKER_RUN_EXTRA_ARGS}"
export DOCKER_RUN_EXTRA_ARGS
./run_tests.sh --dockerize --db postgres --external_tmp --clean_pyc --selenium "$@"
./run_tests.sh --dockerize --db postgres --external_tmp --clean_pyc --skip_flakey_fails --selenium "$@"
client/galaxy/scripts/utils/ajax-queue.js
+5
View File
@@ -129,6 +129,11 @@ class AjaxQueue {
* fn: the deferring fn or ajax call }
*/
class NamedAjaxQueue extends AjaxQueue {
constructor(initialFunctions) {
super(initialFunctions);
this.names = {};
}
/** add the obj.fn to the queue if obj.name hasn't been used before */
add(obj) {
if (!(obj.hasOwnProperty("name") && obj.hasOwnProperty("fn"))) {
doc/source/releases/17.09_announce.rst
+1 -1
View File
@@ -143,7 +143,7 @@ Disclosed on the `mailing list <https://lists.galaxyproject.org/pipermail/galaxy
Vulnerabilities were found by Eric Rasche and Manabu Ishii respectively. Detailed descriptions of these categories of vulnerabilities can be found at:
- https://www.owasp.org/index.php/Cross-site_Scripting_(XSS)
- `<https://www.owasp.org/index.php/Cross-site_Scripting_(XSS)>`__
- https://www.owasp.org/index.php/Session_fixation
The fix for these issues has been applied to Galaxy releases back to 16.10 and can be found in this `diff <https://gist.githubusercontent.com/jmchilton/760bf8ba6055b9a47a48529fcc49a493/raw/01bc98e5a8067a435f38d7cf4fda4e304c4425a2/2017augsecurity_1610.patch>`__
lib/galaxy/datatypes/converters/bgzip.py
+2 -2
View File
@@ -9,7 +9,7 @@ import optparse
import subprocess
import tempfile
from pysam import ctabix
import pysam
def main():
@@ -44,7 +44,7 @@ def main():
grepped.stdout.close()
output, err = after_sort.communicate()
ctabix.tabix_compress(tmpfile.name, output_fname, force=True)
pysam.tabix_compress(tmpfile.name, output_fname, force=True)
if __name__ == "__main__":
lib/galaxy/datatypes/converters/interval_to_tabix_converter.py
+6 -6
View File
@@ -10,7 +10,7 @@ import optparse
import os
import sys
from pysam import ctabix
import pysam
def main():
@@ -26,13 +26,13 @@ def main():
# Create index.
if options.preset:
# Preset type.
ctabix.tabix_index(filename=index_fname, preset=options.preset, keep_original=True,
index_filename=out_fname)
pysam.tabix_index(filename=index_fname, preset=options.preset, keep_original=True,
index_filename=out_fname)
else:
# For interval files; column indices are 0-based.
ctabix.tabix_index(filename=index_fname, seq_col=(options.chrom_col - 1),
start_col=(options.start_col - 1), end_col=(options.end_col - 1),
keep_original=True, index_filename=out_fname)
pysam.tabix_index(filename=index_fname, seq_col=(options.chrom_col - 1),
start_col=(options.start_col - 1), end_col=(options.end_col - 1),
keep_original=True, index_filename=out_fname)
if os.path.getsize(index_fname) == 0:
sys.stderr.write("The converted tabix index file is empty, meaning the input data is invalid.")
lib/galaxy/datatypes/converters/tabular_to_dbnsfp.py
+3 -3
View File
@@ -8,7 +8,7 @@ usage: %prog in_file out_file
import optparse
import os.path
from pysam import ctabix
import pysam
def main():
@@ -26,9 +26,9 @@ def main():
output_dir = os.path.dirname(output_fname)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
ctabix.tabix_compress(input_fname, output_fname, force=True)
pysam.tabix_compress(input_fname, output_fname, force=True)
# Column indices are 0-based.
ctabix.tabix_index(output_fname, seq_col=options.chrom_col, start_col=options.start_col, end_col=options.end_col)
pysam.tabix_index(output_fname, seq_col=options.chrom_col, start_col=options.start_col, end_col=options.end_col)
if __name__ == "__main__":
lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py
+2 -2
View File
@@ -9,7 +9,7 @@ usage: %prog in_file out_file
"""
import optparse
from pysam import ctabix
import pysam
def main():
@@ -18,7 +18,7 @@ def main():
(options, args) = parser.parse_args()
input_fname, output_fname = args
ctabix.tabix_compress(input_fname, output_fname, force=True)
pysam.tabix_compress(input_fname, output_fname, force=True)
if __name__ == "__main__":
lib/galaxy/dependencies/conda-environment.txt
+1 -1
View File
@@ -72,4 +72,4 @@ Fabric
# We still pin these dependencies because of modifications to the upstream packages
# Flexible BAM index naming
#pysam==0.8.4+gx5
pysam>=0.13
lib/galaxy/dependencies/pinned-requirements.txt
+5 -4
View File
@@ -1,13 +1,17 @@
# packages with C extensions
# numpy must come before bx-python when doing source installs to so bx-python
# is built properly (see #4982).
numpy==1.9.2
bx-python==0.7.3
MarkupSafe==0.23
PyYAML==3.11
SQLAlchemy==1.0.15
sqlalchemy-utils==0.32.19
mercurial==3.7.3
numpy==1.9.2
pycrypto==2.6.1
uWSGI==2.0.15
# Flexible BAM index naming is new to core pysam
pysam>=0.13
# Install python_lzo if you want to support indexed access to lzo-compressed
# locally cached maf files via bx-python
@@ -83,8 +87,5 @@ Fabric==1.13.2
paramiko==2.2.1
ecdsa==0.13
# Flexible BAM index naming
pysam==0.8.4+gx5
# GenomeSpace dependencies
python-genomespaceclient==0.1.8
lib/galaxy/dependencies/requirements.txt
+2 -5
View File
@@ -7,6 +7,8 @@ PyYAML
SQLAlchemy
mercurial
pycrypto
# Flexible BAM index naming is new to main pysam
pysam>=0.13
# Install python_lzo if you want to support indexed access to lzo-compressed
# locally cached maf files via bx-python
@@ -66,8 +68,3 @@ pyparsing
Fabric
paramiko
ecdsa
# We still pin these dependencies because of modifications to the upstream packages
# Flexible BAM index naming
pysam==0.8.4+gx5
lib/galaxy/jobs/runners/local.py
+33 -7
View File
@@ -7,6 +7,7 @@ import logging
import os
import subprocess
import tempfile
import threading
from time import sleep
from galaxy import model
@@ -42,6 +43,8 @@ class LocalJobRunner(BaseJobRunner):
# create a local copy of os.environ to use as env for subprocess.Popen
self._environ = os.environ.copy()
self._proc_lock = threading.Lock()
self._procs = []
# Set TEMP if a valid temp value is not already set
if not ('TMPDIR' in self._environ or 'TEMP' in self._environ or 'TMP' in self._environ):
@@ -98,20 +101,36 @@ class LocalJobRunner(BaseJobRunner):
stderr=stderr_file,
env=self._environ,
preexec_fn=os.setpgrp)
job_wrapper.set_job_destination(job_wrapper.job_destination, proc.pid)
job_wrapper.change_state(model.Job.states.RUNNING)
terminated = self.__poll_if_needed(proc, job_wrapper, job_id)
if terminated:
return
proc.terminated_by_shutdown = False
with self._proc_lock:
self._procs.append(proc)
try:
job_wrapper.set_job_destination(job_wrapper.job_destination, proc.pid)
job_wrapper.change_state(model.Job.states.RUNNING)
terminated = self.__poll_if_needed(proc, job_wrapper, job_id)
if terminated:
return
# Reap the process and get the exit code.
exit_code = proc.wait()
finally:
with self._proc_lock:
self._procs.remove(proc)
# Reap the process and get the exit code.
exit_code = proc.wait()
try:
exit_code = int(open(exit_code_path, 'r').read())
except Exception:
log.warning("Failed to read exit code from path %s" % exit_code_path)
pass
if proc.terminated_by_shutdown:
self._fail_job_local(job_wrapper, "job terminated by Galaxy shutdown")
return
stdout_file.seek(0)
stderr_file.seek(0)
stdout = shrink_stream_by_size(stdout_file, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True)
@@ -165,6 +184,13 @@ class LocalJobRunner(BaseJobRunner):
# local jobs can't be recovered
job_wrapper.change_state(model.Job.states.ERROR, info="This job was killed when Galaxy was restarted. Please retry the job.")
def shutdown(self):
super(LocalJobRunner, self).shutdown()
with self._proc_lock:
for proc in self._procs:
proc.terminated_by_shutdown = True
self._terminate(proc)
def _fail_job_local(self, job_wrapper, message):
job_destination = job_wrapper.job_destination
job_state = JobState(job_wrapper, job_destination)
lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh
+1
View File
@@ -22,6 +22,7 @@ _galaxy_setup_environment() {
$integrity_injection
$slots_statement
export GALAXY_SLOTS
$memory_statement
GALAXY_VIRTUAL_ENV="$galaxy_virtual_env"
_GALAXY_VIRTUAL_ENV="$galaxy_virtual_env"
PRESERVE_GALAXY_ENVIRONMENT="$preserve_python_environment"
lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT.sh
+4
View File
@@ -0,0 +1,4 @@
if [ -n "$SLURM_JOB_ID" ]; then
GALAXY_MEMORY_MB=`scontrol -do show job "$SLURM_JOB_ID" | sed 's/.*\( \|^\)Mem=\([0-9][0-9]*\)\( \|$\).*/\2/p;d'` 2>memory_statement.log
fi
[ "${GALAXY_MEMORY_MB--1}" -gt 0 ] 2>>memory_statement.log && export GALAXY_MEMORY_MB || unset GALAXY_MEMORY_MB
lib/galaxy/jobs/runners/util/job_script/__init__.py
+9 -2
View File
@@ -19,6 +19,9 @@ DEFAULT_JOB_FILE_TEMPLATE = Template(
SLOTS_STATEMENT_CLUSTER_DEFAULT = \
resource_string(__name__, 'CLUSTER_SLOTS_STATEMENT.sh').decode('UTF-8')
MEMORY_STATEMENT_DEFAULT = \
resource_string(__name__, 'MEMORY_STATEMENT.sh').decode('UTF-8')
SLOTS_STATEMENT_SINGLE = """
GALAXY_SLOTS="1"
"""
@@ -43,6 +46,7 @@ OPTIONAL_TEMPLATE_PARAMS = {
'headers': '',
'env_setup_commands': [],
'slots_statement': SLOTS_STATEMENT_CLUSTER_DEFAULT,
'memory_statement': MEMORY_STATEMENT_DEFAULT,
'instrument_pre_commands': '',
'instrument_post_commands': '',
'integrity_injection': INTEGRITY_INJECTION,
@@ -74,6 +78,9 @@ def job_script(template=DEFAULT_JOB_FILE_TEMPLATE, **kwds):
>>> script = job_script(working_directory='wd', command='uptime', exit_code_path='ec', slots_statement='GALAXY_SLOTS="$SLURM_JOB_NUM_NODES"')
>>> script.find('GALAXY_SLOTS="$SLURM_JOB_NUM_NODES"\\nexport GALAXY_SLOTS\\n') > 0
True
>>> script = job_script(working_directory='wd', command='uptime', exit_code_path='ec', memory_statement='GALAXY_MEMORY_MB="32768"')
>>> script.find('GALAXY_MEMORY_MB="32768"\\n') > 0
True
"""
if any([param not in kwds for param in REQUIRED_TEMPLATE_PARAMS]):
raise Exception("Failed to create job_script, a required parameter is missing.")
@@ -126,7 +133,7 @@ def _handle_script_integrity(path, config):
script_integrity_verified = True
break
log.debug("Script integrity error: returncode was %d", returncode)
log.debug("Script integrity error for file '%s': returncode was %d", path, returncode)
# Else we will sync and wait to see if the script becomes
# executable.
@@ -144,7 +151,7 @@ def _handle_script_integrity(path, config):
time.sleep(sleep_amt)
if not script_integrity_verified:
raise Exception("Failed to write job script, could not verify job script integrity.")
raise Exception("Failed to write job script '%s', could not verify job script integrity." % path)
__all__ = (
lib/galaxy/tools/cache.py
+2
View File
@@ -83,6 +83,8 @@ class ToolCache(object):
def cache_tool(self, config_filename, tool):
tool_hash = md5_hash_file(config_filename)
if tool_hash is None:
return
tool_id = str(tool.id)
self._hash_by_tool_paths[config_filename] = tool_hash
self._mod_time_by_path[config_filename] = os.path.getmtime(config_filename)
lib/galaxy/tools/imp_exp/__init__.py
+11 -31
View File
@@ -4,7 +4,7 @@ import logging
import os
import shutil
import tempfile
from json import dumps, loads
from json import dumps, load
from sqlalchemy.orm import eagerload_all
from sqlalchemy.sql import expression
@@ -40,21 +40,6 @@ class JobImportHistoryArchiveWrapper(object, UsesAnnotations):
abs_file_path = os.path.abspath(file_path)
return os.path.split(abs_file_path)[0] == a_dir
def read_file_contents(file_path):
""" Read contents of a file. """
fp = open(file_path, 'rb')
buffsize = 1048576
file_contents = ''
try:
while True:
file_contents += fp.read(buffsize)
if not file_contents or len(file_contents) % buffsize != 0:
break
except OverflowError:
pass
fp.close()
return file_contents
def get_tag_str(tag, value):
""" Builds a tag string for a tag, value pair. """
if not value:
@@ -84,11 +69,10 @@ class JobImportHistoryArchiveWrapper(object, UsesAnnotations):
# Create history.
#
history_attr_file_name = os.path.join(archive_dir, 'history_attrs.txt')
history_attr_str = read_file_contents(history_attr_file_name)
history_attrs = loads(history_attr_str)
history_attrs = load(open(history_attr_file_name))
# Create history.
new_history = model.History(name='imported from archive: %s' % history_attrs['name'].encode('utf-8'),
new_history = model.History(name='imported from archive: %s' % history_attrs['name'],
user=user)
new_history.importing = True
new_history.hid_counter = history_attrs['hid_counter']
@@ -110,12 +94,11 @@ class JobImportHistoryArchiveWrapper(object, UsesAnnotations):
# Create datasets.
#
datasets_attrs_file_name = os.path.join(archive_dir, 'datasets_attrs.txt')
datasets_attr_str = read_file_contents(datasets_attrs_file_name)
datasets_attrs = loads(datasets_attr_str)
datasets_attrs = load(open(datasets_attrs_file_name))
provenance_file_name = datasets_attrs_file_name + ".provenance"
if os.path.exists(datasets_attrs_file_name + ".provenance"):
provenance_attr_str = read_file_contents(datasets_attrs_file_name + ".provenance")
provenance_attrs = loads(provenance_attr_str)
if os.path.exists(provenance_file_name):
provenance_attrs = load(open(provenance_file_name))
datasets_attrs += provenance_attrs
# Get counts of how often each dataset file is used; a file can
@@ -133,9 +116,9 @@ class JobImportHistoryArchiveWrapper(object, UsesAnnotations):
metadata = dataset_attrs['metadata']
# Create dataset and HDA.
hda = model.HistoryDatasetAssociation(name=dataset_attrs['name'].encode('utf-8'),
hda = model.HistoryDatasetAssociation(name=dataset_attrs['name'],
extension=dataset_attrs['extension'],
info=dataset_attrs['info'].encode('utf-8'),
info=dataset_attrs['info'],
blurb=dataset_attrs['blurb'],
peek=dataset_attrs['peek'],
designation=dataset_attrs['designation'],
@@ -210,10 +193,6 @@ class JobImportHistoryArchiveWrapper(object, UsesAnnotations):
# Create jobs.
#
# Read jobs attributes.
jobs_attr_file_name = os.path.join(archive_dir, 'jobs_attrs.txt')
jobs_attr_str = read_file_contents(jobs_attr_file_name)
# Decode jobs attributes.
def as_hda(obj_dct):
""" Hook to 'decode' an HDA; method uses history and HID to get the HDA represented by
@@ -222,7 +201,8 @@ class JobImportHistoryArchiveWrapper(object, UsesAnnotations):
return self.sa_session.query(model.HistoryDatasetAssociation) \
.filter_by(history=new_history, hid=obj_dct['hid']).first()
return obj_dct
jobs_attrs = loads(jobs_attr_str, object_hook=as_hda)
jobs_attr_file_name = os.path.join(archive_dir, 'jobs_attrs.txt')
jobs_attrs = load(open(jobs_attr_file_name), object_hook=as_hda)
# Create each job.
for job_attrs in jobs_attrs:
lib/galaxy/tools/toolbox/watcher.py
+35 -19
View File
@@ -100,21 +100,36 @@ class ToolConfWatcher(object):
with self._lock:
paths = list(self.paths.keys())
for path in paths:
if not os.path.exists(path):
continue
mod_time = self.paths[path]
if not hashes.get(path, None):
hashes[path] = md5_hash_file(path)
new_mod_time = None
if os.path.exists(path):
try:
if not os.path.exists(path):
continue
mod_time = self.paths[path]
if not hashes.get(path, None):
hash = md5_hash_file(path)
if hash:
hashes[path] = md5_hash_file(path)
else:
continue
new_mod_time = os.path.getmtime(path)
if new_mod_time > mod_time:
new_hash = md5_hash_file(path)
if hashes[path] != new_hash:
self.paths[path] = new_mod_time
hashes[path] = new_hash
log.debug("The file '%s' has changes.", path)
do_reload = True
if new_mod_time > mod_time:
new_hash = md5_hash_file(path)
if hashes[path] != new_hash:
self.paths[path] = new_mod_time
hashes[path] = new_hash
log.debug("The file '%s' has changes.", path)
do_reload = True
except IOError:
# in rare cases `path` may be deleted between `os.path.exists` calls
# and reading the file from the filesystem. We do not want the watcher
# thread to die in these cases.
try:
del hashes[path]
del paths[path]
except KeyError:
pass
if self.cache:
self.cache.cleanup()
do_reload = True
if not do_reload and self.cache:
removed_ids = self.cache.cleanup()
if removed_ids:
@@ -233,11 +248,12 @@ class LocFileEventHandler(FileSystemEventHandler):
path = os.path.abspath(path)
if path.endswith(".loc"):
cur_hash = md5_hash_file(path)
if self.loc_watcher.path_hash.get(path) == cur_hash:
return
else:
self.loc_watcher.path_hash[path] = cur_hash
self.loc_watcher.tool_data_tables.reload_tables(path=path)
if cur_hash:
if self.loc_watcher.path_hash.get(path) == cur_hash:
return
else:
self.loc_watcher.path_hash[path] = cur_hash
self.loc_watcher.tool_data_tables.reload_tables(path=path)
class ToolFileEventHandler(FileSystemEventHandler):
lib/galaxy/tools/xsd/galaxy.xsd
+1
View File
@@ -2656,6 +2656,7 @@ be escaped with a backslash (``\``) when appearing in ``command`` or ``configfil
Name | Description
---- | -----------
``\${GALAXY_SLOTS:-4}`` | Number of cores/threads allocated by the job runner or resource manager to the tool for the given job (here 4 is the default number of threads to use if running via custom runner that does not configure GALAXY_SLOTS or in an older Galaxy runtime).
``\$GALAXY_MEMORY_MB`` | Amount of memory in megabytes (1024^2 bytes) allocated by the administrator (via the resource manager) to the tool for the given job. If unset, tools should not attempt to limit memory usage.
See the [Planemo docs](https://planemo.readthedocs.io/en/latest/writing_advanced.html#cluster-usage)
on the topic of ``GALAXY_SLOTS`` for more information and examples.
lib/galaxy/util/hash_util.py
+9 -5
View File
@@ -17,13 +17,17 @@ md5 = hashlib.md5
def md5_hash_file(path):
"""
Return a md5 hashdigest for a file.
Return a md5 hashdigest for a file or None if path could not be read.
"""
hasher = hashlib.md5()
with open(path, 'rb') as afile:
buf = afile.read()
hasher.update(buf)
return hasher.hexdigest()
try:
with open(path, 'rb') as afile:
buf = afile.read()
hasher.update(buf)
return hasher.hexdigest()
except IOError:
# This may happen if path has been deleted
return None
def new_secure_hash(text_type=None):
lib/galaxy/visualization/data_providers/genome.py
+2 -3
View File
@@ -14,7 +14,6 @@ import pysam
from bx.bbi.bigbed_file import BigBedFile
from bx.bbi.bigwig_file import BigWigFile
from bx.interval_index_file import Indexes
from pysam import ctabix
from galaxy.datatypes.interval import Bed, Gff, Gtf
from galaxy.datatypes.util.gff_util import convert_gff_coords_to_bed, GFFFeature, GFFInterval, GFFReaderWrapper, parse_gff_attributes
@@ -324,8 +323,8 @@ class TabixDataProvider(FilterableMixin, GenomeDataProvider):
col_name_data_attr_mapping = {4: {'index': 4, 'name': 'Score'}}
def open_data_file(self):
return ctabix.Tabixfile(self.dependencies['bgzip'].file_name,
index=self.converted_dataset.file_name)
return pysam.Tabixfile(self.dependencies['bgzip'].file_name,
index=self.converted_dataset.file_name)
def get_iterator(self, data_file, chrom, start, end, **kwargs):
# chrom must be a string, start/end integers.
lib/tool_shed/util/hg_util.py
+5 -8
View File
@@ -2,6 +2,7 @@ import json
import logging
import os
import struct
import subprocess
import tempfile
from datetime import datetime
from time import gmtime
@@ -10,7 +11,6 @@ from mercurial import cmdutil, commands, hg, ui
from mercurial.changegroup import readexactly
from mercurial.exchange import readbundle
from galaxy.util import listify
from tool_shed.util import basic_util
log = logging.getLogger(__name__)
@@ -55,15 +55,12 @@ def clone_repository(repository_clone_url, repository_file_dir, ctx_rev):
present in the cloned repository.
"""
try:
commands.clone(get_configured_ui(),
str(repository_clone_url),
dest=str(repository_file_dir),
pull=True,
noupdate=False,
rev=listify(str(ctx_rev)))
stdouterr = subprocess.check_output(['hg', 'clone', '-r', ctx_rev, repository_clone_url, repository_file_dir], stderr=subprocess.STDOUT)
return True, None
except Exception as e:
error_message = 'Error cloning repository: %s' % str(e)
error_message = 'Error cloning repository: %s' % e
if isinstance(e, subprocess.CalledProcessError):
error_message += "\nOutput was:\n%s" % stdouterr
log.debug(error_message)
return False, error_message
run_tests.sh
+9
View File
@@ -157,6 +157,8 @@ Extra options:
--external_master_key Master API key used to configure external tests.
--external_user_key User API used for external tests - not required if
external_master_key is specified.
--skip_flakey_fails Skip flakey tests on error (sets
GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR=1).
Environment Variables:
@@ -210,6 +212,8 @@ GALAXY_TEST_FETCH_DATA Fetch remote test data to
command-line.
GALAXY_TEST_DATA_REPO_CACHE Where to cache remote test data to (default to
test-data-cache).
GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR
Skip tests annotated with @flakey on test errors.
HTTP_ACCEPT_LANGUAGE Defaults to 'en'
GALAXY_TEST_NO_CLEANUP Do not cleanup main test directory after tests,
the deprecated option TOOL_SHED_TEST_NO_CLEANUP
@@ -380,6 +384,11 @@ do
find test -iname '*pyc' -exec rm -rf {} \;
shift
;;
-skip_flakey_fails|--skip_flakey_fails)
GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR=1
export GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR
shift
;;
-with_framework_test_tools|--with_framework_test_tools)
with_framework_test_tools_arg="-with_framework_test_tools"
shift
static/maps/utils/ajax-queue.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/analysis.bundled.js
+1 -1
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/analysis.bundled.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/libs.bundled.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
static/scripts/utils/ajax-queue.js
+1 -1
View File
@@ -1,2 +1,2 @@
define("utils/ajax-queue",["exports"],function(e){"use strict";function t(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!=typeof t&&"function"!=typeof t?e:t}function r(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function, not "+typeof t);e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),t&&(Object.setPrototypeOf?Object.setPrototypeOf(e,t):e.__proto__=t)}function n(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var u=function e(t,r,n){null===t&&(t=Function.prototype);var u=Object.getOwnPropertyDescriptor(t,r);if(void 0===u){var o=Object.getPrototypeOf(t);return null===o?void 0:e(o,r,n)}if("value"in u)return u.value;var i=u.get;if(void 0!==i)return i.call(n)},o=function(){function e(e,t){for(var r=0;r<t.length;r++){var n=t[r];n.enumerable=n.enumerable||!1,n.configurable=!0,"value"in n&&(n.writable=!0),Object.defineProperty(e,n.key,n)}}return function(t,r,n){return r&&e(t.prototype,r),n&&e(t,n),t}}(),i=function(){function e(t){n(this,e),this.deferred=jQuery.Deferred(),this.queue=[],this.responses=[],this.numToProcess=0,this.running=!1,this.init(t||[]),this.start()}return o(e,[{key:"init",value:function(e){var t=this;e.forEach(function(e){t.add(e)})}},{key:"add",value:function(e){var t=this,r=this.queue.length;return this.numToProcess+=1,this.queue.push(function(){var n=r,u=e();u.done(function(e){t.deferred.notify({curr:n,total:t.numToProcess,response:e})}),u.always(function(e){t.responses.push(e),t.queue.length?t.queue.shift()():t.stop()})}),this}},{key:"start",value:function(){return this.queue.length&&(this.running=!0,this.queue.shift()()),this}},{key:"stop",value:function(e,t){return this.running=!1,this.queue=[],e?this.deferred.reject(t):this.deferred.resolve(this.responses),this.numToProcess=0,this.deferred=jQuery.Deferred(),this}},{key:"done",value:function(e){return this.deferred.done(e)}},{key:"fail",value:function(e){return this.deferred.fail(e)}},{key:"always",value:function(e){return this.deferred.always(e)}},{key:"progress",value:function(e){return this.deferred.progress(e)}}],[{key:"create",value:function(t){return new e(t).deferred}}]),e}(),s=function(e){function s(){return n(this,s),t(this,(s.__proto__||Object.getPrototypeOf(s)).apply(this,arguments))}return r(s,i),o(s,[{key:"add",value:function(e){if(!e.hasOwnProperty("name")||!e.hasOwnProperty("fn"))throw new Error('NamedAjaxQueue.add requires an object with both "name" and "fn": '+JSON.stringify(e));if(!this.names.hasOwnProperty(e.name))return this.names[e.name]=!0,u(s.prototype.__proto__||Object.getPrototypeOf(s.prototype),"add",this).call(this,e.fn)}},{key:"clear",value:function(){return this.names={},this}}],[{key:"create",value:function(e){return new s(e).deferred}}]),s}();e.default={AjaxQueue:i,NamedAjaxQueue:s}});
define("utils/ajax-queue",["exports"],function(e){"use strict";function t(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!=typeof t&&"function"!=typeof t?e:t}function r(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function, not "+typeof t);e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),t&&(Object.setPrototypeOf?Object.setPrototypeOf(e,t):e.__proto__=t)}function n(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var u=function e(t,r,n){null===t&&(t=Function.prototype);var u=Object.getOwnPropertyDescriptor(t,r);if(void 0===u){var o=Object.getPrototypeOf(t);return null===o?void 0:e(o,r,n)}if("value"in u)return u.value;var i=u.get;if(void 0!==i)return i.call(n)},o=function(){function e(e,t){for(var r=0;r<t.length;r++){var n=t[r];n.enumerable=n.enumerable||!1,n.configurable=!0,"value"in n&&(n.writable=!0),Object.defineProperty(e,n.key,n)}}return function(t,r,n){return r&&e(t.prototype,r),n&&e(t,n),t}}(),i=function(){function e(t){n(this,e),this.deferred=jQuery.Deferred(),this.queue=[],this.responses=[],this.numToProcess=0,this.running=!1,this.init(t||[]),this.start()}return o(e,[{key:"init",value:function(e){var t=this;e.forEach(function(e){t.add(e)})}},{key:"add",value:function(e){var t=this,r=this.queue.length;return this.numToProcess+=1,this.queue.push(function(){var n=r,u=e();u.done(function(e){t.deferred.notify({curr:n,total:t.numToProcess,response:e})}),u.always(function(e){t.responses.push(e),t.queue.length?t.queue.shift()():t.stop()})}),this}},{key:"start",value:function(){return this.queue.length&&(this.running=!0,this.queue.shift()()),this}},{key:"stop",value:function(e,t){return this.running=!1,this.queue=[],e?this.deferred.reject(t):this.deferred.resolve(this.responses),this.numToProcess=0,this.deferred=jQuery.Deferred(),this}},{key:"done",value:function(e){return this.deferred.done(e)}},{key:"fail",value:function(e){return this.deferred.fail(e)}},{key:"always",value:function(e){return this.deferred.always(e)}},{key:"progress",value:function(e){return this.deferred.progress(e)}}],[{key:"create",value:function(t){return new e(t).deferred}}]),e}(),s=function(e){function s(e){n(this,s);var r=t(this,(s.__proto__||Object.getPrototypeOf(s)).call(this,e));return r.names={},r}return r(s,i),o(s,[{key:"add",value:function(e){if(!e.hasOwnProperty("name")||!e.hasOwnProperty("fn"))throw new Error('NamedAjaxQueue.add requires an object with both "name" and "fn": '+JSON.stringify(e));if(!this.names.hasOwnProperty(e.name))return this.names[e.name]=!0,u(s.prototype.__proto__||Object.getPrototypeOf(s.prototype),"add",this).call(this,e.fn)}},{key:"clear",value:function(){return this.names={},this}}],[{key:"create",value:function(e){return new s(e).deferred}}]),s}();e.default={AjaxQueue:i,NamedAjaxQueue:s}});
//# sourceMappingURL=../../maps/utils/ajax-queue.js.map
test/base/populators.py
+19
View File
@@ -1,5 +1,6 @@
import contextlib
import json
import os
import time
from functools import wraps
from operator import itemgetter
@@ -23,6 +24,24 @@ workflow_random_x2_str = resource_string(__name__, "data/test_workflow_2.ga")
DEFAULT_TIMEOUT = 60 # Secs to wait for state to turn ok
SKIP_FLAKEY_TESTS_ON_ERROR = os.environ.get("GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR", None)
def flakey(method):
@wraps(method)
def wrapped_method(test_case, *args, **kwargs):
try:
method(test_case, *args, **kwargs)
except Exception:
if SKIP_FLAKEY_TESTS_ON_ERROR:
from nose.plugins.skip import SkipTest
raise SkipTest()
else:
raise
return wrapped_method
def skip_without_tool(tool_id):
"""Decorate an API test method as requiring a specific tool.
test/selenium_tests/framework.py
+26
View File
@@ -42,6 +42,7 @@ DEFAULT_ADMIN_PASSWORD = "testpass"
TIMEOUT_MULTIPLIER = float(os.environ.get("GALAXY_TEST_TIMEOUT_MULTIPLIER", DEFAULT_TIMEOUT_MULTIPLIER))
GALAXY_TEST_ERRORS_DIRECTORY = os.environ.get("GALAXY_TEST_ERRORS_DIRECTORY", DEFAULT_TEST_ERRORS_DIRECTORY)
GALAXY_TEST_SCREENSHOTS_DIRECTORY = os.environ.get("GALAXY_TEST_SCREENSHOTS_DIRECTORY", None)
# Test browser can be ["CHROME", "FIREFOX", "OPERA", "PHANTOMJS"]
GALAXY_TEST_SELENIUM_BROWSER = os.environ.get("GALAXY_TEST_SELENIUM_BROWSER", DEFAULT_SELENIUM_BROWSER)
GALAXY_TEST_SELENIUM_REMOTE = os.environ.get("GALAXY_TEST_SELENIUM_REMOTE", DEFAULT_SELENIUM_REMOTE)
@@ -225,8 +226,33 @@ class SeleniumTestCase(FunctionalTestCase, NavigatesGalaxy, UsesApiTestCaseMixin
raise exception
def snapshot(self, description):
"""Create a debug snapshot (DOM, screenshot, etc...) that is written out on tool failure.
This information will be automatically written to a per-test directory created for all
failed tests.
"""
self.snapshots.append(TestSnapshot(self.driver, len(self.snapshots), description))
def screenshot(self, label):
"""If GALAXY_TEST_SCREENSHOTS_DIRECTORY is set create a screenshot there named <label>.png.
Unlike the above "snapshot" feature, this will be written out regardless and not in a per-test
directory. The above method is used for debugging failures within a specific test. This method
if more for creating a set of images to augment automated testing with manual human inspection
after a test or test suite has executed.
"""
if GALAXY_TEST_SCREENSHOTS_DIRECTORY is None:
return
if not os.path.exists(GALAXY_TEST_SCREENSHOTS_DIRECTORY):
os.makedirs(GALAXY_TEST_SCREENSHOTS_DIRECTORY)
target = os.path.join(GALAXY_TEST_SCREENSHOTS_DIRECTORY, label + ".png")
copy = 1
while os.path.exists(target):
# Maybe previously a test re-run - keep the original.
target = os.path.join(GALAXY_TEST_SCREENSHOTS_DIRECTORY, "%s-%d.png" % (label, copy))
copy += 1
self.driver.save_screenshot(target)
def reset_driver_and_session(self):
self.tear_down_driver()
self.setup_driver_and_session()
test/selenium_tests/test_collection_builders.py
+3 -3
View File
@@ -17,7 +17,7 @@ class CollectionBuildersTestCase(SeleniumTestCase):
self.history_panel_multi_operation_action_click(self.navigation.history_panel.multi_operations.labels.build_list)
self.collection_builder_set_name("my cool list")
self.screenshot("collection_builder_list")
self.collection_builder_create()
self.history_panel_wait_for_hid_ok(2)
@@ -48,7 +48,7 @@ class CollectionBuildersTestCase(SeleniumTestCase):
self.history_panel_muli_operation_select_hid(2)
self.history_panel_multi_operation_action_click(self.navigation.history_panel.multi_operations.labels.build_pair)
self.collection_builder_set_name("my awesome pair")
self.screenshot("collection_builder_pair")
self.collection_builder_create()
self.history_panel_wait_for_hid_ok(3)
@@ -67,7 +67,7 @@ class CollectionBuildersTestCase(SeleniumTestCase):
self.collection_builder_click_paired_item("forward", 0)
self.collection_builder_click_paired_item("reverse", 1)
self.collection_builder_set_name("my awesome paired list")
self.screenshot("collection_builder_paired_list")
self.collection_builder_create()
self.history_panel_wait_for_hid_ok(3)
test/selenium_tests/test_history_dataset_state.py
+2
View File
@@ -34,6 +34,8 @@ class HistoryDatasetStateTestCase(SeleniumTestCase, UsesHistoryItemAssertions):
self.assert_item_info_includes(1, 'uploaded fasta file')
self.assert_item_peek_includes(1, ">hg17")
self.screenshot("history_panel_dataset_expanded")
self._assert_action_buttons(1)
def _assert_title_buttons(self, hid, expected_buttons=['display', 'edit', 'delete']):
test/selenium_tests/test_history_sharing.py
+3 -1
View File
@@ -106,7 +106,7 @@ class HistorySharingTestCase(SeleniumTestCase):
with self.main_panel():
self.components.histories.sharing.share_with_a_user_button.wait_for_and_click()
def share_history_with_user(self, user_id=None, user_email=None, assert_valid=False):
def share_history_with_user(self, user_id=None, user_email=None, assert_valid=False, screenshot=False):
"""Share the current history with a target user by ID or email.
``user_email`` will be used to enter in the share form unless ``user_id``
@@ -121,6 +121,8 @@ class HistorySharingTestCase(SeleniumTestCase):
# line, in future dispatch on actual select2 div present or not.
# self.select2_set_value(form_selector, email)
self.fill(form, {"email": user_id or user_email})
if screenshot:
self.screenshot("history_sharing_user")
self.click_submit(form)
if assert_valid:
with self.main_panel():
test/selenium_tests/test_jupyter.py
+3
View File
@@ -1,3 +1,5 @@
from base.populators import flakey
from .framework import (
managed_history,
selenium_test,
@@ -11,6 +13,7 @@ class JupyterTestCase(SeleniumTestCase):
ensure_registered = True
@flakey
@selenium_test
@managed_history
def test_jupyter_session(self):
test/selenium_tests/test_library_contents.py
+2
View File
@@ -37,6 +37,7 @@ class LibraryContentsTestCase(SeleniumTestCase):
history_elements[0].find_element_by_css_selector("input").click()
# Add
self.sleep_for(self.wait_types.UX_RENDER)
self.screenshot("libraries_dataset_import")
self.wait_for_and_click(self.navigation.libraries.folder.selectors.import_datasets_ok_button)
# Let the progress bar disappear...
self.wait_for_absent_or_hidden(self.navigation.libraries.folder.selectors.import_progress_bar)
@@ -87,6 +88,7 @@ class LibraryContentsTestCase(SeleniumTestCase):
self.sleep_for(self.wait_types.UX_RENDER)
self.wait_for_selector_clickable(".ui-modal #button-0").click()
self.wait_for_overlays_cleared()
self.screenshot("libraries_show_details")
@retry_assertion_during_transitions
def _assert_num_displayed_items_is(self, n):
test/selenium_tests/test_library_landing.py
+5
View File
@@ -18,6 +18,7 @@ class LibraryLandingTestCase(SeleniumTestCase):
num_displayed_libraries = self._num_displayed_libraries()
self.libraries_index_click_create_new()
self.wait_for_selector_visible(".ui-modal")
self.screenshot("libraries_new")
close_button = self.wait_for_selector_clickable("#button-1")
close_button.click()
self.wait_for_overlays_cleared()
@@ -46,6 +47,7 @@ class LibraryLandingTestCase(SeleniumTestCase):
name_box = self.wait_for_selector_clickable(".input_library_name")
name_box.send_keys(new_name)
self.screenshot("libraries_rename")
save_button = self.wait_for_selector_clickable(".save_library_btn")
save_button.click()
@@ -71,9 +73,12 @@ class LibraryLandingTestCase(SeleniumTestCase):
self.wait_for_overlays_cleared()
self.libraries_index_create(namebase + " c")
self.screenshot("libraries_index")
self.libraries_index_search_for(namebase)
self._assert_num_displayed_libraries_is(3)
self.screenshot("libraries_index_search")
self._assert_names_are([namebase + " a", namebase + " b", namebase + " c"])
self.libraries_index_sort_click()
test/selenium_tests/test_published_histories_grid.py
+2 -1
View File
@@ -15,7 +15,7 @@ class HistoryGridTestCase(SharedStateSeleniumTestCase):
@selenium_test
def test_history_grid_search_standard(self):
self.navigate_to_published_histories_page()
self.screenshot("histories_published_grid")
self.published_grid_search_for(self.history1_name)
self.assert_grid_histories_are([self.history1_name])
@@ -37,6 +37,7 @@ class HistoryGridTestCase(SharedStateSeleniumTestCase):
# Search by name
self.set_filter(name_filter_selector, self.history1_name)
self.screenshot("histories_published_grid_advanced")
self.assert_grid_histories_are([self.history1_name])
self.unset_filter('name', self.history1_name)
test/selenium_tests/test_saved_histories.py
+1
View File
@@ -18,6 +18,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase):
def test_history_switch(self):
self._login()
self.navigate_to_saved_histories_page()
self.screenshot("histories_saved_grid")
self.click_popup_option(self.history2_name, 'Switch')
self.sleep_for(self.wait_types.UX_RENDER)
test/selenium_tests/test_tool_form.py
+1
View File
@@ -95,6 +95,7 @@ class ToolFormTestCase(SeleniumTestCase, UsesHistoryItemAssertions):
self.home()
self.tool_open("head")
self.tool_set_value("input", "1.fasta", expected_type="data")
self.screenshot("tool_form_simple_data")
self.tool_execute()
self.history_panel_wait_for_hid_ok(3)
test/selenium_tests/test_workflow_editor.py
+5
View File
@@ -32,6 +32,7 @@ class WorkflowEditorTestCase(SeleniumTestCase):
self.wait_for_selector_visible("#__workflow__inputs__ .toolTitle")
input_links = self.driver.find_elements_by_css_selector("#__workflow__inputs__ .toolTitle a")
input_links[0].click()
self.screenshot("workflow_editor_data_input")
# TODO: verify box is highlighted and side panel is a form describing input.
# More work needs to be done to develop testing abstractions for doing these things.
@@ -42,6 +43,7 @@ class WorkflowEditorTestCase(SeleniumTestCase):
self.workflow_index_open()
self.workflow_index_click_option("Edit")
self.sleep_for(self.wait_types.UX_RENDER)
self.screenshot("workflow_editor_edit_menu")
self.workflow_editor_click_option("Save As")
@selenium_test
@@ -51,6 +53,7 @@ class WorkflowEditorTestCase(SeleniumTestCase):
self.workflow_index_open()
self.workflow_index_click_option("Edit")
self.assert_modal_has_text("Using version '0.2' instead of version '0.0.1'")
self.screenshot("workflow_editor_tool_upgrade")
@selenium_test
def test_editor_invalid_tool_state(self):
@@ -60,6 +63,7 @@ class WorkflowEditorTestCase(SeleniumTestCase):
self.workflow_index_click_option("Edit")
self.assert_modal_has_text("Using version '0.2' instead of version '0.0.1'")
self.assert_modal_has_text("Using default: '1'")
self.screenshot("workflow_editor_invalid_state")
@selenium_test
def test_missing_tools(self):
@@ -77,6 +81,7 @@ steps:
self.workflow_index_open()
self.workflow_index_click_option("Edit")
self.assert_modal_has_text("Tool is not installed")
self.screenshot("workflow_editor_missing_tool")
def workflow_create_new(self, name=None, annotation=None):
self.workflow_index_open()
test/selenium_tests/test_workflow_management.py
+4 -1
View File
@@ -27,7 +27,7 @@ class WorkflowManagementTestCase(SeleniumTestCase):
self.workflow_index_click_option("View")
title = self.wait_for_selector(".page-body h3")
assert "TestWorkflow1" in title.text
self.screenshot("workflow_manage_view")
# TODO: Test display of steps...
@selenium_test
@@ -64,6 +64,7 @@ class WorkflowManagementTestCase(SeleniumTestCase):
self.assertEqual(self.workflow_index_tags(), ["cooltag"])
check_tags()
self.screenshot("workflow_manage_tags")
@selenium_test
def test_index_search(self):
@@ -71,6 +72,7 @@ class WorkflowManagementTestCase(SeleniumTestCase):
self._workflow_import_from_url()
self.workflow_index_rename("searchforthis")
self._assert_showing_n_workflows(1)
self.screenshot("workflow_manage_search")
self.workflow_index_search_for("doesnotmatch")
self._assert_showing_n_workflows(0)
@@ -100,6 +102,7 @@ class WorkflowManagementTestCase(SeleniumTestCase):
self.workflow_index_open()
assert_published_column_text_is("Yes")
self.screenshot("workflow_manage_published")
@retry_assertion_during_transitions
def _assert_showing_n_workflows(self, n):
test/selenium_tests/test_workflow_run.py
+2
View File
@@ -23,6 +23,7 @@ class WorkflowRunTestCase(SeleniumTestCase, UsesHistoryItemAssertions):
self.workflow_index_open()
self.workflow_index_click_option("Run")
self.screenshot("workflow_manage_run_simple")
self.workflow_run_submit()
self.history_panel_wait_for_hid_ok(2, allowed_force_refreshes=1)
@@ -39,3 +40,4 @@ class WorkflowRunTestCase(SeleniumTestCase, UsesHistoryItemAssertions):
self.sleep_for(self.wait_types.UX_TRANSITION)
# Check that this tool form contains a warning about different versions.
self.assert_warning_message(contains="different versions")
self.screenshot("workflow_manage_run_tool_upgrade")
test/unit/jobs/test_runner_local.py
+42 -5
View File
@@ -3,6 +3,8 @@ import threading
import time
from unittest import TestCase
import psutil
from galaxy import model
from galaxy.jobs import metrics
from galaxy.jobs.runners import local
@@ -83,18 +85,38 @@ class TestLocalJobRunner(TestCase, UsesApp, UsesTools):
t = threading.Thread(target=queue)
t.start()
while True:
if self.job_wrapper.external_id:
break
time.sleep(.01)
external_id = self.job_wrapper.external_id
external_id = self.job_wrapper.wait_for_external_id()
mock_job = bunch.Bunch(
get_external_output_metadata=lambda: None,
get_job_runner_external_id=lambda: str(external_id),
get_id=lambda: 1
)
assert psutil.pid_exists(external_id)
runner.stop_job(mock_job)
t.join(1)
assert not psutil.pid_exists(external_id)
def test_shutdown_no_jobs(self):
self.app.config.monitor_thread_join_timeout = 5
runner = local.LocalJobRunner(self.app, 1)
runner.shutdown()
def test_stopping_job_at_shutdown(self):
self.job_wrapper.command_line = '''python -c "import time; time.sleep(15)"'''
runner = local.LocalJobRunner(self.app, 1)
self.app.config.monitor_thread_join_timeout = 15
def queue():
runner.queue_job(self.job_wrapper)
t = threading.Thread(target=queue)
t.start()
external_id = self.job_wrapper.wait_for_external_id()
assert psutil.pid_exists(external_id)
runner.shutdown()
t.join(1)
assert not psutil.pid_exists(external_id)
assert "job terminated by Galaxy shutdown" in self.job_wrapper.fail_message
class MockJobWrapper(object):
@@ -125,6 +147,7 @@ class MockJobWrapper(object):
self.metadata_command = "touch %s" % self.mock_metadata_path
self.galaxy_virtual_env = None
self.shell = "/bin/bash"
self.cleanup_job = "never"
# Cruft for setting metadata externally, axe at some point.
self.external_output_metadata = bunch.Bunch(
@@ -134,6 +157,16 @@ class MockJobWrapper(object):
build_dependency_shell_commands=lambda: []
)
def wait_for_external_id(self):
"""Test method for waiting til an external id has been registered."""
external_id = None
for i in range(50):
external_id = self.external_id
if external_id:
break
time.sleep(.1)
return external_id
def prepare(self):
self.prepare_called = True
@@ -167,6 +200,10 @@ class MockJobWrapper(object):
def has_limits(self):
return False
def fail(self, message, exception):
self.fail_message = message
self.fail_exception = exception
def finish(self, stdout, stderr, exit_code):
self.stdout = stdout
self.stderr = stderr