diff --git a/.ci/flake8_lint_include_list.txt b/.ci/flake8_lint_include_list.txt index 4cb3e453782..6d90bc4ab6f 100644 --- a/.ci/flake8_lint_include_list.txt +++ b/.ci/flake8_lint_include_list.txt @@ -1,9 +1,6 @@ client/galaxy/style/source_material/circle.py contrib/ -cron/add_manual_builds.py -cron/cleanup_datasets.py -cron/parse_builds_3_sites.py -cron/parse_builds.py +cron/ doc/parse_gx_xsd.py lib/galaxy/actions/ lib/galaxy/auth/ @@ -77,6 +74,7 @@ lib/galaxy/visualization/genome/ lib/galaxy/visualization/genomes.py lib/galaxy/visualization/__init__.py lib/galaxy/visualization/plugins/__init__.py +lib/galaxy/visualization/plugins/plugin.py lib/galaxy/visualization/plugins/utils.py lib/galaxy/visualization/tracks/ lib/galaxy/web/base/ @@ -152,182 +150,7 @@ lib/log_tempfile.py lib/mimeparse.py lib/psyco_full.py lib/tool_shed/ -scripts/api/common.py -scripts/api/copy_hda_to_library_folder.py -scripts/api/create.py -scripts/api/delete.py -scripts/api/example_watch_folder.py -scripts/api/filter_failed_datasets_from_collection.py -scripts/api/form_create_from_xml.py -scripts/api/history_create_history.py -scripts/api/history_delete_history.py -scripts/api/import_library_dataset_to_history.py -scripts/api/import_workflows_from_installed_tool_shed_repository.py -scripts/api/install_tool_shed_repositories.py -scripts/api/library_create_folder.py -scripts/api/library_create_library.py -scripts/api/library_upload_dir.py -scripts/api/library_upload_from_import_dir.py -scripts/api/load_data_with_metadata.py -scripts/api/repair_tool_shed_repository.py -scripts/api/requests_update_state.py -scripts/api/request_type_create_from_xml.py -scripts/api/reset_metadata_on_installed_repositories.py -scripts/api/sample_dataset_update_status.py -scripts/api/sample_update_state.py -scripts/api/sequencer_configuration_create.py -scripts/api/update.py -scripts/api/upload_to_history.py -scripts/api/workflow_delete.py -scripts/api/workflow_execute_parameters.py -scripts/api/workflow_execute.py -scripts/api/workflow_import_from_file_rpark.py -scripts/api/workflow_import.py -scripts/auth/ -scripts/bootstrap_history.py -scripts/build_toolbox.py -scripts/build_universe_config.py -scripts/check_galaxy.py -scripts/check_python.py -scripts/cleanup_datasets/ -scripts/communication/ -scripts/create_db.py -scripts/data_libraries/ -scripts/db_shell.py -scripts/drmaa_external_killer.py -scripts/drmaa_external_runner.py -scripts/external_chown_script.py -scripts/extract_dataset_part.py -scripts/extract_toolbox_sections.py -scripts/functional_tests.py -scripts/get_platforms.py -scripts/helper.py -scripts/__init__.py -scripts/loc_files/ -scripts/manage_db.py -scripts/manage_tools.py -scripts/metagenomics/ -scripts/microbes/create_bacteria_loc_file.py -scripts/microbes/create_bacteria_table.py -scripts/microbes/create_nib_seq_loc_file.py -scripts/microbes/get_builds_lengths.py -scripts/microbes/ncbi_to_ucsc.py -scripts/microbes/util.py -scripts/migrate_tools/ -scripts/nosetests.py -scripts/others/ -scripts/runtime_stats.py -scripts/secret_decoder_ring.py -scripts/set_dataset_sizes.py -scripts/set_user_disk_usage.py -scripts/sync_reports_config.py -scripts/tool_shed/api/add_repository_registry_entry.py -scripts/tool_shed/api/common.py -scripts/tool_shed/api/create_categories.py -scripts/tool_shed/api/create_users.py -scripts/tool_shed/api/import_capsule.py -scripts/tool_shed/api/remove_repository_registry_entry.py -scripts/tool_shed/api/reset_metadata_on_repositories.py -scripts/tool_shed/api/tool_shed_repository_revision_update.py -scripts/tool_shed/build_ts_whoosh_index.py -scripts/tool_shed/check_download_urls.py -scripts/tool_shed/check_filesystem_for_empty_tool_dependency_installation_paths.py -scripts/tool_shed/clean_up_tool_dependency_directory.py -scripts/tool_shed/migrate_tools_to_repositories.py -scripts/tool_shed/show_tool_dependency_installation_dir_contents.py -scripts/tools/re_escape_output.py -test/api/__init__.py -test/api/test_dataset_collections.py -test/api/test_datasets.py -test/api/test_framework.py -test/api/test_history_contents_provenance.py -test/api/test_jobs.py -test/api/test_libraries.py -test/api/test_page_revisions.py -test/api/test_search.py -test/api/test_tool_data.py -test/api/test_tours.py -test/api/test_workflow_extraction.py -test/api/test_workflows_from_yaml.py -test/base/api_asserts.py -test/base/api.py -test/base/api_util.py -test/base/__init__.py -test/base/instrument.py -test/base/integration_util.py -test/base/interactor.py -test/base/nose_util.py -test/base/test_logging.py -test/base/tool_shed_util.py -test/base/twilltestcase.py -test/base/workflows_format_2/ -test/casperjs/ -test/functional/database_contexts.py -test/functional/__init__.py -test/functional/test_data_managers.py -test/functional/tool-data/ -test/functional/tools/ -test/integration/ -test/manual/__init__.py -test/shed_functional/base/common.py -test/shed_functional/base/__init__.py -test/shed_functional/functional/ -test/shed_functional/functional_tests.py -test/shed_functional/__init__.py -test/unit/dataset_collections/ -test/unit/datatypes/dataproviders/__init__.py -test/unit/datatypes/dataproviders/test_line_dataproviders.py -test/unit/datatypes/__init__.py -test/unit/datatypes/test_data.py -test/unit/jobs/dynamic_tool_destination/ -test/unit/jobs/__init__.py -test/unit/jobs/test_command_factory.py -test/unit/jobs/test_datasets.py -test/unit/jobs/test_job_configuration.py -test/unit/jobs/test_job_output_checker.py -test/unit/jobs/test_mapper.py -test/unit/jobs/test_rule_helper.py -test/unit/jobs/test_rules/ -test/unit/jobs/test_runner_params.py -test/unit/managers/__init__.py -test/unit/managers/test_CollectionManager.py -test/unit/managers/test_DatasetManager.py -test/unit/managers/test_HDAManager.py -test/unit/managers/test_HDCAManager.py -test/unit/managers/test_HistoryContentsManager.py -test/unit/managers/test_UserManager.py -test/unit/shed_unit/__init__.py -test/unit/shed_unit/test_fabric_util.py -test/unit/shed_unit/test_td_common_util.py -test/unit/test_galaxy_transactions.py -test/unit/test_lazy_process.py -test/unit/test_objectstore.py -test/unit/test_routes.py -test/unit/test_security_helper.py -test/unit/test_sockets.py -test/unit/test_sqlite_utils.py -test/unit/test_topsort.py -test/unit/tools/filter_modules/ -test/unit/tools/__init__.py -test/unit/tools/test_citations.py -test/unit/tools/test_conda_resolution.py -test/unit/tools/test_parsing.py -test/unit/tools/test_select_parameters.py -test/unit/tools/test_toolbox_filters.py -test/unit/tools/test_tool_dependency_description.py -test/unit/tools/test_tool_deps.py -test/unit/tools/test_tool_external_files.py -test/unit/tools/test_tool_loader.py -test/unit/tools/test_tool_panel.py -test/unit/tools/test_watcher.py -test/unit/tools/test_wrappers.py -test/unit/tools_support.py -test/unit/unittest_utils/ -test/unit/visualizations/__init__.py -test/unit/visualizations/plugins/__init__.py -test/unit/web/base/__init__.py -test/unit/web/framework/__init__.py -test/unit/web/__init__.py -test/unit/workflows/ +scripts/ +test/ tool_list.py tools/ diff --git a/CITATION b/CITATION index 2151602534b..2142df9d514 100644 --- a/CITATION +++ b/CITATION @@ -1,54 +1,31 @@ -If you use or extend Galaxy in your published work, please cite each of the -following publications: - -- Goecks, J, Nekrutenko, A, Taylor, J and The Galaxy Team. "Galaxy: a - comprehensive approach for supporting accessible, reproducible, and - transparent computational research in the life sciences." - Genome Biol. 2010 Aug 25;11(8):R86. - -- Blankenberg D, Von Kuster G, Coraor N, Ananda G, Lazarus R, Mangan M, - Nekrutenko A, Taylor J. "Galaxy: a web-based genome analysis tool for - experimentalists". Current Protocols in Molecular Biology. - 2010 Jan; Chapter 19:Unit 19.10.1-21. - -- Giardine B, Riemer C, Hardison RC, Burhans R, Elnitski L, Shah P, Zhang Y, - Blankenberg D, Albert I, Taylor J, Miller W, Kent WJ, Nekrutenko A. "Galaxy: - a platform for interactive large-scale genome analysis." - Genome Research. 2005 Oct; 15(10):1451-5. - -See also: https://galaxyproject.org/citing-galaxy - +If you use or extend Galaxy in your published work, please cite this +publication: +- Afgan E, Baker D, van den Beek M, Blankenberg D, Bouvier D, Čech M, Chilton + J, Clements D, Coraor N, Eberhard C, Grüning B, Guerler A, Hillman-Jackson + J, Von Kuster G, Rasche E, Soranzo N, Turaga N, Taylor J, Nekrutenko A, Goecks + J. "The Galaxy platform for accessible, reproducible and collaborative + biomedical analyses: 2016 update" + Nucleic Acids Research (2016) 44(W1):W3-W10 doi:10.1093/nar/gkw343 BibTeX format: -@article{goecks2010galaxy, - title={Galaxy: a comprehensive approach for supporting accessible, reproducible, and transparent computational research in the life sciences}, - author={Goecks, Jeremy and Nekrutenko, Anton and Taylor, James and The Galaxy Team}, - journal={Genome Biol}, - volume={11}, - number={8}, - pages={R86}, - year={2010} +@article{afgan2016galaxy, + title={The {Galaxy} platform for accessible, reproducible and collaborative biomedical analyses: 2016 update}, + author={Afgan, Enis and Baker, Dannon and van den Beek, Marius and Blankenberg, Daniel and Bouvier, Dave and \v{C}ech, Martin and Chilton, John and Clements, Dave and Coraor, Nate and Eberhard, Carl and Gr\"uning, Bj\"orn and Guerler, Aysam and Hillman-Jackson, Jennifer and Von Kuster, Greg and Rasche, Eric and Soranzo, Nicola and Turaga, Nitesh and Taylor, James and Nekrutenko, Anton and Goecks, Jeremy}, + journal={Nucleic Acids Res.}, + volume={44}, + number={W1}, + pages={W3-W10}, + year={2016}, + publisher={Oxford University Press} } -@article{blankenberg2010galaxy, - title={Galaxy: A Web-Based Genome Analysis Tool for Experimentalists}, - author={Blankenberg, Daniel and Kuster, Gregory Von and Coraor, Nathaniel and Ananda, Guruprasad and Lazarus, Ross and Mangan, Mary and Nekrutenko, Anton and Taylor, James}, - journal={Current protocols in molecular biology}, - pages={19--10}, - year={2010}, - publisher={John Wiley \& Sons, Inc.} -} -@article{giardine2005galaxy, - title={Galaxy: a platform for interactive large-scale genome analysis}, - author={Giardine, Belinda and Riemer, Cathy and Hardison, Ross C and Burhans, Richard and Elnitski, Laura and Shah, Prachi and Zhang, Yi and Blankenberg, Daniel and Albert, Istvan and Taylor, James and Miller, Webb C and Kent, W James and Nekrutenko, Anton}, - journal={Genome research}, - volume={15}, - number={10}, - pages={1451--1455}, - year={2005}, - publisher={Cold Spring Harbor Lab} -} +More publications +----------------- +To cite specific Galaxy components/features or other Galaxy project +publications, see + +https://galaxyproject.org/citing-galaxy diff --git a/cron/build_chrom_db.py b/cron/build_chrom_db.py index f6b2b8efc2f..60c79b27b9b 100644 --- a/cron/build_chrom_db.py +++ b/cron/build_chrom_db.py @@ -17,10 +17,10 @@ import fileinput import os import sys +import requests from six.moves.urllib.parse import urlencode -from six.moves.urllib.request import urlopen -import parse_builds +import parse_builds # noqa: I100 def getchrominfo(url, db): @@ -36,8 +36,8 @@ def getchrominfo(url, db): "hgta_regionType": "", "position": "", "hgta_doTopSubmit": "get info"}) - page = urlopen(URL) - for line in page: + page = requests.get(URL).text + for line in page.split('\n'): line = line.rstrip("\r\n") if line.startswith("#"): continue diff --git a/cron/parse_builds.py b/cron/parse_builds.py index c30197184f8..243440a331b 100644 --- a/cron/parse_builds.py +++ b/cron/parse_builds.py @@ -9,18 +9,17 @@ from __future__ import print_function import sys import xml.etree.ElementTree as ElementTree -from six.moves.urllib.request import urlopen +import requests def getbuilds(url): try: - page = urlopen(url) + text = requests.get(url).text except: print("#Unable to open " + url) print("?\tunspecified (?)") sys.exit(1) - text = page.read() try: tree = ElementTree.fromstring(text) except: diff --git a/cron/parse_builds_3_sites.py b/cron/parse_builds_3_sites.py index b22cd449bf9..4ae43e52765 100644 --- a/cron/parse_builds_3_sites.py +++ b/cron/parse_builds_3_sites.py @@ -6,7 +6,7 @@ from __future__ import print_function import xml.etree.ElementTree as ElementTree -from six.moves.urllib.request import urlopen +import requests sites = ['http://genome.ucsc.edu/cgi-bin/', 'http://archaea.ucsc.edu/cgi-bin/', @@ -20,11 +20,11 @@ def main(): trackurl = sites[i] + "hgTracks?" builds = [] try: - page = urlopen(site) + text = requests.get(site).text except: print("#Unable to connect to " + site) continue - text = page.read() + try: tree = ElementTree.fromstring(text) except: diff --git a/lib/galaxy/containers/__init__.py b/lib/galaxy/containers/__init__.py index 4b33dbe8d5a..6cebdd82e14 100644 --- a/lib/galaxy/containers/__init__.py +++ b/lib/galaxy/containers/__init__.py @@ -306,7 +306,7 @@ def parse_containers_config(containers_config_file): conf = DEFAULT_CONF.copy() try: with open(containers_config_file) as fh: - c = yaml.load(fh) + c = yaml.safe_load(fh) conf.update(c.get('containers', {})) except (OSError, IOError) as exc: if exc.errno == errno.ENOENT: diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index b4ab524b778..87cf9aeee92 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -252,12 +252,8 @@ class Bam(Binary): message = 'Attempting to use functionality requiring samtools, but it cannot be located on Galaxy\'s PATH.' raise Exception(message) - # Get the version of samtools via --version-only, if available - p = subprocess.Popen(['samtools', '--version-only'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p = subprocess.Popen(['samtools', '--version-only'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = p.communicate() - # --version-only is available # Format is +htslib- if p.returncode == 0: @@ -294,10 +290,8 @@ class Bam(Binary): def _is_coordinate_sorted(self, file_name): """See if the input BAM file is sorted from the header information.""" - params = ["samtools", "view", "-H", file_name] - output = subprocess.Popen(params, stderr=subprocess.PIPE, stdout=subprocess.PIPE).communicate()[0] - # find returns -1 if string is not found - return output.find("SO:coordinate") != -1 or output.find("SO:sorted") != -1 + output = subprocess.check_output(["samtools", "view", "-H", file_name]) + return 'SO:coordinate' in output or 'SO:sorted' in output def dataset_content_needs_grooming(self, file_name): """See if file_name is a sorted BAM file""" @@ -322,8 +316,7 @@ class Bam(Binary): return False index_name = tempfile.NamedTemporaryFile(prefix="bam_index").name stderr_name = tempfile.NamedTemporaryFile(prefix="bam_index_stderr").name - command = 'samtools index %s %s' % (file_name, index_name) - proc = subprocess.Popen(args=command, shell=True, stderr=open(stderr_name, 'wb')) + proc = subprocess.Popen(['samtools', 'index', file_name, index_name], stderr=open(stderr_name, 'wb')) proc.wait() stderr = open(stderr_name).read().strip() if stderr: @@ -366,8 +359,8 @@ class Bam(Binary): tmp_sorted_dataset_file_name_prefix = os.path.join(tmp_dir, 'sorted') stderr_name = tempfile.NamedTemporaryFile(dir=tmp_dir, prefix="bam_sort_stderr").name samtools_created_sorted_file_name = "%s.bam" % tmp_sorted_dataset_file_name_prefix # samtools accepts a prefix, not a filename, it always adds .bam to the prefix - command = "samtools sort %s %s" % (file_name, tmp_sorted_dataset_file_name_prefix) - proc = subprocess.Popen(args=command, shell=True, cwd=tmp_dir, stderr=open(stderr_name, 'wb')) + proc = subprocess.Popen(['samtools', 'sort', file_name, tmp_sorted_dataset_file_name_prefix], + cwd=tmp_dir, stderr=open(stderr_name, 'wb')) exit_code = proc.wait() # Did sort succeed? stderr = open(stderr_name).read().strip() @@ -1309,11 +1302,8 @@ class ExcelXls(Binary): edam_format = "format_3468" def sniff(self, filename): - mime_type = subprocess.check_output("file --mime-type '{}'".format(filename), shell=True).rstrip() - if mime_type.find("application/vnd.ms-excel") != -1: - return True - else: - return False + mime_type = subprocess.check_output(['file', '--mime-type', filename]) + return "application/vnd.ms-excel" in mime_type def get_mime(self): """Returns the mime type of the datatype""" diff --git a/lib/galaxy/datatypes/converters/interval_to_coverage.py b/lib/galaxy/datatypes/converters/interval_to_coverage.py index b0b8bb4695c..d4ddd98927b 100644 --- a/lib/galaxy/datatypes/converters/interval_to_coverage.py +++ b/lib/galaxy/datatypes/converters/interval_to_coverage.py @@ -133,8 +133,9 @@ if __name__ == "__main__": # Sort through a tempfile first temp_file = tempfile.NamedTemporaryFile(mode="r") environ['LC_ALL'] = 'POSIX' - commandline = "sort -f -n -k %d -k %d -k %d -o %s %s" % (chr_col_1 + 1, start_col_1 + 1, end_col_1 + 1, temp_file.name, in_fname) - subprocess.check_call(commandline, shell=True) + subprocess.check_call([ + 'sort', '-f', '-n', '-k', chr_col_1 + 1, '-k', start_col_1 + 1, '-k', end_col_1 + 1, '-o', temp_file.name, in_fname + ]) coverage = CoverageWriter(out_stream=open(out_fname, "a"), chromCol=chr_col_2, positionCol=position_col_2, diff --git a/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py b/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py index 777259c3e0e..83b8050e46a 100644 --- a/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py +++ b/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py @@ -72,9 +72,9 @@ def rgConv(inpedfilepath, outhtmlname, outfilepath, plink): if not missval: print('### lped_to_pbed_converter.py cannot identify missing value in %s' % pedf) missval = '0' - cl = '%s --noweb --file %s --make-bed --out %s --missing-genotype %s' % (plink, inpedfilepath, outroot, missval) - p = subprocess.Popen(cl, shell=True, cwd=outfilepath) - p.wait() # run plink + subprocess.check_call([plink, '--noweb', '--file', inpedfilepath, + '--make-bed', '--out', outroot, + '--missing-genotype', missval], cwd=outfilepath) def main(): diff --git a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py index 86c4a4d051f..fbe1834b8b1 100644 --- a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py +++ b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py @@ -41,8 +41,7 @@ def pruneLD(plinktasks=[], cd='./', vclbase=[]): for task in plinktasks: # each is a list vcl = vclbase + task with open(plog, 'w') as sto: - x = subprocess.Popen(' '.join(vcl), shell=True, stdout=sto, stderr=sto, cwd=cd) - x.wait() + subprocess.check_call(vcl, stdout=sto, stderr=sto, cwd=cd) try: lplog = open(plog, 'r').readlines() lplog = [elem for elem in lplog if elem.find('Pruning SNP') == -1] diff --git a/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py b/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py index 023c6ee5f30..24a052384a3 100644 --- a/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py +++ b/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py @@ -40,9 +40,7 @@ def rgConv(inpedfilepath, outhtmlname, outfilepath, plink): """ basename = os.path.split(inpedfilepath)[-1] # get basename outroot = os.path.join(outfilepath, basename) - cl = '%s --noweb --bfile %s --recode --out %s ' % (plink, inpedfilepath, outroot) - p = subprocess.Popen(cl, shell=True, cwd=outfilepath) - p.wait() # run plink + subprocess.check_call([plink, '--noweb', '--bfile', inpedfilepath, '--recode', '--out', outroot], cwd=outfilepath) def main(): diff --git a/lib/galaxy/datatypes/converters/sam_to_bam.py b/lib/galaxy/datatypes/converters/sam_to_bam.py index dcc84b6c518..8c54258eb9a 100644 --- a/lib/galaxy/datatypes/converters/sam_to_bam.py +++ b/lib/galaxy/datatypes/converters/sam_to_bam.py @@ -34,11 +34,8 @@ def _get_samtools_version(): if not cmd_exists('samtools'): raise Exception('This tool needs samtools, but it is not on PATH.') # Get the version of samtools via --version-only, if available - p = subprocess.Popen(['samtools', '--version-only'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p = subprocess.Popen(['samtools', '--version-only'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = p.communicate() - # --version-only is available # Format is +htslib- if p.returncode == 0: diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 31902b14513..f11b8498719 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -359,7 +359,7 @@ class Registry(object): build_sites_config_file = getattr(self.config, "build_sites_config_file", None) if build_sites_config_file and os.path.exists(build_sites_config_file): with open(build_sites_config_file, "r") as f: - build_sites_config = yaml.load(f) + build_sites_config = yaml.safe_load(f) if not isinstance(build_sites_config, list): self.log.exception("Build sites configuration YAML file does not declare list of sites.") return diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 1267ff3da4a..ee47ad70bbd 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -7,6 +7,7 @@ import logging import os import re import string +import subprocess import sys from cgi import escape from itertools import islice @@ -693,8 +694,7 @@ class BaseFastq (Sequence): else: commands = Sequence.get_split_commands_sequential(is_gzip(input_name), input_name, output_name, start_sequence, sequence_count) for cmd in commands: - if 0 != os.system(cmd): - raise Exception("Executing '%s' failed" % cmd) + subprocess.check_call(cmd, shell=True) return True process_split_file = staticmethod(process_split_file) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 0100448e65a..234a436a648 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -521,15 +521,12 @@ class Sam(Tabular): Multiple SAM files may each have headers. Since the headers should all be the same, remove the headers from files 1-n, keeping them in the first file only """ - cmd = 'mv %s %s' % (split_files[0], output_file) - result = os.system(cmd) - if result != 0: - raise Exception('Result %s from %s' % (result, cmd)) + shutil.move(split_files[0], output_file) + if len(split_files) > 1: - cmd = 'egrep -v -h "^@" %s >> %s' % (' '.join(split_files[1:]), output_file) - result = os.system(cmd) - if result != 0: - raise Exception('Result %s from %s' % (result, cmd)) + cmd = ['egrep', '-v', '-h', '^@'] + split_files[1:] + ['>>', output_file] + subprocess.check_call(cmd, shell=True) + merge = staticmethod(merge) # Dataproviders diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index 4d71302b3f0..12235e566c4 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -10,6 +10,8 @@ import re import subprocess import tempfile +from six.moves import shlex_quote + from galaxy.datatypes.data import get_file_peek, Text from galaxy.datatypes.metadata import MetadataElement, MetadataParameter from galaxy.datatypes.sniff import iter_headers @@ -148,13 +150,12 @@ class Ipynb(Json): ofilename = ofile_handle.name ofile_handle.close() try: - cmd = 'jupyter nbconvert --to html --template full %s --output %s' % (dataset.file_name, ofilename) - log.info("Calling command %s" % cmd) - subprocess.call(cmd, shell=True) + cmd = ['jupyter', 'nbconvert', '--to', 'html', '--template', 'full', dataset.file_name, '--output', ofilename] + subprocess.check_call(cmd) ofilename = '%s.html' % ofilename - except: + except subprocess.CalledProcessError: ofilename = dataset.file_name - log.exception('Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.', cmd) + log.exception('Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.', ' '.join(map(shlex_quote, cmd))) return open(ofilename) def set_meta(self, dataset, **kwd): diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 4c4f7601c3b..80b2bcffa29 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -57,7 +57,7 @@ anyjson==0.3.3 # Pulsar requirements psutil==4.1.0 -pulsar-galaxy-lib==0.7.0.dev5 +pulsar-galaxy-lib==0.8.0 # sqlalchemy-migrate and dependencies sqlalchemy-migrate==0.10.0 diff --git a/lib/galaxy/external_services/actions.py b/lib/galaxy/external_services/actions.py index 69fd20ca57b..13dc83703d3 100644 --- a/lib/galaxy/external_services/actions.py +++ b/lib/galaxy/external_services/actions.py @@ -1,6 +1,6 @@ # Contains actions that are used in External Services import logging -from urllib import urlopen +import requests from galaxy.web import url_for from galaxy.util.template import fill_template from result_handlers.basic import ExternalServiceActionResultHandler @@ -104,7 +104,7 @@ class ExternalServiceWebAPIActionResult(ExternalServiceResult): @property def content(self): if self._content is None: - self._content = urlopen(self.url).read() + self._content = requests.get(self.url).text return self._content diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index c910c7dca2a..08457df9aa2 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -884,6 +884,9 @@ class JobWrapper(object, HasResourceParameters): self.galaxy_lib_dir # Shell fragment to inject dependencies self.dependency_shell_commands = self.tool.build_dependency_shell_commands(job_directory=self.working_directory) + if self.tool.requires_galaxy_python_environment: + # These tools (upload, metadata, data_source) may need access to the datatypes registry. + self.app.datatypes_registry.to_xml_file(os.path.join(self.working_directory, 'registry.xml')) # We need command_line persisted to the db in order for Galaxy to re-queue the job # if the server was stopped and restarted before the job finished job.command_line = unicodify(self.command_line) @@ -1568,6 +1571,9 @@ class JobWrapper(object, HasResourceParameters): paths.append(DatasetPath(da.id, real_path=real_path, false_path=false_path, mutable=False)) return paths + def get_output_basenames(self): + return map(os.path.basename, map(str, self.get_output_fnames())) + def get_output_fnames(self): if self.output_paths is None: self.compute_outputs() @@ -1645,7 +1651,8 @@ class JobWrapper(object, HasResourceParameters): def setup_external_metadata(self, exec_dir=None, tmp_dir=None, dataset_files_path=None, config_root=None, - config_file=None, resolve_metadata_dependencies=False, + config_file=None, datatypes_config=None, + resolve_metadata_dependencies=False, set_extension=True, **kwds): # extension could still be 'auto' if this is the upload tool. job = self.get_job() @@ -1664,8 +1671,9 @@ class JobWrapper(object, HasResourceParameters): config_root = self.app.config.root if config_file is None: config_file = self.app.config.config_file - datatypes_config = os.path.join(self.working_directory, 'registry.xml') - self.app.datatypes_registry.to_xml_file(path=datatypes_config) + if datatypes_config is None: + datatypes_config = os.path.join(self.working_directory, 'registry.xml') + self.app.datatypes_registry.to_xml_file(path=datatypes_config) command = self.external_output_metadata.setup_external_metadata([output_dataset_assoc.dataset for output_dataset_assoc in job.output_datasets + job.output_library_datasets], @@ -1974,7 +1982,7 @@ class TaskWrapper(JobWrapper): pass def setup_external_metadata(self, exec_dir=None, tmp_dir=None, dataset_files_path=None, - config_root=None, config_file=None, + config_root=None, config_file=None, datatypes_config=None, set_extension=True, **kwds): # There is no metadata setting for tasks. This is handled after the merge, at the job level. return "" @@ -1994,6 +2002,10 @@ class ComputeEnvironment(object): compute server. """ + @abstractmethod + def output_names(self): + """ Output unqualified filenames defined by job. """ + @abstractmethod def output_paths(self): """ Output DatasetPaths defined by job. """ @@ -2059,6 +2071,9 @@ class SharedComputeEnvironment(SimpleComputeEnvironment): self.job_wrapper = job_wrapper self.job = job + def output_names(self): + return self.job_wrapper.get_output_basenames() + def output_paths(self): return self.job_wrapper.get_output_fnames() diff --git a/lib/galaxy/jobs/actions/post.py b/lib/galaxy/jobs/actions/post.py index 2f7402f399c..545328bbeb1 100644 --- a/lib/galaxy/jobs/actions/post.py +++ b/lib/galaxy/jobs/actions/post.py @@ -308,7 +308,6 @@ class DeleteIntermediatesAction(DefaultJobAction): safe_to_delete = True for job_to_check in [d_j.job for d_j in input_dataset.dependent_jobs]: if job_to_check != job and job_to_check.state not in [job.states.OK, job.states.DELETED]: - log.debug("Workflow Intermediates cleanup attempted, but non-terminal state '%s' detected for job %s" % (job_to_check.state, job_to_check.id)) safe_to_delete = False if safe_to_delete: # Support purging here too. diff --git a/lib/galaxy/jobs/command_factory.py b/lib/galaxy/jobs/command_factory.py index abf1a0e84de..7922531ee8b 100644 --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -17,19 +17,7 @@ log = getLogger(__name__) CAPTURE_RETURN_CODE = "return_code=$?" YIELD_CAPTURED_CODE = 'sh -c "exit $return_code"' SETUP_GALAXY_FOR_METADATA = """ -if [ "$GALAXY_LIB" != "None" ]; then - if [ -n "$PYTHONPATH" ]; then - PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" - else - PYTHONPATH="$GALAXY_LIB" - fi - export PYTHONPATH -fi -if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" \ - -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then - . "$GALAXY_VIRTUAL_ENV/bin/activate" -fi -GALAXY_PYTHON=`command -v python` +[ "$GALAXY_VIRTUAL_ENV" = "None" ] && GALAXY_VIRTUAL_ENV="$_GALAXY_VIRTUAL_ENV"; _galaxy_setup_environment True """ @@ -198,6 +186,7 @@ def __handle_metadata(commands_builder, job_wrapper, runner, remote_command_para output_fnames = metadata_kwds.get('output_fnames', job_wrapper.get_output_fnames()) config_root = metadata_kwds.get('config_root', None) config_file = metadata_kwds.get('config_file', None) + datatypes_config = metadata_kwds.get('datatypes_config', None) compute_tmp_dir = metadata_kwds.get('compute_tmp_dir', None) resolve_metadata_dependencies = job_wrapper.commands_in_new_shell metadata_command = job_wrapper.setup_external_metadata( @@ -208,6 +197,7 @@ def __handle_metadata(commands_builder, job_wrapper, runner, remote_command_para set_extension=False, config_root=config_root, config_file=config_file, + datatypes_config=datatypes_config, compute_tmp_dir=compute_tmp_dir, resolve_metadata_dependencies=resolve_metadata_dependencies, kwds={'overwrite': False} diff --git a/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py b/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py index a0e2707c951..b90f9940bcd 100644 --- a/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py +++ b/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py @@ -2,11 +2,10 @@ Module for managing jobs in Pacific Bioscience's SMRT Portal and automatically transferring files produced by SMRT Portal. """ -import json import logging from string import Template -from six.moves.urllib.request import urlopen +import requests from .data_transfer import DataTransfer @@ -88,8 +87,8 @@ class SMRTPortalPlugin(DataTransfer): if self._missing_params(job.params, ['smrt_host', 'smrt_job_id']): return self.job_states.INVALID url = 'http://' + job.params['smrt_host'] + self.api_path + '/Jobs/' + job.params['smrt_job_id'] + '/Status' - r = urlopen(url) - status = json.loads(r.read()) + r = requests.get(url) + status = r.json() # TODO: error handling: unexpected json or bad response, bad url, etc. if status['Code'] == 'Completed': log.debug("SMRT Portal job '%s' is Completed. Initiating transfer." % job.params['smrt_job_id']) diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 9722ea32736..bc201e74626 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -7,6 +7,7 @@ from __future__ import absolute_import # Need to import pulsar_client absolutel import errno import logging import os +import subprocess from distutils.version import LooseVersion from time import sleep @@ -51,7 +52,10 @@ __all__ = ( 'PulsarEmbeddedJobRunner', ) -MINIMUM_PULSAR_VERSION = LooseVersion("0.7.0.dev3") +MINIMUM_PULSAR_VERSIONS = { + '_default_': LooseVersion("0.7.0.dev3"), + 'remote_metadata': LooseVersion("0.8.0"), +} NO_REMOTE_GALAXY_FOR_METADATA_MESSAGE = "Pulsar misconfiguration - Pulsar client configured to set metadata remotely, but remote Pulsar isn't properly configured with a galaxy_home directory." NO_REMOTE_DATATYPES_CONFIG = "Pulsar client is configured to use remote datatypes configuration when setting metadata externally, but Pulsar is not configured with this information. Defaulting to datatypes_conf.xml." @@ -59,7 +63,7 @@ GENERIC_REMOTE_ERROR = "Failed to communicate with remote job server." FAILED_REMOTE_ERROR = "Remote job server indicated a problem running or monitoring this job." LOST_REMOTE_ERROR = "Remote job server could not determine this job's state." -UPGRADE_PULSAR_ERROR = "Galaxy is misconfigured, please contact administrator. The target Pulsar server is unsupported, this version of Galaxy requires Pulsar version %s or newer." % MINIMUM_PULSAR_VERSION +UPGRADE_PULSAR_ERROR = "Galaxy is misconfigured, please contact administrator. The target Pulsar server is unsupported, this version of Galaxy requires Pulsar version %s or newer." # Is there a good way to infer some default for this? Can only use # url_for from web threads. https://gist.github.com/jmchilton/9098762 @@ -217,7 +221,7 @@ class PulsarJobRunner(AsynchronousJobRunner): else: log.info("Loading Pulsar app configuration from %s" % pulsar_conf_path) with open(pulsar_conf_path, "r") as f: - conf.update(yaml.load(f) or {}) + conf.update(yaml.safe_load(f) or {}) if "job_metrics_config_file" not in conf: conf["job_metrics"] = self.app.job_metrics if "staging_directory" not in conf: @@ -277,8 +281,10 @@ class PulsarJobRunner(AsynchronousJobRunner): dependencies_description = PulsarJobRunner.__dependencies_description(client, job_wrapper) rewrite_paths = not PulsarJobRunner.__rewrite_parameters(client) unstructured_path_rewrites = {} + output_names = [] if compute_environment: unstructured_path_rewrites = compute_environment.unstructured_path_rewrites + output_names = compute_environment.output_names() client_job_description = ClientJobDescription( command_line=command_line, @@ -292,6 +298,7 @@ class PulsarJobRunner(AsynchronousJobRunner): env=client.env, rewrite_paths=rewrite_paths, arbitrary_files=unstructured_path_rewrites, + touch_outputs=output_names, ) job_id = pulsar_submit_job(client, client_job_description, remote_job_config) log.info("Pulsar job submitted with job_id %s" % job_id) @@ -310,6 +317,11 @@ class PulsarJobRunner(AsynchronousJobRunner): pulsar_job_state.job_destination = job_destination self.monitor_job(pulsar_job_state) + def __needed_features(self, client): + return { + 'remote_metadata': PulsarJobRunner.__remote_metadata(client), + } + def __prepare_job(self, job_wrapper, job_destination): """Build command-line and Pulsar client for this job.""" command_line = None @@ -319,8 +331,9 @@ class PulsarJobRunner(AsynchronousJobRunner): try: client = self.get_client_from_wrapper(job_wrapper) tool = job_wrapper.tool - remote_job_config = client.setup(tool.id, tool.version) - PulsarJobRunner.check_job_config(remote_job_config) + remote_job_config = client.setup(tool.id, tool.version, tool.requires_galaxy_python_environment) + needed_features = self.__needed_features(client) + PulsarJobRunner.check_job_config(remote_job_config, check_features=needed_features) rewrite_parameters = PulsarJobRunner.__rewrite_parameters(client) prepare_kwds = {} if rewrite_parameters: @@ -382,8 +395,7 @@ class PulsarJobRunner(AsynchronousJobRunner): prepare_input_files_cmds = getattr(job_wrapper, 'prepare_input_files_cmds', None) if prepare_input_files_cmds is not None: for cmd in prepare_input_files_cmds: # run the commands to stage the input files - if 0 != os.system(cmd): - raise Exception('Error running file staging command: %s' % cmd) + subprocess.check_call(cmd, shell=True) job_wrapper.prepare_input_files_cmds = None # prevent them from being used in-line def _populate_parameter_defaults(self, job_destination): @@ -593,12 +605,17 @@ class PulsarJobRunner(AsynchronousJobRunner): return client_outputs @staticmethod - def check_job_config(remote_job_config): + def check_job_config(remote_job_config, check_features=None): + check_features = check_features or {} # 0.6.0 was newest Pulsar version that did not report it's version. pulsar_version = LooseVersion(remote_job_config.get('pulsar_version', "0.6.0")) + needed_version = LooseVersion("0.0.0") log.info("pulsar_version is %s" % pulsar_version) - if pulsar_version < MINIMUM_PULSAR_VERSION: - raise UnsupportedPulsarException() + for feature in list(check_features.keys()) + ['_default_']: + if pulsar_version < MINIMUM_PULSAR_VERSIONS[feature]: + needed_version = max(needed_version, MINIMUM_PULSAR_VERSIONS[feature]) + if pulsar_version < needed_version: + raise UnsupportedPulsarException(needed_version) @staticmethod def __dependencies_description(pulsar_client, job_wrapper): @@ -656,7 +673,6 @@ class PulsarJobRunner(AsynchronousJobRunner): raise Exception(NO_REMOTE_GALAXY_FOR_METADATA_MESSAGE) metadata_kwds['exec_dir'] = remote_galaxy_home outputs_directory = remote_job_config['outputs_directory'] - configs_directory = remote_job_config['configs_directory'] working_directory = remote_job_config['working_directory'] metadata_directory = remote_job_config['metadata_directory'] # For metadata calculation, we need to build a list of of output @@ -687,7 +703,7 @@ class PulsarJobRunner(AsynchronousJobRunner): remote_datatypes_config = os.path.join(remote_galaxy_home, 'datatypes_conf.xml') metadata_kwds['datatypes_config'] = remote_datatypes_config else: - datatypes_config = os.path.join(configs_directory, 'registry.xml') + datatypes_config = os.path.join(job_wrapper.working_directory, 'registry.xml') self.app.datatypes_registry.to_xml_file(path=datatypes_config) # Ensure this file gets pushed out to the remote config dir. job_wrapper.extra_filenames.append(datatypes_config) @@ -783,6 +799,10 @@ class PulsarComputeEnvironment(ComputeEnvironment): version_path = new_version_path self._version_path = version_path + def output_names(self): + # Maybe this should use the path mapper, but the path mapper just uses basenames + return self.job_wrapper.get_output_basenames() + def output_paths(self): local_output_paths = self._wrapper_output_paths @@ -854,5 +874,5 @@ class PulsarComputeEnvironment(ComputeEnvironment): class UnsupportedPulsarException(Exception): - def __init__(self): - super(UnsupportedPulsarException, self).__init__(UPGRADE_PULSAR_ERROR) + def __init__(self, needed): + super(UnsupportedPulsarException, self).__init__(UPGRADE_PULSAR_ERROR % needed) diff --git a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh index 801df6acbc1..68da1b3e362 100644 --- a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh +++ b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh @@ -1,25 +1,33 @@ #!$shell $headers + +_galaxy_setup_environment() { + local _use_framework_galaxy="$1" + if [ "$GALAXY_LIB" != "None" -a "$_use_framework_galaxy" = "True" ]; then + if [ -n "$PYTHONPATH" ]; then + PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" + else + PYTHONPATH="$GALAXY_LIB" + fi + export PYTHONPATH + fi + $env_setup_commands + if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" \ + -a "`command -v python`" != "$GALAXY_VIRTUAL_ENV/bin/python" ]; then + . "$GALAXY_VIRTUAL_ENV/bin/activate" + fi +} + $integrity_injection $slots_statement export GALAXY_SLOTS +GALAXY_VIRTUAL_ENV="$galaxy_virtual_env" +_GALAXY_VIRTUAL_ENV="$galaxy_virtual_env" PRESERVE_GALAXY_ENVIRONMENT="$preserve_python_environment" GALAXY_LIB="$galaxy_lib" -if [ "$GALAXY_LIB" != "None" -a "$PRESERVE_GALAXY_ENVIRONMENT" = "True" ]; then - if [ -n "$PYTHONPATH" ]; then - PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" - else - PYTHONPATH="$GALAXY_LIB" - fi - export PYTHONPATH -fi -$env_setup_commands -GALAXY_VIRTUAL_ENV="$galaxy_virtual_env" -if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" \ - -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" -a "$PRESERVE_GALAXY_ENVIRONMENT" = "True" ]; then - . "$GALAXY_VIRTUAL_ENV/bin/activate" -fi +_galaxy_setup_environment "$PRESERVE_GALAXY_ENVIRONMENT" +GALAXY_PYTHON=`command -v python` $instrument_pre_commands cd $working_directory $command diff --git a/lib/galaxy/jobs/runners/util/job_script/__init__.py b/lib/galaxy/jobs/runners/util/job_script/__init__.py index 77af249e5c4..2c717c8f13d 100644 --- a/lib/galaxy/jobs/runners/util/job_script/__init__.py +++ b/lib/galaxy/jobs/runners/util/job_script/__init__.py @@ -119,9 +119,8 @@ def _handle_script_integrity(path, config): sleep_amt = getattr(config, "check_job_script_integrity_sleep", DEFAULT_INTEGRITY_SLEEP) for i in range(count): try: - proc = subprocess.Popen([path], shell=True, env={"ABC_TEST_JOB_SCRIPT_INTEGRITY_XYZ": "1"}) - proc.wait() - if proc.returncode == 42: + returncode = subprocess.call([path], env={"ABC_TEST_JOB_SCRIPT_INTEGRITY_XYZ": "1"}) + if returncode == 42: script_integrity_verified = True break diff --git a/lib/galaxy/jobs/runners/util/kill.py b/lib/galaxy/jobs/runners/util/kill.py index c0dbd913b13..1a458c63913 100644 --- a/lib/galaxy/jobs/runners/util/kill.py +++ b/lib/galaxy/jobs/runners/util/kill.py @@ -1,6 +1,6 @@ import os +import subprocess from platform import system -from subprocess import Popen from time import sleep try: @@ -41,8 +41,8 @@ def _stock_kill_pid(pid): def __kill_windows(pid): try: - Popen("taskkill /F /T /PID %i" % pid, shell=True) - except Exception: + subprocess.check_call(['taskkill', '/F', '/T', '/PID', pid]) + except subprocess.CalledProcessError: pass diff --git a/lib/galaxy/jobs/transfer_manager.py b/lib/galaxy/jobs/transfer_manager.py index 01acd6aae8f..8e06024832c 100644 --- a/lib/galaxy/jobs/transfer_manager.py +++ b/lib/galaxy/jobs/transfer_manager.py @@ -9,6 +9,8 @@ import socket import subprocess import threading +from six.moves import shlex_quote + from galaxy.util import listify, sleeper from galaxy.util.json import jsonrpc_request, validate_jsonrpc_response @@ -23,7 +25,7 @@ class TransferManager(object): def __init__(self, app): self.app = app self.sa_session = app.model.context.current - self.command = 'python %s' % os.path.abspath(os.path.join(os.getcwd(), 'scripts', 'transfer.py')) + self.command = ['python', os.path.abspath(os.path.join(os.getcwd(), 'scripts', 'transfer.py'))] if app.config.get_bool('enable_job_recovery', True): # Only one Galaxy server process should be able to recover jobs! (otherwise you'll have nasty race conditions) self.running = True @@ -68,9 +70,9 @@ class TransferManager(object): # The transfer script should daemonize fairly quickly - if this is # not the case, this process will need to be moved to a # non-blocking method. - cmd = '%s %s' % (self.command, tj.id) - log.debug('Transfer command is: %s' % cmd) - p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + cmd = self.command + [tj.id] + log.debug('Transfer command is: %s', ' '.join(map(shlex_quote, cmd))) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) p.wait() output = p.stdout.read(32768) if p.returncode != 0: diff --git a/lib/galaxy/managers/citations.py b/lib/galaxy/managers/citations.py index 7b88b2a3728..5ff70e7c666 100644 --- a/lib/galaxy/managers/citations.py +++ b/lib/galaxy/managers/citations.py @@ -1,6 +1,6 @@ import functools import os -import urllib2 +import requests from beaker.cache import CacheManager from beaker.util import parse_cache_config_options @@ -47,10 +47,8 @@ class DoiCache(object): def _raw_get_bibtex(self, doi): dx_url = "http://dx.doi.org/" + doi headers = {'Accept': 'text/bibliography; style=bibtex, application/x-bibtex'} - req = urllib2.Request(dx_url, data="", headers=headers) - response = urllib2.urlopen(req) - bibtex = response.read() - return bibtex + req = requests.get(dx_url, headers=headers) + return req.text def get_bibtex(self, doi): createfunc = functools.partial(self._raw_get_bibtex, doi) diff --git a/lib/galaxy/model/metadata.py b/lib/galaxy/model/metadata.py index a1dd075ffd8..fa6e9b3537a 100644 --- a/lib/galaxy/model/metadata.py +++ b/lib/galaxy/model/metadata.py @@ -836,7 +836,7 @@ class JobExternalOutputMetadataWrapper(object): sa_session.add(metadata_files) sa_session.flush() metadata_files_list.append(metadata_files) - args = '"%s" "%s" %s %s' % (datatypes_config, + args = '"%s" "%s" %s %s' % (metadata_path_on_compute(datatypes_config), job_metadata, " ".join(map(__metadata_files_list_to_cmd_line, metadata_files_list)), max_metadata_value_size) diff --git a/lib/galaxy/objectstore/s3.py b/lib/galaxy/objectstore/s3.py index f832688741e..34a2fc39d74 100644 --- a/lib/galaxy/objectstore/s3.py +++ b/lib/galaxy/objectstore/s3.py @@ -18,6 +18,7 @@ from galaxy.util import ( safe_relpath, string_as_bool, umask_fix_perms, + which, ) from galaxy.util.sleeper import Sleeper @@ -67,10 +68,9 @@ class S3ObjectStore(ObjectStore): self.cache_monitor_thread.start() log.info("Cache cleaner manager started") # Test if 'axel' is available for parallel download and pull the key into cache - try: - subprocess.call('axel') + if which('axel'): self.use_axel = True - except OSError: + else: self.use_axel = False def _configure_connection(self): @@ -341,7 +341,7 @@ class S3ObjectStore(ObjectStore): log.debug("Parallel pulled key '%s' into cache to %s", rel_path, self._get_cache_path(rel_path)) ncores = multiprocessing.cpu_count() url = key.generate_url(7200) - ret_code = subprocess.call("axel -a -n %s '%s'" % (ncores, url)) + ret_code = subprocess.call(['axel', '-a', '-n', ncores, url]) if ret_code == 0: return True else: diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py index 92aa6c995ed..d3ec66072b7 100644 --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -16,7 +16,7 @@ import time from glob import glob from tempfile import NamedTemporaryFile -from six.moves.urllib.request import urlopen +import requests from galaxy import util from galaxy.util.dictifiable import Dictifiable @@ -340,7 +340,7 @@ class TabularToolDataTable(ToolDataTable, Dictifiable): if filename: tmp_file = NamedTemporaryFile(prefix='TTDT_URL_%s-' % self.name) try: - tmp_file.write(urlopen(filename, timeout=url_timeout).read()) + tmp_file.write(requests.get(filename, timeout=url_timeout).text) except Exception as e: log.error('Error loading Data Table URL "%s": %s', filename, e) continue diff --git a/lib/galaxy/tools/deps/conda_compat.py b/lib/galaxy/tools/deps/conda_compat.py index c45573ef49d..10527bf890d 100644 --- a/lib/galaxy/tools/deps/conda_compat.py +++ b/lib/galaxy/tools/deps/conda_compat.py @@ -61,7 +61,7 @@ def _render_jinja2(recipe_dir): @_Memoized def yamlize(data): - res = yaml.load(data) + res = yaml.safe_load(data) # ensure the result is a dict if res is None: res = {} diff --git a/lib/galaxy/tools/deps/container_resolvers/mulled.py b/lib/galaxy/tools/deps/container_resolvers/mulled.py index fc551cb883a..cf215a8f742 100644 --- a/lib/galaxy/tools/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tools/deps/container_resolvers/mulled.py @@ -3,6 +3,7 @@ import collections import logging import os +import subprocess import six @@ -11,7 +12,6 @@ from ..container_resolvers import ( ) from ..docker_util import build_docker_images_command from ..mulled.mulled_build import ( - check_output, DEFAULT_CHANNELS, ensure_installed, InvolucroContext, @@ -54,8 +54,8 @@ CachedV2MulledImageMultiTarget.package_hash = _package_hash def list_docker_cached_mulled_images(namespace=None, hash_func="v2"): command = build_docker_images_command(truncate=True, sudo=False) - command = "%s | tail -n +2 | tr -s ' ' | cut -d' ' -f1,2" % command - images_and_versions = check_output(command) + images_and_versions = subprocess.check_output(command).strip().split('\n') + images_and_versions = [line.split()[0:2] for line in images_and_versions[1:]] name_filter = get_filter(namespace) def output_line_to_image(line): diff --git a/lib/galaxy/tools/deps/mulled/mulled_build.py b/lib/galaxy/tools/deps/mulled/mulled_build.py index 5d362f12cd3..381afbe2718 100644 --- a/lib/galaxy/tools/deps/mulled/mulled_build.py +++ b/lib/galaxy/tools/deps/mulled/mulled_build.py @@ -116,17 +116,12 @@ def get_affected_packages(args): """ recipes_dir = args.recipes_dir hours = args.diff_hours - cmd = """cd '%s' && git log --diff-filter=ACMRTUXB --name-only --pretty="" --since="%s hours ago" | grep -E '^recipes/.*/meta.yaml' | sort | uniq""" % (recipes_dir, hours) - pkg_list = check_output(cmd, shell=True) - ret = list() - for pkg in pkg_list.strip().split('\n'): + cmd = ['git', 'log', '--diff-filter=ACMRTUXB', '--name-only', '--pretty=""', '--since="%s hours ago"' % hours] + changed_files = subprocess.check_output(cmd, cwd=recipes_dir).strip().split('\n') + pkg_list = set([x for x in changed_files if x.startswith('recipes/') and x.endswith('meta.yaml')]) + for pkg in pkg_list: if pkg and os.path.exists(os.path.join(recipes_dir, pkg)): - ret.append((get_pkg_name(args, pkg), get_tests(args, pkg))) - return ret - - -def check_output(cmd, shell=True): - return subprocess.check_output(cmd, shell=shell) + yield (get_pkg_name(args, pkg), get_tests(args, pkg)) def conda_versions(pkg_name, file_name): diff --git a/lib/galaxy/tools/deps/mulled/mulled_build_channel.py b/lib/galaxy/tools/deps/mulled/mulled_build_channel.py index 23e3021187b..06e0c7afe6f 100644 --- a/lib/galaxy/tools/deps/mulled/mulled_build_channel.py +++ b/lib/galaxy/tools/deps/mulled/mulled_build_channel.py @@ -19,6 +19,7 @@ See recent changes that would be built with: from __future__ import print_function import os +import subprocess import sys import time @@ -27,7 +28,6 @@ from .mulled_build import ( add_build_arguments, args_to_mull_targets_kwds, build_target, - check_output, conda_versions, get_affected_packages, mull_targets, @@ -42,7 +42,13 @@ def _fetch_repo_data(args): repo_data = "%s-repodata.json" % channel if not os.path.exists(repo_data): platform_tag = 'osx-64' if sys.platform == 'darwin' else 'linux-64' - check_output("wget --quiet https://conda.anaconda.org/%s/%s/repodata.json.bz2 -O '%s.bz2' && bzip2 -d '%s.bz2'" % (channel, platform_tag, repo_data, repo_data)) + subprocess.check_call([ + 'wget', '--quiet', 'https://conda.anaconda.org/%s/%s/repodata.json.bz2' % (channel, platform_tag), + '-O', '%s.bz2' % repo_data + ]) + subprocess.check_call([ + 'bzip2', '-d', '%s.bz2' % repo_data + ]) return repo_data @@ -55,8 +61,7 @@ def _new_versions(quay, conda): def run_channel(args, build_last_n_versions=1): """Build list of involucro commands (as shell snippet) to run.""" - pkgs = get_affected_packages(args) - for pkg_name, pkg_tests in pkgs: + for pkg_name, pkg_tests in get_affected_packages(args): repo_data = _fetch_repo_data(args) c = conda_versions(pkg_name, repo_data) # only package the most recent N versions diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py index e8188e0964a..5c3811024e0 100644 --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -97,7 +97,7 @@ class MappableDependencyResolver: @staticmethod def _mapping_file_to_list(mapping_file): with open(mapping_file, "r") as f: - raw_mapping = yaml.load(f) or [] + raw_mapping = yaml.safe_load(f) or [] return map(RequirementMapping.from_dict, raw_mapping) def _expand_mappings(self, requirement): diff --git a/lib/galaxy/tools/deps/resolvers/modules.py b/lib/galaxy/tools/deps/resolvers/modules.py index 2e4412b34fe..50fd9c549d3 100644 --- a/lib/galaxy/tools/deps/resolvers/modules.py +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -173,6 +173,14 @@ class ModuleDependency(Dependency): self.module_version = module_version self._exact = exact + @property + def name(self): + return self.module_name + + @property + def version(self): + return self.module_version + @property def exact(self): return self._exact diff --git a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py index c00a636d5a7..c8562f959d4 100644 --- a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py +++ b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py @@ -15,7 +15,7 @@ import tarfile import tempfile from base64 import b64decode -from six.moves.urllib.request import urlopen +import requests # Set max size of archive/file that will be handled to be 100 GB. This is # arbitrary and should be adjusted as needed. @@ -27,18 +27,16 @@ def url_to_file(url, dest_file): Transfer a file from a remote URL to a temporary file. """ try: - url_reader = urlopen(url) + url_reader = requests.get(url, stream=True) CHUNK = 10 * 1024 # 10k total = 0 fp = open(dest_file, 'wb') - while True: - chunk = url_reader.read(CHUNK) - if not chunk: - break - fp.write(chunk) - total += CHUNK - if total > MAX_SIZE: - break + for chunk in url_reader.iter_content(chunk_size=CHUNK): + if chunk: + fp.write(chunk) + total += CHUNK + if total > MAX_SIZE: + break fp.close() return dest_file except Exception as e: diff --git a/lib/galaxy/tools/locations/dockstore.py b/lib/galaxy/tools/locations/dockstore.py index b379c23c38f..3eb96e55e24 100644 --- a/lib/galaxy/tools/locations/dockstore.py +++ b/lib/galaxy/tools/locations/dockstore.py @@ -57,7 +57,7 @@ class _Ga4ghToolClient(object): if as_string: return descriptor_str else: - return yaml.load(descriptor_str) + return yaml.safe_load(descriptor_str) @property def _requests(self): diff --git a/lib/galaxy/tools/toolbox/filters/__init__.py b/lib/galaxy/tools/toolbox/filters/__init__.py index 6e638076482..5d36d24c19b 100644 --- a/lib/galaxy/tools/toolbox/filters/__init__.py +++ b/lib/galaxy/tools/toolbox/filters/__init__.py @@ -83,7 +83,7 @@ class FilterFactory(object): try: __import__(full_module_name) except ImportError: - # log.debug("Failed to load module.", exc_info=True) + # log.debug("Failed to load module %s using sys.path %s", full_module_name, sys.path, exc_info=True) continue module = sys.modules[full_module_name] if hasattr(module, function_name): diff --git a/lib/galaxy/tools/toolbox/parser.py b/lib/galaxy/tools/toolbox/parser.py index 3c79a8b4d7b..db1b582404c 100644 --- a/lib/galaxy/tools/toolbox/parser.py +++ b/lib/galaxy/tools/toolbox/parser.py @@ -59,7 +59,7 @@ class YamlToolConfSource(ToolConfSource): def __init__(self, config_filename): with open(config_filename, "r") as f: - as_dict = yaml.load(f) + as_dict = yaml.safe_load(f) self.as_dict = as_dict def parse_tool_path(self): diff --git a/lib/galaxy/tours/__init__.py b/lib/galaxy/tours/__init__.py index 0511aa29e83..c35d147759c 100644 --- a/lib/galaxy/tours/__init__.py +++ b/lib/galaxy/tours/__init__.py @@ -68,7 +68,7 @@ class ToursRegistry(object): tour_id = os.path.splitext(filename)[0] try: with open(tour_path) as handle: - conf = yaml.load(handle) + conf = yaml.safe_load(handle) tour = tour_loader(conf) self.tours[tour_id] = tour_loader(conf) log.info("Loaded tour '%s'" % tour_id) diff --git a/lib/galaxy/util/plugin_config.py b/lib/galaxy/util/plugin_config.py index 133998f210f..7f6866d6f2d 100644 --- a/lib/galaxy/util/plugin_config.py +++ b/lib/galaxy/util/plugin_config.py @@ -83,4 +83,4 @@ def __read_yaml(path): raise ImportError("Attempting to read YAML configuration file - but PyYAML dependency unavailable.") with open(path, "rb") as f: - return yaml.load(f) + return yaml.safe_load(f) diff --git a/lib/galaxy/util/properties.py b/lib/galaxy/util/properties.py index 2ca160c27f7..8c22e0aaf59 100644 --- a/lib/galaxy/util/properties.py +++ b/lib/galaxy/util/properties.py @@ -67,7 +67,7 @@ def load_app_properties( config_section = "galaxy" with open(config_file, "r") as f: - raw_properties = yaml.load(f) + raw_properties = yaml.safe_load(f) properties = raw_properties[config_section] or {} override_prefix = "%sOVERRIDE_" % config_prefix diff --git a/lib/galaxy/visualization/plugins/interactive_environments.py b/lib/galaxy/visualization/plugins/interactive_environments.py index 161882e9409..e56449320db 100644 --- a/lib/galaxy/visualization/plugins/interactive_environments.py +++ b/lib/galaxy/visualization/plugins/interactive_environments.py @@ -111,7 +111,7 @@ class InteractiveEnvironmentRequest(object): raise Exception("[{0}] Could not find allowed_images.yml, or image tag in {0}.ini file for ".format(self.attr.viz_id)) with open(fn, 'r') as handle: - self.allowed_images = [x['image'] for x in yaml.load(handle)] + self.allowed_images = [x['image'] for x in yaml.safe_load(handle)] if len(self.allowed_images) == 0: raise Exception("No allowed images specified for " + self.attr.viz_id) @@ -373,7 +373,7 @@ class InteractiveEnvironmentRequest(object): p = Popen(raw_cmd, stdout=PIPE, stderr=PIPE, close_fds=True, shell=True) stdout, stderr = p.communicate() if p.returncode != 0: - log.error("%s\n%s" % (stdout, stderr)) + log.error("Container Launch error\n\n%s\n%s" % (stdout, stderr)) return None else: container_id = stdout.strip() @@ -484,7 +484,7 @@ class InteractiveEnvironmentRequest(object): p = Popen(command, stdout=PIPE, stderr=PIPE, close_fds=True, shell=True) stdout, stderr = p.communicate() if p.returncode != 0: - log.error("%s\n%s" % (stdout, stderr)) + log.error("Container Launch error\n\n%s\n%s" % (stdout, stderr)) return None inspect_data = json.loads(stdout) diff --git a/lib/galaxy/visualization/plugins/plugin.py b/lib/galaxy/visualization/plugins/plugin.py index e1c5f0ddcfa..82ccdd701fd 100644 --- a/lib/galaxy/visualization/plugins/plugin.py +++ b/lib/galaxy/visualization/plugins/plugin.py @@ -2,17 +2,18 @@ Visualization plugins: instantiate/deserialize data and models from a query string and render a webpage based on those data. """ - -import logging import copy -import mako +import logging import os -from galaxy.managers import api_keys +import mako.lookup -from galaxy.visualization.plugins import interactive_environments -from galaxy.visualization.plugins import resource_parser -from galaxy.visualization.plugins import utils +from galaxy.managers import api_keys +from galaxy.visualization.plugins import ( + interactive_environments, + resource_parser, + utils +) log = logging.getLogger(__name__) diff --git a/lib/galaxy/web/proxy/__init__.py b/lib/galaxy/web/proxy/__init__.py index 6ff3ee3399f..581050a3d5e 100644 --- a/lib/galaxy/web/proxy/__init__.py +++ b/lib/galaxy/web/proxy/__init__.py @@ -8,9 +8,10 @@ from galaxy.util import sockets from galaxy.util.lazy_process import LazyProcess, NoOpLazyProcess from galaxy.util import sqlite from galaxy.util import unique_id -import urllib2 import time +import requests + log = logging.getLogger(__name__) @@ -301,21 +302,17 @@ class RestGolangProxyIpc(object): 'ContainerIds': container_ids, } - req = urllib2.Request(self.api_url) - req.add_header('Content-Type', 'application/json') - # Sometimes it takes our poor little proxy a second or two to get # going, so if this fails, re-call ourselves with an increased timeout. try: - urllib2.urlopen(req, json.dumps(values)) - except urllib2.URLError as err: - log.debug(err) + requests.get(self.api_url, headers={'Content-Type': 'application/json'}, data=json.dumps(values)) + except requests.exceptions.ConnectionError as err: + log.exception(err) if sleep > 5: excp = "Could not contact proxy after %s seconds" % sum(range(sleep + 1)) raise Exception(excp) time.sleep(sleep) - self.handle_requests(authentication, proxy_requests, route_name, container_ids, sleep=sleep + 1) - pass + self.handle_requests(authentication, proxy_requests, route_name, container_ids, container_interface, sleep=sleep + 1) ProxyMapping = namedtuple('ProxyMapping', ['host', 'port', 'container_ids', 'container_interface']) diff --git a/lib/galaxy/webapps/config_manage.py b/lib/galaxy/webapps/config_manage.py index 07d0bc03658..685b5abcf56 100644 --- a/lib/galaxy/webapps/config_manage.py +++ b/lib/galaxy/webapps/config_manage.py @@ -11,8 +11,8 @@ import string import sys import tempfile from textwrap import TextWrapper -import urllib2 +import requests import six from six import StringIO @@ -307,9 +307,8 @@ def _write_option_rst(args, rst, key, heading_level, option_value): def _build_uwsgi_schema(args, app_desc): - req = urllib2.Request('https://raw.githubusercontent.com/unbit/uwsgi-docs/master/Options.rst') - response = urllib2.urlopen(req) - rst_options = response.read() + req = requests.get('https://raw.githubusercontent.com/unbit/uwsgi-docs/master/Options.rst') + rst_options = req.text last_line = None current_opt = None diff --git a/lib/galaxy/webapps/galaxy/api/users.py b/lib/galaxy/webapps/galaxy/api/users.py index 67ec64351f3..63c9e3a78a0 100644 --- a/lib/galaxy/webapps/galaxy/api/users.py +++ b/lib/galaxy/webapps/galaxy/api/users.py @@ -282,7 +282,7 @@ class UserAPIController(BaseAPIController, UsesTagsMixin, CreatesUsersMixin, Cre path = trans.app.config.user_preferences_extra_config_file try: with open(path, 'r') as stream: - config = yaml.load(stream) + config = yaml.safe_load(stream) except: log.warning('Config file (%s) could not be found or is malformed.' % path) return {} diff --git a/lib/galaxy/webapps/galaxy/controllers/async.py b/lib/galaxy/webapps/galaxy/controllers/async.py index 7db4af7667f..f002e29a553 100644 --- a/lib/galaxy/webapps/galaxy/controllers/async.py +++ b/lib/galaxy/webapps/galaxy/controllers/async.py @@ -2,9 +2,13 @@ Upload class """ +from __future__ import absolute_import + import logging import urllib +import requests + from galaxy import jobs, web from galaxy.util import Params from galaxy.util.hash_util import hmac_new @@ -162,8 +166,7 @@ class ASync(BaseUIController): url = "%s%s%s" % (url, url_join_char, urllib.urlencode(params.flatten())) log.debug("connecting to -> %s" % url) trans.log_event("Async connecting to -> %s" % url) - text = urllib.urlopen(url).read(-1) - text = text.strip() + text = requests.get(url).text.strip() if not text.endswith('OK'): raise Exception(text) data.state = data.blurb = data.states.RUNNING diff --git a/lib/galaxy/webapps/galaxy/controllers/library_common.py b/lib/galaxy/webapps/galaxy/controllers/library_common.py index 365dd48c6b9..955fede10f0 100644 --- a/lib/galaxy/webapps/galaxy/controllers/library_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/library_common.py @@ -1,4 +1,7 @@ +from __future__ import absolute_import + import glob +import json import logging import operator import os @@ -8,10 +11,9 @@ import sys import tarfile import tempfile import urllib -import urllib2 import zipfile -from json import dumps, loads +import requests from markupsafe import escape from sqlalchemy import and_, false from sqlalchemy.orm import eagerload_all @@ -555,7 +557,7 @@ class LibraryCommon(BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMeta if len(em_string): payload = None try: - payload = loads(em_string) + payload = json.loads(em_string) except Exception: message = 'Invalid JSON input' status = 'error' @@ -1122,8 +1124,8 @@ class LibraryCommon(BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMeta json_file_path = upload_common.create_paramfile(trans, uploaded_datasets) data_list = [ud.data for ud in uploaded_datasets] job_params = {} - job_params['link_data_only'] = dumps(kwd.get('link_data_only', 'copy_files')) - job_params['uuid'] = dumps(kwd.get('uuid', None)) + job_params['link_data_only'] = json.dumps(kwd.get('link_data_only', 'copy_files')) + job_params['uuid'] = json.dumps(kwd.get('uuid', None)) job, output = upload_common.create_job(trans, tool_params, tool, json_file_path, data_list, folder=library_bunch.folder, job_params=job_params) trans.sa_session.add(job) trans.sa_session.flush() @@ -2762,9 +2764,7 @@ def lucene_search(trans, cntrller, search_term, search_url, **kwd): message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') full_url = "%s/find?%s" % (search_url, urllib.urlencode({"kwd" : search_term})) - response = urllib2.urlopen(full_url) - ldda_ids = loads(response.read())["ids"] - response.close() + ldda_ids = requests.get(full_url).json()['ids'] lddas = [trans.sa_session.query(trans.app.model.LibraryDatasetDatasetAssociation).get(ldda_id) for ldda_id in ldda_ids] return status, message, get_sorted_accessible_library_items(trans, cntrller, lddas, 'name') diff --git a/lib/galaxy/webapps/galaxy/controllers/root.py b/lib/galaxy/webapps/galaxy/controllers/root.py index 4817aee5b84..17b82a9251b 100644 --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -1,10 +1,12 @@ """ Contains the main interface in the Universe class """ +from __future__ import absolute_import + import cgi import os -import urllib +import requests from paste.httpexceptions import HTTPNotFound, HTTPBadGateway from galaxy import web @@ -470,8 +472,8 @@ class RootController(controller.JSAppLauncher, UsesAnnotations): def bucket_proxy(self, trans, bucket=None, **kwd): if bucket: trans.response.set_content_type('text/xml') - b_list_xml = urllib.urlopen('http://s3.amazonaws.com/%s/' % bucket) - return b_list_xml.read() + b_list_xml = requests.get('http://s3.amazonaws.com/%s/' % bucket) + return b_list_xml.text raise Exception("You must specify a bucket") # ---- Debug methods ---------------------------------------------------- diff --git a/lib/galaxy/webapps/galaxy/controllers/visualization.py b/lib/galaxy/webapps/galaxy/controllers/visualization.py index 7ec3f702fd7..f951542bc74 100644 --- a/lib/galaxy/webapps/galaxy/controllers/visualization.py +++ b/lib/galaxy/webapps/galaxy/controllers/visualization.py @@ -931,7 +931,7 @@ class VisualizationController(BaseUIController, SharableMixin, UsesVisualization continue with open(image_file, 'r') as handle: - self.gie_image_map[gie] = yaml.load(handle) + self.gie_image_map[gie] = yaml.safe_load(handle) return trans.fill_template_mako( "visualization/gie.mako", diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py index d5216798bed..823078c8ecc 100644 --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -1,10 +1,12 @@ +from __future__ import absolute_import + import base64 import httplib import json import logging import os import sgmllib -import urllib2 +import requests from sqlalchemy import and_ from sqlalchemy.orm import joinedload @@ -763,7 +765,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi # Load workflow from external URL # NOTE: blocks the web thread. try: - workflow_data = urllib2.urlopen(url).read() + workflow_data = requests.get(url).text except Exception as e: message = "Failed to open URL: %s. Exception: %s" % (escape(url), escape(str(e))) status = 'error' diff --git a/lib/galaxy/webapps/reports/config.py b/lib/galaxy/webapps/reports/config.py index b69c3f2640f..80b20d3b46e 100644 --- a/lib/galaxy/webapps/reports/config.py +++ b/lib/galaxy/webapps/reports/config.py @@ -64,7 +64,7 @@ class Configuration(object): def check(self): # Check that required directories exist - for path in self.root, self.file_path, self.template_path: + for path in self.root, self.template_path: if not os.path.isdir(path): raise ConfigurationError("Directory does not exist: %s" % path) diff --git a/lib/galaxy/webapps/reports/controllers/system.py b/lib/galaxy/webapps/reports/controllers/system.py index 6359c1defd8..5c325c2239e 100644 --- a/lib/galaxy/webapps/reports/controllers/system.py +++ b/lib/galaxy/webapps/reports/controllers/system.py @@ -1,5 +1,6 @@ import logging import os +import subprocess from datetime import datetime, timedelta from decimal import Decimal @@ -148,12 +149,11 @@ class System(BaseUIController): message=message) def get_disk_usage(self, file_path): - df_cmd = 'df -h ' + file_path is_sym_link = os.path.islink(file_path) file_system = disk_size = disk_used = disk_avail = disk_cap_pct = mount = None - df_file = os.popen(df_cmd) - while True: - df_line = df_file.readline() + df_output = subprocess.check_output(['df', '-h', file_path]) + + for df_line in df_output: df_line = df_line.strip() if df_line: df_line = df_line.lower() @@ -176,7 +176,6 @@ class System(BaseUIController): pass else: break # EOF - df_file.close() return (file_system, disk_size, disk_used, disk_avail, disk_cap_pct, mount) @web.expose diff --git a/lib/galaxy/webapps/tool_shed/controllers/upload.py b/lib/galaxy/webapps/tool_shed/controllers/upload.py index 1695cf69507..2fa3d3be406 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -3,7 +3,8 @@ import os import shutil import tarfile import tempfile -import urllib + +import requests from galaxy import util from galaxy import web @@ -74,7 +75,7 @@ class UploadController(BaseUIController): elif url: valid_url = True try: - stream = urllib.urlopen(url) + stream = requests.get(url, stream=True) except Exception as e: valid_url = False message = 'Error uploading file via http: %s' % str(e) @@ -83,11 +84,9 @@ class UploadController(BaseUIController): if valid_url: fd, uploaded_file_name = tempfile.mkstemp() uploaded_file = open(uploaded_file_name, 'wb') - while 1: - chunk = stream.read(util.CHUNK_SIZE) - if not chunk: - break - uploaded_file.write(chunk) + for chunk in stream.iter_content(chunk_size=util.CHUNK_SIZE): + if chunk: + uploaded_file.write(chunk) uploaded_file.flush() uploaded_file_filename = url.split('/')[-1] isempty = os.path.getsize(os.path.abspath(uploaded_file_name)) == 0 diff --git a/lib/galaxy/webhooks/__init__.py b/lib/galaxy/webhooks/__init__.py index 7e9c9482433..6cc2c34d9d9 100644 --- a/lib/galaxy/webhooks/__init__.py +++ b/lib/galaxy/webhooks/__init__.py @@ -64,7 +64,7 @@ class WebhooksRegistry(object): def load_webhook_from_config(self, config_dir, config_file): try: with open(os.path.join(config_dir, config_file)) as file: - config = yaml.load(file) + config = yaml.safe_load(file) path = os.path.normpath(os.path.join(config_dir, '..')) webhook = Webhook( config['name'], diff --git a/lib/tool_shed/capsule/capsule_manager.py b/lib/tool_shed/capsule/capsule_manager.py index 422d663c4a9..5daa0b21c04 100644 --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -7,7 +7,7 @@ import tempfile import threading from time import gmtime, strftime -from six.moves.urllib.request import urlopen +import requests from sqlalchemy import and_, false import tool_shed.repository_types.util as rt_util @@ -809,24 +809,20 @@ class ImportRepositoryManager(object): uploaded_file=None, capsule_file_name=None) if url: - valid_url = True try: - stream = urlopen(url) + stream = requests.get(url, stream=True) except Exception as e: - valid_url = False return_dict['error_message'] = 'Error importing file via http: %s' % str(e) return_dict['status'] = 'error' return return_dict - if valid_url: - fd, uploaded_file_name = tempfile.mkstemp() - uploaded_file = open(uploaded_file_name, 'wb') - while 1: - chunk = stream.read(CHUNK_SIZE) - if not chunk: - break + + fd, uploaded_file_name = tempfile.mkstemp() + uploaded_file = open(uploaded_file_name, 'wb') + for chunk in stream.iter_content(chunk_size=CHUNK_SIZE): + if chunk: uploaded_file.write(chunk) - uploaded_file.flush() - uploaded_file_filename = url.split('/')[-1] + uploaded_file.flush() + uploaded_file_filename = url.split('/')[-1] elif file_data not in ('', None): uploaded_file = file_data.file uploaded_file_name = uploaded_file.name diff --git a/scripts/api/data_manager_example_execute.py b/scripts/api/data_manager_example_execute.py index 27715ab726f..991bef50b6e 100644 --- a/scripts/api/data_manager_example_execute.py +++ b/scripts/api/data_manager_example_execute.py @@ -4,11 +4,13 @@ # Script makes the naive assumption that dbkey==sequence id, which in many cases is not true nor desired # *** This script is not recommended for use as-is on a production server *** from __future__ import print_function + import optparse import time + from six.moves.urllib.parse import urljoin -from common import get, post +from common import get, post # noqa: I100 DEFAULT_SLEEP_TIME = 3 FETCH_GENOME_TOOL_ID = 'testtoolshed.g2.bx.psu.edu/repos/blankenberg/data_manager_fetch_genome_all_fasta/data_manager_fetch_genome_all_fasta/0.0.1' diff --git a/scripts/api/display.py b/scripts/api/display.py index bcf3885641c..d6fe219b3de 100755 --- a/scripts/api/display.py +++ b/scripts/api/display.py @@ -6,7 +6,7 @@ import sys from six.moves.urllib.error import URLError -from common import display +from common import display # noqa: I100 try: display(*sys.argv[1:3]) diff --git a/scripts/api/search.py b/scripts/api/search.py index bbb905d621d..73fc22b0451 100644 --- a/scripts/api/search.py +++ b/scripts/api/search.py @@ -2,10 +2,12 @@ Sample script for Galaxy Search API """ from __future__ import print_function + import json -import requests import sys +import requests + class RemoteGalaxy(object): diff --git a/scripts/binary_compatibility.py b/scripts/binary_compatibility.py deleted file mode 100644 index 29141d15d82..00000000000 --- a/scripts/binary_compatibility.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python -""" -Creates a Python binary-compatibility.cfg as described here: - - https://mail.python.org/pipermail/distutils-sig/2015-July/026617.html -""" -from __future__ import print_function - -# Vaguely Python 2.6 compatibile ArgumentParser import -try: - from argparse import ArgumentParser -except ImportError: - from optparse import OptionParser - - class ArgumentParser(OptionParser): - - def __init__(self, **kwargs): - self.delegate = OptionParser(**kwargs) - - def add_argument(self, *args, **kwargs): - if "required" in kwargs: - del kwargs["required"] - return self.delegate.add_option(*args, **kwargs) - - def parse_args(self, args=None): - (options, args) = self.delegate.parse_args(args) - return options - -import sys -import json - -from pip.pep425tags import get_supported -from pip.platform import get_specific_platform - - -compatible_platforms = { - 'centos': 'rhel', - 'scientific': 'rhel', -} - - -def install_compat(): - spec_plat = get_specific_platform() - if spec_plat is None: - return None - this_plat = spec_plat[0] - compat_plat = compatible_platforms.get(this_plat, None) - rval = {} - if compat_plat: - print('{0} is binary compatible with {1} (and can install {2} wheels)' - .format(this_plat, compat_plat, compat_plat), - file=sys.stderr) - for py, abi, plat in get_supported(): - if this_plat in plat: - rval[plat] = {'install': [plat.replace(this_plat, compat_plat)]} - return rval - - -def main(): - arg_parser = ArgumentParser() - arg_parser.add_argument('-o', '--output', default=None, help='Output to file') - args = arg_parser.parse_args() - - compat = install_compat() - - if compat: - if args.output is not None: - with open(args.output, 'w') as out: - json.dump(compat, out) - else: - print(json.dumps(install_compat())) - - -if __name__ == '__main__': - main() diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index 679a88c7a2c..a1e64df09b8 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -139,9 +139,9 @@ if [ $REPLACE_PIP -eq 1 ]; then fi if [ $FETCH_WHEELS -eq 1 ]; then - pip install -r requirements.txt --index-url "${GALAXY_WHEELS_INDEX_URL}" + pip install -r requirements.txt --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url https://pypi.python.org/pypi GALAXY_CONDITIONAL_DEPENDENCIES=$(PYTHONPATH=lib python -c "import galaxy.dependencies; print '\n'.join(galaxy.dependencies.optional('$GALAXY_CONFIG_FILE'))") - [ -z "$GALAXY_CONDITIONAL_DEPENDENCIES" ] || echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" + [ -z "$GALAXY_CONDITIONAL_DEPENDENCIES" ] || echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url https://pypi.python.org/pypi fi if [ $FETCH_WHEELS -eq 1 -a $DEV_WHEELS -eq 1 ]; then diff --git a/scripts/common_startup_functions.sh b/scripts/common_startup_functions.sh index 8deed854db5..6b2e0cc3d6e 100644 --- a/scripts/common_startup_functions.sh +++ b/scripts/common_startup_functions.sh @@ -1,6 +1,8 @@ #!/bin/sh -uwsgi_args="--master --pythonpath=lib" +: ${GALAXY_STATIC_DIRECTORY:=static} +: ${GALAXY_STYLE_DIRECTORY:=static/style/blue} +uwsgi_args="--master --pythonpath=lib --static-map /static=$GALAXY_STATIC_DIRECTORY --static-map /static/style=$GALAXY_STYLE_DIRECTORY" parse_common_args() { INITIALIZE_TOOL_DEPENDENCIES=1 diff --git a/scripts/data_libraries/build_lucene_index.py b/scripts/data_libraries/build_lucene_index.py index a53ec522163..b3f787e8b02 100644 --- a/scripts/data_libraries/build_lucene_index.py +++ b/scripts/data_libraries/build_lucene_index.py @@ -14,7 +14,8 @@ import csv import os import sys import urllib -import urllib2 + +import requests sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib'))) @@ -39,9 +40,7 @@ def main(ini_file): def build_index(search_url, dataset_file): url = "%s/index?%s" % (search_url, urllib.urlencode({"docfile": dataset_file})) - request = urllib2.Request(url) - request.get_method = lambda: "PUT" - urllib2.urlopen(request) + requests.put(url) def create_dataset_file(dataset_iter): diff --git a/scripts/dump_tour.py b/scripts/dump_tour.py index dde0e8e556f..b13020581cb 100755 --- a/scripts/dump_tour.py +++ b/scripts/dump_tour.py @@ -2,8 +2,8 @@ import argparse import datetime import os -import time import sys +import time sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'test'))) diff --git a/scripts/edam_mapping.py b/scripts/edam_mapping.py index 7858d9f5abb..c3756ad97c7 100644 --- a/scripts/edam_mapping.py +++ b/scripts/edam_mapping.py @@ -16,13 +16,14 @@ from __future__ import print_function import os import sys -import urllib2 from xml import etree +import requests + sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib'))) -import galaxy.model import galaxy.datatypes.registry +import galaxy.model SCRIPTS_DIR = os.path.dirname(__file__) PROJECT_DIR = os.path.abspath(os.path.join(SCRIPTS_DIR, os.pardir)) @@ -35,7 +36,7 @@ EDAM_OWL_URL = "http://data.bioontology.org/ontologies/EDAM/submissions/25/downl if not os.path.exists("/tmp/edam.owl"): - open("/tmp/edam.owl", "w").write(urllib2.urlopen(EDAM_OWL_URL).read()) + open("/tmp/edam.owl", "w").write(requests.get(EDAM_OWL_URL).text) owl_xml_tree = etree.ElementTree.parse("/tmp/edam.owl") diff --git a/scripts/fetch_eggs.py b/scripts/fetch_eggs.py index 5236a733266..61451a6ef1b 100755 --- a/scripts/fetch_eggs.py +++ b/scripts/fetch_eggs.py @@ -1,7 +1,12 @@ #!/usr/bin/env python from __future__ import print_function + from os import pardir -from os.path import join, abspath, dirname +from os.path import ( + abspath, + dirname, + join +) from sys import exit msg = """ diff --git a/scripts/grt/export.py b/scripts/grt/export.py index 4619c6929c6..f5a3480c14b 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -4,25 +4,24 @@ See doc/source/admin/grt.rst for more detailed usage information. """ import argparse -import tarfile import json +import logging import os -import sqlalchemy as sa import subprocess import sys +import tarfile import time -import yaml -import logging - from collections import defaultdict -sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib'))) +import sqlalchemy as sa +import yaml -from galaxy.util.properties import load_app_properties +sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib'))) import galaxy import galaxy.config -from galaxy.objectstore import build_object_store_from_config from galaxy.model import mapping +from galaxy.objectstore import build_object_store_from_config +from galaxy.util.properties import load_app_properties sample_config = os.path.abspath(os.path.join(os.path.dirname(__file__), 'grt.yml.sample')) default_config = os.path.abspath(os.path.join(os.path.dirname(__file__), 'grt.yml')) @@ -191,11 +190,11 @@ def main(argv): annotate('init_start', 'Loading GRT configuration...') try: with open(args.config) as handle: - config = yaml.load(handle) + config = yaml.safe_load(handle) except Exception: logging.info('Using default GRT configuration') with open(sample_config) as handle: - config = yaml.load(handle) + config = yaml.safe_load(handle) annotate('init_end') REPORT_DIR = args.report_directory diff --git a/scripts/grt/upload.py b/scripts/grt/upload.py index 332cbe89e6d..8ec619fc5b7 100644 --- a/scripts/grt/upload.py +++ b/scripts/grt/upload.py @@ -4,11 +4,12 @@ See doc/source/admin/grt.rst for more detailed usage information. """ import argparse +import logging import os import sys -import yaml -import logging + import requests +import yaml sample_config = os.path.abspath(os.path.join(os.path.dirname(__file__), 'grt.yml.sample')) default_config = os.path.abspath(os.path.join(os.path.dirname(__file__), 'grt.yml')) @@ -28,11 +29,11 @@ def main(argv): logging.info('Loading GRT configuration...') try: with open(args.config) as handle: - config = yaml.load(handle) + config = yaml.safe_load(handle) except Exception: logging.info('Using default GRT configuration') with open(sample_config) as handle: - config = yaml.load(handle) + config = yaml.safe_load(handle) REPORT_DIR = args.report_directory GRT_URL = config['grt']['url'].rstrip('/') + '/' diff --git a/scripts/manage_tool_dependencies.py b/scripts/manage_tool_dependencies.py index 45a89cc82ae..40e09217d48 100644 --- a/scripts/manage_tool_dependencies.py +++ b/scripts/manage_tool_dependencies.py @@ -4,7 +4,6 @@ import sys sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib'))) -from galaxy.util.properties import find_config_file, load_app_properties from galaxy.config import ( configure_logging, find_path, @@ -12,6 +11,7 @@ from galaxy.config import ( parse_dependency_options, ) from galaxy.tools.deps import CachedDependencyManager, DependencyManager, NullDependencyManager +from galaxy.util.properties import find_config_file, load_app_properties DESCRIPTION = "Script to manage tool dependencies (with focus on a Conda environments)." diff --git a/scripts/microbes/BeautifulSoup.py b/scripts/microbes/BeautifulSoup.py index cd104595187..b9262d7a828 100644 --- a/scripts/microbes/BeautifulSoup.py +++ b/scripts/microbes/BeautifulSoup.py @@ -45,12 +45,15 @@ from __future__ import generators import codecs import re +import sgmllib import string import sys import types -import sgmllib from htmlentitydefs import name2codepoint -from sgmllib import SGMLParser, SGMLParseError +from sgmllib import ( + SGMLParseError, + SGMLParser +) __author__ = "Leonard Richardson (crummy.com)" __contributors__ = ["Sam Ruby (intertwingly.net)", diff --git a/scripts/microbes/harvest_bacteria.py b/scripts/microbes/harvest_bacteria.py index 7161669b5c9..762d9cfa821 100644 --- a/scripts/microbes/harvest_bacteria.py +++ b/scripts/microbes/harvest_bacteria.py @@ -8,11 +8,16 @@ import os import sys import time from ftplib import FTP -from urllib2 import urlopen from urllib import urlretrieve +import requests from BeautifulSoup import BeautifulSoup -from util import get_bed_from_genbank, get_bed_from_glimmer3, get_bed_from_GeneMarkHMM, get_bed_from_GeneMark +from util import ( + get_bed_from_genbank, + get_bed_from_GeneMark, + get_bed_from_GeneMarkHMM, + get_bed_from_glimmer3 +) assert sys.version_info[:2] >= (2, 4) @@ -26,7 +31,7 @@ desired_ftp_files = {'GeneMark': {'ext': 'GeneMark-2.5f', 'parser': 'process_Gen # number, name, chroms, kingdom, group, genbank, refseq, info_url, ftp_url def iter_genome_projects(url="http://www.ncbi.nlm.nih.gov/genomes/lproks.cgi?view=1", info_url_base="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=genomeprj&cmd=Retrieve&dopt=Overview&list_uids="): - for row in BeautifulSoup(urlopen(url)).findAll(name='tr', bgcolor=["#EEFFDD", "#E8E8DD"]): + for row in BeautifulSoup(requests.get(url).text).findAll(name='tr', bgcolor=["#EEFFDD", "#E8E8DD"]): row = str(row).replace("\n", "").replace("\r", "") fields = row.split("") @@ -65,7 +70,7 @@ def get_chroms_by_project_id(org_num, base_url="http://www.ncbi.nlm.nih.gov/entr html_count += 1 url = "%s%s" % (base_url, org_num) try: - html = urlopen(url) + html = requests.get(url).text except: print "GENOME PROJECT FAILED:", html_count, "org:", org_num, url html = None diff --git a/scripts/paster.py b/scripts/paster.py index e1548b3ed6e..861a98075ae 100755 --- a/scripts/paster.py +++ b/scripts/paster.py @@ -11,7 +11,7 @@ sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pa from galaxy.util.pastescript import serve -from check_python import check_python +from check_python import check_python # noqa: I100 # ensure supported version try: diff --git a/scripts/rst2html.py b/scripts/rst2html.py index ef6e1fcc20a..409d8e26c2f 100755 --- a/scripts/rst2html.py +++ b/scripts/rst2html.py @@ -16,8 +16,10 @@ try: except: pass -from docutils.core import publish_cmdline, default_description - +from docutils.core import ( + default_description, + publish_cmdline +) description = ('Generates (X)HTML documents from standalone reStructuredText ' 'sources. ' + default_description) diff --git a/scripts/summarize_timings.py b/scripts/summarize_timings.py index 5cf4295bc79..cfa1169594b 100644 --- a/scripts/summarize_timings.py +++ b/scripts/summarize_timings.py @@ -1,8 +1,8 @@ """Script to parse timings out of a Galaxy log and summarize.""" from __future__ import print_function -from argparse import ArgumentParser import re +from argparse import ArgumentParser import numpy diff --git a/scripts/tool_shed/api/export.py b/scripts/tool_shed/api/export.py index a198fffd966..05b43f53f78 100644 --- a/scripts/tool_shed/api/export.py +++ b/scripts/tool_shed/api/export.py @@ -11,12 +11,13 @@ import argparse import os import sys import tempfile -import urllib2 + +import requests sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 'lib')) from tool_shed.util import basic_util -from common import display, submit +from common import display, submit # noqa: I100 CAPSULE_FILENAME = 'capsule' CAPSULE_WITH_DEPENDENCIES_FILENAME = 'capsule_with_dependencies' @@ -98,24 +99,11 @@ def main(options): download_url = export_dict['download_url'] download_dir = os.path.abspath(options.download_dir) file_path = os.path.join(download_dir, repositories_archive_filename) - src = None - dst = None - try: - src = urllib2.urlopen(download_url) - dst = open(file_path, 'wb') - while True: - chunk = src.read(CHUNK_SIZE) + src = requests.get(download_url, stream=True) + with open(file_path, 'wb') as dst: + for chunk in src.iter_content(chunk_size=CHUNK_SIZE): if chunk: dst.write(chunk) - else: - break - except: - raise - finally: - if src: - src.close() - if dst: - dst.close() print "Successfully exported revision ", options.changeset_revision, " of repository ", options.name, " owned by ", options.owner print "to location ", file_path else: diff --git a/scripts/tool_shed/api/get_filtered_repository_revisions.py b/scripts/tool_shed/api/get_filtered_repository_revisions.py index a59911fc40c..b5805e697bc 100644 --- a/scripts/tool_shed/api/get_filtered_repository_revisions.py +++ b/scripts/tool_shed/api/get_filtered_repository_revisions.py @@ -28,7 +28,7 @@ sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, from galaxy.util import asbool from tool_shed.util import hg_util -from common import get_api_url, get_repository_dict, json_from_url +from common import get_api_url, get_repository_dict, json_from_url # noqa: I100 def main(options): diff --git a/scripts/tool_shed/bootstrap_tool_shed/bootstrap_util.py b/scripts/tool_shed/bootstrap_tool_shed/bootstrap_util.py index 2a899238b88..6d91d9bf87a 100755 --- a/scripts/tool_shed/bootstrap_tool_shed/bootstrap_util.py +++ b/scripts/tool_shed/bootstrap_tool_shed/bootstrap_util.py @@ -1,6 +1,6 @@ #!/usr/bin/python -import optparse import ConfigParser +import optparse import os import sys diff --git a/scripts/tool_shed/bootstrap_tool_shed/create_user_with_api_key.py b/scripts/tool_shed/bootstrap_tool_shed/create_user_with_api_key.py index 4d1d20658ad..001c001a87d 100644 --- a/scripts/tool_shed/bootstrap_tool_shed/create_user_with_api_key.py +++ b/scripts/tool_shed/bootstrap_tool_shed/create_user_with_api_key.py @@ -2,10 +2,10 @@ import ConfigParser import logging +import optparse import os import re import sys -import optparse sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 'lib')) sys.path.insert(1, os.path.join(os.path.dirname(__file__))) @@ -13,7 +13,8 @@ sys.path.insert(1, os.path.join(os.path.dirname(__file__))) import galaxy.webapps.tool_shed.config as tool_shed_config from galaxy.web import security from galaxy.webapps.tool_shed.model import mapping -from bootstrap_util import admin_user_info + +from bootstrap_util import admin_user_info # noqa: I100 log = logging.getLogger(__name__) diff --git a/scripts/tool_shed/deprecate_repositories_without_metadata.py b/scripts/tool_shed/deprecate_repositories_without_metadata.py index 95357ceafbe..1a31604e6e1 100644 --- a/scripts/tool_shed/deprecate_repositories_without_metadata.py +++ b/scripts/tool_shed/deprecate_repositories_without_metadata.py @@ -8,8 +8,8 @@ import sys import textwrap import time from datetime import datetime, timedelta -from time import strftime from optparse import OptionParser +from time import strftime import sqlalchemy as sa from sqlalchemy import and_, distinct, false, not_ @@ -18,8 +18,10 @@ sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, import galaxy.webapps.tool_shed.config as tool_shed_config import galaxy.webapps.tool_shed.model.mapping -from galaxy.util import send_mail as galaxy_send_mail -from galaxy.util import build_url +from galaxy.util import ( + build_url, + send_mail as galaxy_send_mail +) log = logging.getLogger() log.setLevel(10) diff --git a/scripts/tools/maf/check_loc_file.py b/scripts/tools/maf/check_loc_file.py index cc0078a238d..96be0d8f7ce 100644 --- a/scripts/tools/maf/check_loc_file.py +++ b/scripts/tools/maf/check_loc_file.py @@ -1,10 +1,12 @@ # Dan Blankenberg # This script checks maf_index.loc file for inconsistencies between what is listed as available and what is really available. # Make sure that required dependencies (e.g. galaxy_root/lib) are included in your PYTHONPATH -import bx.align.maf -from galaxy.tools.util import maf_utilities import sys +import bx.align.maf + +from galaxy.tools.util import maf_utilities + assert sys.version_info[:2] >= (2, 4) diff --git a/scripts/transfer.py b/scripts/transfer.py index 8bcb23f432e..3938ca48846 100644 --- a/scripts/transfer.py +++ b/scripts/transfer.py @@ -215,6 +215,7 @@ def transfer(app, transfer_job_id): def http_transfer(transfer_job): """Plugin" for handling http(s) transfers.""" url = transfer_job.params['url'] + assert url.startswith('http://') or url.startswith('https://') try: f = urllib2.urlopen(url) except urllib2.URLError as e: diff --git a/templates/display_base.mako b/templates/display_base.mako index f0780e2f31c..9015da34e3a 100644 --- a/templates/display_base.mako +++ b/templates/display_base.mako @@ -117,10 +117,10 @@ ## Get URL to other published items owned by user that owns this item. <% ##TODO: is there a better way to create this URL? Can't use 'f-username' as a key b/c it's not a valid identifier. - controller_name = get_controller_name( item ) + modern_route = modern_route_for_controller(get_controller_name(item)) item_plural = get_item_plural( item ) - href_to_all_items = h.url_for( controller='/' + controller_name, action='list_published') - href_to_user_items = h.url_for( controller='/' + controller_name, action='list_published', xxx=item.user.username) + href_to_all_items = h.url_for( controller='/' + modern_route, action='list_published') + href_to_user_items = h.url_for( controller='/' + modern_route, action='list_published', xxx=item.user.username) href_to_user_items = href_to_user_items.replace( 'xxx', 'f-username') %> @@ -162,10 +162,10 @@ <% ## FIXME: duplicated from above for now - controller_name = get_controller_name( item ) + modern_route = modern_route_for_controller(get_controller_name(item)) item_plural = get_item_plural( item ) - href_to_all_items = h.url_for( controller='/' + controller_name, action='list_published') - href_to_user_items = h.url_for( controller='/' + controller_name, action='list_published', xxx=item.user.username) + href_to_all_items = h.url_for( controller='/' + modern_route , action='list_published') + href_to_user_items = h.url_for( controller='/' + modern_route, action='list_published', xxx=item.user.username) href_to_user_items = href_to_user_items.replace( 'xxx', 'f-username') %> diff --git a/templates/display_common.mako b/templates/display_common.mako index d78633df939..2fa1f3e49b9 100644 --- a/templates/display_common.mako +++ b/templates/display_common.mako @@ -111,6 +111,21 @@ %> +<%def name="modern_route_for_controller( controller )"> + <% + if controller == "history": + return "histories" + elif controller == "workflow": + return "workflows" + elif controller == "dataset": + return "datasets" + elif controller == "page": + return "pages" + else: + return controller + %> + + ## Returns item user/owner. <%def name="get_item_user( item )"> <% diff --git a/test/api/test_api_batch.py b/test/api/test_api_batch.py index fef24d0a87d..e7a8d1ccc39 100644 --- a/test/api/test_api_batch.py +++ b/test/api/test_api_batch.py @@ -2,7 +2,7 @@ import json from requests import post -from base import api +from base import api # noqa: I100 class ApiBatchTestCase(api.ApiTestCase): diff --git a/test/api/test_authenticate.py b/test/api/test_authenticate.py index a99e2ff4728..5e25e69426d 100644 --- a/test/api/test_authenticate.py +++ b/test/api/test_authenticate.py @@ -2,7 +2,7 @@ import base64 from requests import get -from base import api +from base import api # noqa: I100 TEST_USER_EMAIL = "auth_user_test@bx.psu.edu" TEST_USER_PASSWORD = "testpassword1" diff --git a/test/api/test_datatypes.py b/test/api/test_datatypes.py index d41cd98596b..70341332e97 100644 --- a/test/api/test_datatypes.py +++ b/test/api/test_datatypes.py @@ -2,7 +2,7 @@ import time from requests import put -from base import api +from base import api # noqa: I100 HIDDEN_DURING_UPLOAD_DATATYPE = "fli" diff --git a/test/api/test_histories.py b/test/api/test_histories.py index cf57354d342..ed26d3056f6 100644 --- a/test/api/test_histories.py +++ b/test/api/test_histories.py @@ -1,14 +1,16 @@ # -*- coding: utf-8 -*- - from requests import ( get, post, put ) -from base import api - -from base.populators import DatasetPopulator, DatasetCollectionPopulator, wait_on +from base import api # noqa: I100 +from base.populators import ( # noqa: I100 + DatasetCollectionPopulator, + DatasetPopulator, + wait_on +) class HistoriesApiTestCase(api.ApiTestCase): diff --git a/test/api/test_history_contents.py b/test/api/test_history_contents.py index c82e8cb4cdd..f395452f97c 100644 --- a/test/api/test_history_contents.py +++ b/test/api/test_history_contents.py @@ -4,8 +4,12 @@ import json from requests import delete, put -from base import api -from base.populators import DatasetCollectionPopulator, LibraryPopulator, TestsDatasets +from base import api # noqa: I100 +from base.populators import ( # noqa: I100 + DatasetCollectionPopulator, + LibraryPopulator, + TestsDatasets +) # TODO: Test anonymous access. diff --git a/test/api/test_pages.py b/test/api/test_pages.py index f0b5755eec4..8d72b0601ad 100644 --- a/test/api/test_pages.py +++ b/test/api/test_pages.py @@ -1,6 +1,6 @@ from requests import delete -from base import api +from base import api # noqa: I100 from galaxy.exceptions import error_codes diff --git a/test/api/test_tools.py b/test/api/test_tools.py index 0dcfe546bac..7b7e7cd878b 100644 --- a/test/api/test_tools.py +++ b/test/api/test_tools.py @@ -6,8 +6,8 @@ from base.populators import ( DatasetCollectionPopulator, DatasetPopulator, LibraryPopulator, - skip_without_tool, skip_without_datatype, + skip_without_tool, ) from galaxy.tools.verify.test_data import TestDataResolver diff --git a/test/api/test_users.py b/test/api/test_users.py index 977cac886c3..7453e3210a2 100644 --- a/test/api/test_users.py +++ b/test/api/test_users.py @@ -1,8 +1,11 @@ import json -from requests import put, get +from requests import ( + get, + put +) -from base import api +from base import api # noqa: I100 TEST_USER_EMAIL = "user_for_users_index_test@bx.psu.edu" diff --git a/test/api/test_visualizations.py b/test/api/test_visualizations.py index 7ed68af6321..702dd890c02 100644 --- a/test/api/test_visualizations.py +++ b/test/api/test_visualizations.py @@ -5,8 +5,8 @@ from requests import ( put ) -from base import api -from base.api_asserts import assert_has_keys +from base import api # noqa: I100 +from base.api_asserts import assert_has_keys # noqa: I100 INDEX_KEYS = [ 'id', 'title', 'type', 'dbkey', 'url' diff --git a/test/api/test_workflows.py b/test/api/test_workflows.py index 79a0c1c2218..c0162be0883 100644 --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -8,8 +8,8 @@ from uuid import uuid4 import yaml from requests import delete, put -from base import api -from base.populators import ( +from base import api # noqa: I100 +from base.populators import ( # noqa: I100 DatasetCollectionPopulator, DatasetPopulator, skip_without_tool, @@ -172,7 +172,7 @@ class BaseWorkflowsApiTestCase(api.ApiTestCase): ) if jobs_descriptions is None: assert source_type != "path" - jobs_descriptions = yaml.load(has_workflow) + jobs_descriptions = yaml.safe_load(has_workflow) test_data = jobs_descriptions.get("test_data", {}) diff --git a/test/api/test_workflows_from_yaml.py b/test/api/test_workflows_from_yaml.py index 9beddf7aa2e..2b39628c27d 100644 --- a/test/api/test_workflows_from_yaml.py +++ b/test/api/test_workflows_from_yaml.py @@ -55,11 +55,11 @@ steps: assert tool_count['random_lines1'] == 1 assert tool_count['cat1'] == 2 -# FIXME: This test fails on some machines due to (we're guessing) yaml loading +# FIXME: This test fails on some machines due to (we're guessing) yaml.safe_loading # order being not guaranteed and inconsistent across platforms. The workflow -# yaml loader probably needs to enforce order using something like the +# yaml.safe_loader probably needs to enforce order using something like the # approach described here: -# https://stackoverflow.com/questions/13297744/pyyaml-control-ordering-of-items-called-by-yaml-load +# https://stackoverflow.com/questions/13297744/pyyaml-control-ordering-of-items-called-by-yaml.safe_load # def test_multiple_input( self ): # history_id = self.dataset_populator.new_history() # self._run_jobs(""" diff --git a/test/base/populators.py b/test/base/populators.py index e7d19e463a2..336ef523918 100644 --- a/test/base/populators.py +++ b/test/base/populators.py @@ -1,17 +1,15 @@ import contextlib import json import time - from functools import wraps from operator import itemgetter import requests - from pkg_resources import resource_string from six import StringIO -from base import api_asserts -from base.workflows_format_2 import ( +from . import api_asserts +from .workflows_format_2 import ( convert_and_import_workflow, ImporterGalaxyInterface, ) diff --git a/test/base/workflows_format_2/converter.py b/test/base/workflows_format_2/converter.py index 05a9a319c49..c35ec2797c0 100644 --- a/test/base/workflows_format_2/converter.py +++ b/test/base/workflows_format_2/converter.py @@ -32,7 +32,7 @@ RUN_ACTIONS_TO_STEPS = { def yaml_to_workflow(has_yaml, galaxy_interface, workflow_directory): """Convert a Format 2 workflow into standard Galaxy format from supplied stream.""" - as_python = yaml.load(has_yaml) + as_python = yaml.safe_load(has_yaml) return python_to_workflow(as_python, galaxy_interface, workflow_directory) @@ -109,7 +109,7 @@ def _python_to_workflow(as_python, conversion_context): run_action_path = run_action["@import"] runnable_path = os.path.join(conversion_context.workflow_directory, run_action_path) with open(runnable_path, "r") as f: - runnable_description = yaml.load(f) + runnable_description = yaml.safe_load(f) run_action = runnable_description run_class = run_action["class"] diff --git a/test/base/workflows_format_2/main.py b/test/base/workflows_format_2/main.py index 70d59a8db8d..a4af30e8da6 100644 --- a/test/base/workflows_format_2/main.py +++ b/test/base/workflows_format_2/main.py @@ -20,7 +20,7 @@ def convert_and_import_workflow(has_workflow, **kwds): if workflow_directory is None: workflow_directory = os.path.dirname(has_workflow) with open(workflow_path, "r") as f: - has_workflow = yaml.load(f) + has_workflow = yaml.safe_load(f) if workflow_directory is not None: workflow_directory = os.path.abspath(workflow_directory) diff --git a/test/functional/twilltestcase.py b/test/functional/twilltestcase.py index 249c8a0b889..56c0aea45b8 100644 --- a/test/functional/twilltestcase.py +++ b/test/functional/twilltestcase.py @@ -15,7 +15,7 @@ from six import string_types, StringIO from six.moves.urllib.parse import urlencode, urlparse from twill.other_packages._mechanize_dist import ClientForm -from base.testcase import FunctionalTestCase +from base.testcase import FunctionalTestCase # noqa: I100 # Force twill to log to a buffer -- FIXME: Should this go to stdout and be captured by nose? buffer = StringIO() diff --git a/test/functional/webhooks/phdcomics/helper/__init__.py b/test/functional/webhooks/phdcomics/helper/__init__.py index b6828d2c21a..d3ec4d7fbe5 100644 --- a/test/functional/webhooks/phdcomics/helper/__init__.py +++ b/test/functional/webhooks/phdcomics/helper/__init__.py @@ -1,7 +1,7 @@ -import urllib -import re -import random import logging +import random +import re +import urllib log = logging.getLogger(__name__) diff --git a/test/galaxy_selenium/data.py b/test/galaxy_selenium/data.py index a3d969f8a4f..1ed3baf0a5a 100644 --- a/test/galaxy_selenium/data.py +++ b/test/galaxy_selenium/data.py @@ -1,6 +1,5 @@ +import yaml from pkg_resources import resource_string -import yaml - data_yaml = resource_string(__name__, 'navigation-data.yml').decode("UTF-8") -NAVIGATION_DATA = yaml.load(data_yaml) +NAVIGATION_DATA = yaml.safe_load(data_yaml) diff --git a/test/galaxy_selenium/has_driver.py b/test/galaxy_selenium/has_driver.py index 021de2eac0a..30b4a4f9e88 100644 --- a/test/galaxy_selenium/has_driver.py +++ b/test/galaxy_selenium/has_driver.py @@ -4,9 +4,9 @@ This should be mixed into classes with a self.driver and self.default_timeout attribute. """ +from selenium.common.exceptions import TimeoutException from selenium.webdriver.common.action_chains import ActionChains from selenium.webdriver.common.by import By -from selenium.common.exceptions import TimeoutException from selenium.webdriver.common.keys import Keys from selenium.webdriver.support import expected_conditions as ec from selenium.webdriver.support.ui import WebDriverWait @@ -140,5 +140,17 @@ class HasDriver: ) +def exception_indicates_not_clickable(exception): + return "not clickable" in str(exception) + + def exception_indicates_stale_element(exception): return "stale" in str(exception) + + +__all__ = ( + "exception_indicates_not_clickable", + "exception_indicates_stale_element", + "HasDriver", + "TimeoutException", +) diff --git a/test/galaxy_selenium/navigates_galaxy.py b/test/galaxy_selenium/navigates_galaxy.py index 7bdcc96d987..71a77e711ac 100644 --- a/test/galaxy_selenium/navigates_galaxy.py +++ b/test/galaxy_selenium/navigates_galaxy.py @@ -8,19 +8,26 @@ import contextlib import random import string import time - from functools import partial, wraps import requests import yaml -from .data import NAVIGATION_DATA -from .has_driver import exception_indicates_stale_element, HasDriver, TimeoutException from . import sizzle +from .data import NAVIGATION_DATA +from .has_driver import ( + exception_indicates_not_clickable, + exception_indicates_stale_element, + HasDriver, + TimeoutException, +) # Test case data DEFAULT_PASSWORD = '123456' +RETRY_DURING_TRANSITIONS_SLEEP_DEFAULT = .1 +RETRY_DURING_TRANSITIONS_ATTEMPTS_DEFAULT = 10 + class NullTourCallback(object): @@ -28,7 +35,24 @@ class NullTourCallback(object): pass -def retry_call_during_transitions(f, attempts=5, sleep=.1, exception_check=exception_indicates_stale_element): +def excepion_seems_to_indicate_transition(e): + """True if exception seems to indicate the page state is transitioning. + + Galaxy features many different transition effects that change the page state over time. + These transitions make it slightly more difficult to test Galaxy because atomic input + actions take an indeterminate amount of time to be reflected on the screen. This method + takes a Selenium assertion and tries to infer if such a transition could be the root + cause of the exception. The methods that follow use it to allow retrying actions during + transitions. + + Currently the two kinds of exceptions that we say may indicate a transition are + StaleElement exceptions (a DOM element grabbed at one step is no longer available) + and "not clickable" exceptions (so perhaps a popup modal is blocking a click). + """ + return exception_indicates_stale_element(e) or exception_indicates_not_clickable(e) + + +def retry_call_during_transitions(f, attempts=RETRY_DURING_TRANSITIONS_ATTEMPTS_DEFAULT, sleep=RETRY_DURING_TRANSITIONS_SLEEP_DEFAULT, exception_check=excepion_seems_to_indicate_transition): previous_attempts = 0 while True: try: @@ -44,7 +68,7 @@ def retry_call_during_transitions(f, attempts=5, sleep=.1, exception_check=excep previous_attempts += 1 -def retry_during_transitions(f, attempts=5, sleep=.1, exception_check=exception_indicates_stale_element): +def retry_during_transitions(f, attempts=RETRY_DURING_TRANSITIONS_ATTEMPTS_DEFAULT, sleep=RETRY_DURING_TRANSITIONS_SLEEP_DEFAULT, exception_check=excepion_seems_to_indicate_transition): @wraps(f) def _retry(*args, **kwds): @@ -176,6 +200,12 @@ class NavigatesGalaxy(HasDriver): self.history_panel_refresh_click() return rval + def history_panel_wait_for_history_loaded(self): + # Use the search box showing up as a proxy that the history display + # has left the "loading" state and is showing a valid set of history contents + # (even if empty). + self.wait_for_selector_visible("#current-history-panel input.search-query") + def history_panel_wait_for_hid_hidden(self, hid, timeout=60): current_history_id = self.current_history_id() contents = self.api_get("histories/%s/contents" % current_history_id) @@ -201,12 +231,27 @@ class NavigatesGalaxy(HasDriver): raise self.prepend_timeout_message(e, message) return history_item_selector_state + def published_grid_search_for(self, search_term=None): + return self._inline_search_for( + '#input-free-text-search-filter', + search_term, + ) + def get_logged_in_user(self): return self.api_get("users/current") def is_logged_in(self): return "email" in self.get_logged_in_user() + @retry_during_transitions + def _inline_search_for(self, selector, search_term=None): + search_box = self.wait_for_and_click_selector(selector) + search_box.clear() + if search_term is not None: + search_box.send_keys(search_term) + self.send_enter(search_box) + return search_box + def _get_random_name(self, prefix=None, suffix=None, len=10): return '%s%s%s' % ( prefix or '', @@ -420,13 +465,19 @@ class NavigatesGalaxy(HasDriver): @retry_during_transitions def upload_build(self): - build_button = self.wait_for_selector_clickable("div#collection button#btn-build") - # TODO: Eliminate the need for this hack. This hack is in here because the test - # occasionally fails at the next step because the UI has not transitioned to the - # new content. I assume that means this click is sent before the callback is - # registered. + build_selector = "div#collection button#btn-build" + # Pause a bit to let the callback on the build button be registered. time.sleep(.5) - build_button.click() + # Click the Build button and make sure it disappears. + self.wait_for_and_click_selector(build_selector) + try: + self.wait_for_selector_absent_or_hidden(build_selector) + except TimeoutException: + # Sometimes the callback in the JS hasn't be registered by the + # time that the build button is clicked. By the time the timeout + # has been registered - it should have been. + self.wait_for_and_click_selector(build_selector) + self.wait_for_selector_absent_or_hidden(build_selector) def upload_queue_local_file(self, test_path, tab_id="regular"): self.wait_for_and_click_selector("div#%s button#btn-local" % tab_id) @@ -480,6 +531,12 @@ class NavigatesGalaxy(HasDriver): def workflow_index_click_search(self): return self.wait_for_and_click_selector("input.search-wf") + def workflow_index_search_for(self, search_term=None): + return self._inline_search_for( + "input.search-wf", + search_term, + ) + def workflow_index_click_import(self): self.wait_for_and_click_selector(self.test_data["selectors"]["workflows"]["import_button"]) @@ -516,6 +573,7 @@ class NavigatesGalaxy(HasDriver): tag_display = workflow_row_element.find_element_by_css_selector(".tags-display") tag_display.click() + @retry_during_transitions def workflow_index_tags(self, workflow_index=0): workflow_row_element = self.workflow_index_table_row(workflow_index) tag_display = workflow_row_element.find_element_by_css_selector(".tags-display") @@ -612,6 +670,11 @@ class NavigatesGalaxy(HasDriver): menu_selector = self.test_data["historyOptions"]["selectors"]["menu"] return menu_selector + @retry_during_transitions + def histories_click_advanced_search(self): + search_selector = '#standard-search .advanced-search-toggle' + self.wait_for_and_click_selector(search_selector) + def history_panel_add_tags(self, tags): tag_icon_selector = self.test_data['historyPanel']['selectors']['history']['tagIcon'] tag_area_selector = self.test_data['historyPanel']['selectors']['history']['tagArea'] @@ -708,10 +771,14 @@ class NavigatesGalaxy(HasDriver): else: item_selector = self.history_panel_item_selector(kwds["hid"]) title_selector = "%s .title" % item_selector + details_selector = "%s .details" % item_selector + details_displayed = self.selector_is_displayed(details_selector) self.wait_for_and_click_selector(title_selector) if kwds.get("wait", False): - # Find a better way to wait for transition - time.sleep(.5) + if details_displayed: + self.wait_for_selector_absent_or_hidden(details_selector) + else: + self.wait_for_selector_visible(details_selector) def click_hda_title(self, hda_id, wait=False): # TODO: Replace with calls to history_panel_click_item_title. @@ -751,7 +818,7 @@ class NavigatesGalaxy(HasDriver): self.home() with open(path, "r") as f: - tour_dict = yaml.load(f) + tour_dict = yaml.safe_load(f) steps = tour_dict["steps"] for i, step in enumerate(steps): title = step.get("title", None) @@ -796,6 +863,18 @@ class NavigatesGalaxy(HasDriver): self.click_center() return text + @retry_during_transitions + def assert_selector_absent_or_hidden_after_transitions(self, selector): + """Variant of assert_selector_absent_or_hidden that retries during transitions. + + In the parent method - the element is found and then it is checked to see + if it is visible. It may disappear from the page in the middle there + and cause a StaleElement error. For checks where we care about the final + resting state after transitions - this method can be used to retry + during those transitions. + """ + return self.assert_selector_absent_or_hidden(selector) + def assert_tooltip_text(self, element, expected, sleep=0, click_away=True): text = self.get_tooltip_text(element, sleep=sleep, click_away=click_away) assert text == expected, "Tooltip text [%s] was not expected text [%s]." % (text, expected) diff --git a/test/manual/workflows_scaling.py b/test/manual/workflows_scaling.py index 69ed5514580..677eff160df 100644 --- a/test/manual/workflows_scaling.py +++ b/test/manual/workflows_scaling.py @@ -19,12 +19,12 @@ from bioblend import galaxy galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)) sys.path[1:1] = [os.path.join(galaxy_root, "lib"), os.path.join(galaxy_root, "test")] -from base.populators import ( +from base.populators import ( # noqa: I100 GiDatasetCollectionPopulator, GiDatasetPopulator, GiWorkflowPopulator, ) -from api.workflows_format_2.converter import python_to_workflow +from api.workflows_format_2.converter import python_to_workflow # noqa: I100 LONG_TIMEOUT = 1000000000 DESCRIPTION = "Script to exercise the workflow engine." diff --git a/test/selenium_tests/framework.py b/test/selenium_tests/framework.py index f775ade1679..d4f318596f0 100644 --- a/test/selenium_tests/framework.py +++ b/test/selenium_tests/framework.py @@ -8,31 +8,26 @@ import os import time import traceback import unittest - from functools import partial, wraps import requests - -from galaxy_selenium import ( - driver_factory, -) -from galaxy_selenium.navigates_galaxy import NavigatesGalaxy, retry_during_transitions - try: from pyvirtualdisplay import Display except ImportError: Display = None - from six.moves.urllib.parse import urljoin -from base import populators -from base.driver_util import classproperty, DEFAULT_WEB_HOST, get_ip_address -from base.testcase import FunctionalTestCase -from base.workflows_format_2 import ( - ImporterGalaxyInterface, +from base import populators # noqa: I100 +from base.driver_util import classproperty, DEFAULT_WEB_HOST, get_ip_address # noqa: I100 +from base.testcase import FunctionalTestCase # noqa: I100 +from base.workflows_format_2 import ( # noqa: I100 convert_and_import_workflow, + ImporterGalaxyInterface, ) - +from galaxy_selenium import ( # noqa: I100 + driver_factory, +) +from galaxy_selenium.navigates_galaxy import NavigatesGalaxy, retry_during_transitions # noqa: I100 from galaxy.util import asbool DEFAULT_WAIT_TIMEOUT = 60 @@ -149,8 +144,8 @@ class SeleniumTestCase(FunctionalTestCase, NavigatesGalaxy): self.target_url_from_selenium = GALAXY_TEST_EXTERNAL_FROM_SELENIUM else: self.target_url_from_selenium = self.url - self.setup_driver_and_session() self.snapshots = [] + self.setup_driver_and_session() def tearDown(self): exception = None diff --git a/test/selenium_tests/test_anon_history.py b/test/selenium_tests/test_anon_history.py index a037bfefa54..b37c8448391 100644 --- a/test/selenium_tests/test_anon_history.py +++ b/test/selenium_tests/test_anon_history.py @@ -1,5 +1,7 @@ -from .framework import SeleniumTestCase -from .framework import selenium_test +from .framework import ( + selenium_test, + SeleniumTestCase +) class AnonymousHistoriesTestCase(SeleniumTestCase): @@ -38,10 +40,7 @@ class AnonymousHistoriesTestCase(SeleniumTestCase): @selenium_test def test_anon_history_after_registration(self): - self.home() - self.perform_upload(self.get_filename("1.txt")) - self.wait_for_history() - self.register() + self._upload_file_anonymous_then_register_user() hda = self.latest_history_item() self.home() element = self.wait_for_selector(self.hda_div_selector(hda["id"])) @@ -49,10 +48,16 @@ class AnonymousHistoriesTestCase(SeleniumTestCase): @selenium_test def test_clean_anon_history_after_logout(self): + self._upload_file_anonymous_then_register_user() + self.logout_if_needed() + # Give Galaxy a chance to load the new empty history for that now + # anonymous user. Make sure this new history is empty. + self.history_panel_wait_for_history_loaded() + history_contents = self.current_history_contents() + assert len(history_contents) == 0 + + def _upload_file_anonymous_then_register_user(self): self.home() self.perform_upload(self.get_filename("1.txt")) self.wait_for_history() self.register() - self.logout_if_needed() - history_contents = self.current_history_contents() - assert len(history_contents) == 0 diff --git a/test/selenium_tests/test_collection_builders.py b/test/selenium_tests/test_collection_builders.py index f63febae6b3..6b59e1070da 100644 --- a/test/selenium_tests/test_collection_builders.py +++ b/test/selenium_tests/test_collection_builders.py @@ -1,5 +1,7 @@ -from .framework import SeleniumTestCase -from .framework import selenium_test +from .framework import ( + selenium_test, + SeleniumTestCase +) class CollectionBuildersTestCase(SeleniumTestCase): diff --git a/test/selenium_tests/test_custom_builds.py b/test/selenium_tests/test_custom_builds.py index f29d04e189b..f15cef7b4df 100644 --- a/test/selenium_tests/test_custom_builds.py +++ b/test/selenium_tests/test_custom_builds.py @@ -9,19 +9,19 @@ from .framework import ( class CustomBuildsTestcase(SharedStateSeleniumTestCase): - def setUp(self): - super(CustomBuildsTestcase, self).setUp() - self.home() # ensure Galaxy is loaded - self.submit_login(self.user_email, retries=2) - @selenium_test def test_build_add(self): + self._login() + self.navigate_to_custom_builds_page() + self.add_custom_build(self.build_name1, self.build_key1) self.assert_custom_builds_in_grid([self.build_name1]) @selenium_test def test_build_delete(self): + self._login() + self.navigate_to_custom_builds_page() self.add_custom_build(self.build_name2, self.build_key2) @@ -39,6 +39,10 @@ class CustomBuildsTestcase(SharedStateSeleniumTestCase): else: self.assertEqual(intersection, set()) + def _login(self): + self.home() # ensure Galaxy is loaded + self.submit_login(self.user_email, retries=2) + def add_custom_build(self, build_name, build_key): name_div = self.wait_for_selector('div[tour_id="name"') name_input = name_div.find_element_by_css_selector('.ui-form-field input') diff --git a/test/selenium_tests/test_history_dataset_state.py b/test/selenium_tests/test_history_dataset_state.py index 07ecb9becf5..95b86b015de 100644 --- a/test/selenium_tests/test_history_dataset_state.py +++ b/test/selenium_tests/test_history_dataset_state.py @@ -1,6 +1,6 @@ from .framework import ( - SeleniumTestCase, selenium_test, + SeleniumTestCase, UsesHistoryItemAssertions, ) diff --git a/test/selenium_tests/test_history_options.py b/test/selenium_tests/test_history_options.py index 8e8f975801c..d4dad146da4 100644 --- a/test/selenium_tests/test_history_options.py +++ b/test/selenium_tests/test_history_options.py @@ -1,5 +1,7 @@ -from .framework import SeleniumTestCase -from .framework import selenium_test +from .framework import ( + selenium_test, + SeleniumTestCase +) class HistoryOptionsTestCase(SeleniumTestCase): @@ -27,4 +29,4 @@ class HistoryOptionsTestCase(SeleniumTestCase): self.click_hda_title(hda_id, wait=True) - self.assert_selector_absent_or_hidden(hda_body_selector) + self.assert_selector_absent_or_hidden_after_transitions(hda_body_selector) diff --git a/test/selenium_tests/test_history_panel.py b/test/selenium_tests/test_history_panel.py index 12c514b7306..f1aae2d5026 100644 --- a/test/selenium_tests/test_history_panel.py +++ b/test/selenium_tests/test_history_panel.py @@ -1,7 +1,9 @@ import time -from .framework import SeleniumTestCase -from .framework import selenium_test +from .framework import ( + selenium_test, + SeleniumTestCase +) class HistoryPanelTestCase(SeleniumTestCase): diff --git a/test/selenium_tests/test_history_sharing.py b/test/selenium_tests/test_history_sharing.py index dc77592b02f..7129c2713ff 100644 --- a/test/selenium_tests/test_history_sharing.py +++ b/test/selenium_tests/test_history_sharing.py @@ -1,5 +1,7 @@ -from .framework import SeleniumTestCase -from .framework import selenium_test +from .framework import ( + selenium_test, + SeleniumTestCase +) # Remove hack when submit_login works more consistently. VALID_LOGIN_RETRIES = 3 diff --git a/test/selenium_tests/test_login.py b/test/selenium_tests/test_login.py index e782728ea0c..cbdac4fc3d1 100644 --- a/test/selenium_tests/test_login.py +++ b/test/selenium_tests/test_login.py @@ -1,5 +1,7 @@ -from .framework import SeleniumTestCase -from .framework import selenium_test +from .framework import ( + selenium_test, + SeleniumTestCase +) class LoginTestCase(SeleniumTestCase): diff --git a/test/selenium_tests/test_published_histories_grid.py b/test/selenium_tests/test_published_histories_grid.py index 9937a19e602..7cf86e06884 100644 --- a/test/selenium_tests/test_published_histories_grid.py +++ b/test/selenium_tests/test_published_histories_grid.py @@ -9,10 +9,6 @@ from .framework import ( class HistoryGridTestCase(SharedStateSeleniumTestCase): - def setUp(self): - super(HistoryGridTestCase, self).setUp() - self.home() - @selenium_test def test_history_grid_histories(self): self.navigate_to_published_histories_page() @@ -22,17 +18,11 @@ class HistoryGridTestCase(SharedStateSeleniumTestCase): def test_history_grid_search_standard(self): self.navigate_to_published_histories_page() - input_selector = '#input-free-text-search-filter' - search_input = self.wait_for_selector(input_selector) - search_input.send_keys(self.history1_name) - self.send_enter(search_input) - + self.published_grid_search_for(self.history1_name) self.assert_grid_histories_are([self.history1_name]) self.unset_filter('free-text-search', self.history1_name) - search_input = self.wait_for_selector(input_selector) - search_input.send_keys(self.history4_name) - self.send_enter(search_input) + self.published_grid_search_for(self.history4_name) self.assert_grid_histories_are(['No Items']) diff --git a/test/selenium_tests/test_saved_histories.py b/test/selenium_tests/test_saved_histories.py index 4272ec46b93..6e3221c0db4 100644 --- a/test/selenium_tests/test_saved_histories.py +++ b/test/selenium_tests/test_saved_histories.py @@ -2,6 +2,7 @@ import time from .framework import ( retry_assertion_during_transitions, + retry_during_transitions, selenium_test, SharedStateSeleniumTestCase, ) @@ -9,26 +10,29 @@ from .framework import ( class SavedHistoriesTestCase(SharedStateSeleniumTestCase): - def setUp(self): - super(SavedHistoriesTestCase, self).setUp() - self.home() - self.submit_login(self.user_email, retries=3) - @selenium_test def test_saved_histories_list(self): + self._login() self.navigate_to_saved_histories_page() self.assert_histories_in_grid([self.history2_name, self.history3_name]) @selenium_test def test_history_switch(self): + self._login() self.navigate_to_saved_histories_page() self.click_popup_option(self.history2_name, 'Switch') - time.sleep(1) - history_name = self.history_panel_name_element() - self.assertEqual(history_name.text, self.history2_name) + time.sleep(.5) + + @retry_assertion_during_transitions + def assert_history_name_switched(): + history_name = self.history_panel_name_element() + self.assertEqual(history_name.text, self.history2_name) + + assert_history_name_switched() @selenium_test def test_history_view(self): + self._login() self.navigate_to_saved_histories_page() self.click_popup_option(self.history2_name, 'View') history_name = self.wait_for_selector('.name.editable-text') @@ -36,6 +40,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): @selenium_test def test_history_publish(self): + self._login() self.navigate_to_saved_histories_page() # Publish the history @@ -44,7 +49,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self.navigate_to_saved_histories_page() - self.show_advanced_search() + self.histories_click_advanced_search() self.select_filter('sharing', 'published') time.sleep(1) @@ -52,19 +57,17 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): @selenium_test def test_rename_history(self): + self._login() self.navigate_to_saved_histories_page() self.click_popup_option('Unnamed history', 'Rename') # Rename the history - history_name_input = self.wait_for_selector('input[name="name"]') + history_name_input = self.wait_for_selector('.ui-form-element input.ui-input') history_name_input.clear() history_name_input.send_keys(self.history1_name) - self.send_enter(history_name_input) - message = self.wait_for_selector_visible('.infomessagelarge') - expected_message = 'History: Unnamed history renamed to: %s' % self.history1_name - self.assertEqual(expected_message, message.text) + self.wait_for_and_click_selector("button#submit") self.navigate_to_saved_histories_page() @@ -72,6 +75,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): @selenium_test def test_delete_and_undelete_history(self): + self._login() self.navigate_to_saved_histories_page() # Delete the history @@ -79,7 +83,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self.assert_histories_in_grid([self.history2_name], False) - self.show_advanced_search() + self.histories_click_advanced_search() self.select_filter('deleted', 'True') time.sleep(1) @@ -93,6 +97,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): @selenium_test def test_permanently_delete_history(self): + self._login() self.create_history(self.history4_name) self.navigate_to_saved_histories_page() @@ -104,13 +109,14 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self.assert_histories_in_grid([self.history4_name], False) - self.show_advanced_search() + self.histories_click_advanced_search() self.select_filter('deleted', 'True') self.assert_histories_in_grid([self.history4_name]) @selenium_test def test_delete_and_undelete_multiple_histories(self): + self._login() self.navigate_to_saved_histories_page() delete_button_selector = 'input[type="button"][value="Delete"]' @@ -122,7 +128,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self.assert_histories_in_grid([self.history2_name, self.history3_name], False) - self.show_advanced_search() + self.histories_click_advanced_search() self.select_filter('deleted', 'True') time.sleep(1) @@ -139,6 +145,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): @selenium_test def test_sort_by_name(self): + self._login() self.navigate_to_saved_histories_page() self.wait_for_and_click_selector('.sort-link[sort_key="name"]') @@ -156,6 +163,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): @selenium_test def test_standard_search(self): + self._login() self.navigate_to_saved_histories_page() input_selector = '#input-free-text-search-filter' @@ -170,13 +178,14 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): search_input.send_keys(self.history4_name) self.send_enter(search_input) - self.assert_grid_histories_are(['No Items']) + self.assert_grid_histories_are([]) @selenium_test def test_advanced_search(self): + self._login() self.navigate_to_saved_histories_page() - self.show_advanced_search() + self.histories_click_advanced_search() name_filter_selector = '#input-name-filter' tags_filter_selector = '#input-tags-filter' @@ -187,7 +196,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self.unset_filter('name', self.history2_name) self.set_filter(name_filter_selector, self.history4_name) - self.assert_grid_histories_are(['No Items']) + self.assert_grid_histories_are([]) self.unset_filter('name', self.history4_name) # Search by tags @@ -196,11 +205,12 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self.unset_filter('tags', self.history3_tags[0]) self.set_filter(tags_filter_selector, self.history4_tags[0]) - self.assert_grid_histories_are(['No Items']) + self.assert_grid_histories_are([]) self.unset_filter('tags', self.history4_tags[0]) @selenium_test def test_tags(self): + self._login() self.navigate_to_saved_histories_page() # Click the add tag button @@ -221,6 +231,10 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self.assert_grid_histories_are([self.history2_name], False) + def _login(self): + self.home() + self.submit_login(self.user_email, retries=3) + @retry_assertion_during_transitions def assert_grid_histories_are(self, expected_histories, sort_matters=True): actual_histories = self.get_histories() @@ -238,8 +252,9 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): else: self.assertEqual(intersection, set()) + @retry_during_transitions def get_histories(self): - time.sleep(1.5) + time.sleep(.5) names = [] grid = self.wait_for_selector('#grid-table-body') for row in grid.find_elements_by_tag_name('tr'): @@ -288,10 +303,6 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self.click_history_option('Create New') self.history_panel_rename(name) - def show_advanced_search(self): - search_selector = '#standard-search .advanced-search-toggle' - self.wait_for_and_click_selector(search_selector) - def select_filter(self, filter_key, filter_value): filter_selector = 'a[filter_key="%s"][filter_val="%s"]' % \ (filter_key, filter_value) diff --git a/test/selenium_tests/test_sizzle_loading.py b/test/selenium_tests/test_sizzle_loading.py index 022e6fdfb67..94fdd6bb588 100644 --- a/test/selenium_tests/test_sizzle_loading.py +++ b/test/selenium_tests/test_sizzle_loading.py @@ -1,5 +1,7 @@ -from .framework import SeleniumTestCase -from .framework import selenium_test +from .framework import ( + selenium_test, + SeleniumTestCase +) class SizzleLoadingTestCase(SeleniumTestCase): diff --git a/test/selenium_tests/test_stock_tours.py b/test/selenium_tests/test_stock_tours.py index 4e3b1059bab..66ac07a5409 100644 --- a/test/selenium_tests/test_stock_tours.py +++ b/test/selenium_tests/test_stock_tours.py @@ -1,10 +1,12 @@ import os -from .framework import SeleniumTestCase -from .framework import selenium_test - from galaxy.util import galaxy_root_path +from .framework import ( + selenium_test, + SeleniumTestCase +) + STOCK_TOURS_DIRECTORY = os.path.join(galaxy_root_path, "config", "plugins", "tours") diff --git a/test/selenium_tests/test_tool_form.py b/test/selenium_tests/test_tool_form.py index 6d726c13621..38ad877971c 100644 --- a/test/selenium_tests/test_tool_form.py +++ b/test/selenium_tests/test_tool_form.py @@ -1,11 +1,11 @@ +from galaxy_selenium.navigates_galaxy import retry_call_during_transitions + from .framework import ( - SeleniumTestCase, selenium_test, + SeleniumTestCase, UsesHistoryItemAssertions, ) -from galaxy_selenium.navigates_galaxy import retry_call_during_transitions - class ToolFormTestCase(SeleniumTestCase, UsesHistoryItemAssertions): diff --git a/test/selenium_tests/test_uploads.py b/test/selenium_tests/test_uploads.py index 7e5cdcd55d3..d9ad6cdd570 100644 --- a/test/selenium_tests/test_uploads.py +++ b/test/selenium_tests/test_uploads.py @@ -1,6 +1,6 @@ from .framework import ( - SeleniumTestCase, selenium_test, + SeleniumTestCase, UsesHistoryItemAssertions, ) diff --git a/test/selenium_tests/test_workflow_editor.py b/test/selenium_tests/test_workflow_editor.py index 6dd7044b500..d44d252bd03 100644 --- a/test/selenium_tests/test_workflow_editor.py +++ b/test/selenium_tests/test_workflow_editor.py @@ -1,14 +1,14 @@ import time -from .framework import ( - selenium_test, - SeleniumTestCase -) - from ._workflow_fixtures import ( WORKFLOW_SIMPLE_CAT_TWICE, - WORKFLOW_WITH_OLD_TOOL_VERSION, WORKFLOW_WITH_INVALID_STATE, + WORKFLOW_WITH_OLD_TOOL_VERSION, +) +from .framework import ( + retry_assertion_during_transitions, + selenium_test, + SeleniumTestCase ) @@ -52,10 +52,7 @@ class WorkflowEditorTestCase(SeleniumTestCase): workflow_populator.upload_yaml_workflow(WORKFLOW_WITH_OLD_TOOL_VERSION, exact_tools=True) self.workflow_index_open() self.workflow_index_click_option("Edit") - time.sleep(.5) - modal_element = self.wait_for_selector_visible(self.modal_body_selector()) - text = modal_element.text - assert "Using version '0.2' instead of version '0.0.1'" in text, text + self.assert_modal_has_text("Using version '0.2' instead of version '0.0.1'") @selenium_test def test_editor_invalid_tool_state(self): @@ -63,11 +60,8 @@ class WorkflowEditorTestCase(SeleniumTestCase): workflow_populator.upload_yaml_workflow(WORKFLOW_WITH_INVALID_STATE, exact_tools=True) self.workflow_index_open() self.workflow_index_click_option("Edit") - time.sleep(.5) - modal_element = self.wait_for_selector_visible(self.modal_body_selector()) - text = modal_element.text - assert "Using version '0.2' instead of version '0.0.1'" in text, text - assert "Using default: '1'" in text, text + self.assert_modal_has_text("Using version '0.2' instead of version '0.0.1'") + self.assert_modal_has_text("Using default: '1'") @selenium_test def test_missing_tools(self): @@ -84,10 +78,7 @@ steps: """) self.workflow_index_open() self.workflow_index_click_option("Edit") - time.sleep(.5) - modal_element = self.wait_for_selector_visible(self.modal_body_selector()) - text = modal_element.text - assert "Tool is not installed" in text, text + self.assert_modal_has_text("Tool is not installed") def workflow_create_new(self, name=None, annotation=None): self.workflow_index_open() @@ -104,3 +95,9 @@ steps: 'workflow_annotation': annotation, }) self.click_submit(form_element) + + @retry_assertion_during_transitions + def assert_modal_has_text(self, expected_text): + modal_element = self.wait_for_selector_visible(self.modal_body_selector()) + text = modal_element.text + assert expected_text in text, "Failed to find expected text [%s] in modal text [%s]" % (expected_text, text) diff --git a/test/selenium_tests/test_workflow_management.py b/test/selenium_tests/test_workflow_management.py index 776c4ed9ee7..760d1c85108 100644 --- a/test/selenium_tests/test_workflow_management.py +++ b/test/selenium_tests/test_workflow_management.py @@ -73,20 +73,13 @@ class WorkflowManagementTestCase(SeleniumTestCase): self.workflow_index_rename("searchforthis") self._assert_showing_n_workflows(1) - search_box = self.workflow_index_click_search() - search_box.send_keys("doesnotmatch") + self.workflow_index_search_for("doesnotmatch") self._assert_showing_n_workflows(0) - # Prevent stale element textbox by re-fetching, seems to be - # needed but I don't understand why exactly. -John - search_box = self.workflow_index_click_search() - search_box.clear() - self.send_enter(search_box) + self.workflow_index_search_for() self._assert_showing_n_workflows(1) - search_box = self.workflow_index_click_search() - search_box.send_keys("searchforthis") - self.send_enter(search_box) + self.workflow_index_search_for("searchforthis") self._assert_showing_n_workflows(1) @selenium_test diff --git a/test/selenium_tests/test_workflow_run.py b/test/selenium_tests/test_workflow_run.py index 4533ecacabf..8bdc834f7b2 100644 --- a/test/selenium_tests/test_workflow_run.py +++ b/test/selenium_tests/test_workflow_run.py @@ -1,15 +1,14 @@ import time -from .framework import ( - selenium_test, - SeleniumTestCase, - UsesHistoryItemAssertions, -) - from ._workflow_fixtures import ( WORKFLOW_SIMPLE_CAT_TWICE, WORKFLOW_WITH_OLD_TOOL_VERSION, ) +from .framework import ( + selenium_test, + SeleniumTestCase, + UsesHistoryItemAssertions, +) class WorkflowRunTestCase(SeleniumTestCase, UsesHistoryItemAssertions): diff --git a/test/shed_functional/base/test_db_util.py b/test/shed_functional/base/test_db_util.py index c9ba1356555..f1db34bc614 100644 --- a/test/shed_functional/base/test_db_util.py +++ b/test/shed_functional/base/test_db_util.py @@ -5,9 +5,11 @@ from sqlalchemy import and_, false, true import galaxy.model import galaxy.model.tool_shed_install import galaxy.webapps.tool_shed.model as model - -from base.driver_util import (galaxy_context as ga_session, - install_context as install_session, tool_shed_context as sa_session) +from base.driver_util import ( # noqa: I100 + galaxy_context as ga_session, + install_context as install_session, + tool_shed_context as sa_session +) log = logging.getLogger('test.tool_shed.test_db_util') diff --git a/test/shed_functional/base/twilltestcase.py b/test/shed_functional/base/twilltestcase.py index 0ab2da9ddf4..681e51b3a4d 100644 --- a/test/shed_functional/base/twilltestcase.py +++ b/test/shed_functional/base/twilltestcase.py @@ -15,8 +15,8 @@ from six.moves.urllib.parse import quote_plus, urlencode import galaxy.model.tool_shed_install as galaxy_model import galaxy.util import galaxy.webapps.tool_shed.util.hgweb_config -from base.tool_shed_util import repository_installation_timeout -from functional.twilltestcase import TwillTestCase +from base.tool_shed_util import repository_installation_timeout # noqa: I100 +from functional.twilltestcase import TwillTestCase # noqa: I100 from galaxy.web import security from tool_shed.util import hg_util, xml_util from tool_shed.util.encoding_util import tool_shed_encode diff --git a/test/unit/__init__.py b/test/unit/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/unit/datatypes/dataproviders/test_base_dataproviders.py b/test/unit/datatypes/dataproviders/test_base_dataproviders.py index 883e12ea2f5..e081cc41ac4 100644 --- a/test/unit/datatypes/dataproviders/test_base_dataproviders.py +++ b/test/unit/datatypes/dataproviders/test_base_dataproviders.py @@ -3,17 +3,13 @@ Unit tests for base DataProviders. .. seealso:: galaxy.datatypes.dataproviders.base """ import logging -import os.path -import sys import unittest from six import StringIO from galaxy.datatypes.dataproviders import base, exceptions -unit_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) -sys.path.insert(1, unit_root) -from unittest_utils import tempfilecache, utility +from ...unittest_utils import tempfilecache, utility log = logging.getLogger(__name__) diff --git a/test/unit/jobs/test_job_wrapper.py b/test/unit/jobs/test_job_wrapper.py index 8341261e892..268daf2315f 100644 --- a/test/unit/jobs/test_job_wrapper.py +++ b/test/unit/jobs/test_job_wrapper.py @@ -14,7 +14,7 @@ from galaxy.model import ( from galaxy.tools import evaluation from galaxy.util.bunch import Bunch -from tools_support import UsesApp +from ..tools_support import UsesApp TEST_TOOL_ID = "cufftest" TEST_VERSION_COMMAND = "bwa --version" @@ -162,6 +162,7 @@ class MockTool(object): self.version_string_cmd = TEST_VERSION_COMMAND self.tool_dir = "/path/to/tools" self.dependencies = [] + self.requires_galaxy_python_environment = False def build_dependency_shell_commands(self, job_directory): return TEST_DEPENDENCIES_COMMANDS diff --git a/test/unit/jobs/test_runner_local.py b/test/unit/jobs/test_runner_local.py index 70857bb3bb6..e9c85a301c0 100644 --- a/test/unit/jobs/test_runner_local.py +++ b/test/unit/jobs/test_runner_local.py @@ -8,7 +8,7 @@ from galaxy.jobs import metrics from galaxy.jobs.runners import local from galaxy.util import bunch -from tools_support import ( +from ..tools_support import ( UsesApp, UsesTools ) diff --git a/test/unit/managers/base.py b/test/unit/managers/base.py index 5291d636e9a..c1722e3c8dd 100644 --- a/test/unit/managers/base.py +++ b/test/unit/managers/base.py @@ -3,8 +3,6 @@ from __future__ import print_function import json -import os -import sys import unittest import sqlalchemy @@ -12,9 +10,7 @@ from six import string_types from galaxy.managers.users import UserManager -unit_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) -sys.path.insert(1, unit_root) -from unittest_utils import galaxy_mock +from ..unittest_utils import galaxy_mock # ============================================================================= admin_email = 'admin@admin.admin' diff --git a/test/unit/managers/test_HistoryManager.py b/test/unit/managers/test_HistoryManager.py index 1d741400374..530295b9c82 100644 --- a/test/unit/managers/test_HistoryManager.py +++ b/test/unit/managers/test_HistoryManager.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- """ """ -import os -import sys import unittest import sqlalchemy @@ -24,11 +22,8 @@ from galaxy.managers.histories import ( HistorySerializer ) -unit_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) -sys.path.insert(1, unit_root) -from unittest_utils import galaxy_mock - from .base import BaseTestCase +from ..unittest_utils import galaxy_mock default_password = '123456' user2_data = dict(email='user2@user2.user2', username='user2', password=default_password) diff --git a/test/unit/shed_unit/test_installed_repository_manager.py b/test/unit/shed_unit/test_installed_repository_manager.py index 3fc33637363..ffd3f57ff06 100644 --- a/test/unit/shed_unit/test_installed_repository_manager.py +++ b/test/unit/shed_unit/test_installed_repository_manager.py @@ -1,5 +1,6 @@ from tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager -from tools.test_toolbox import BaseToolBoxTestCase + +from ..tools.test_toolbox import BaseToolBoxTestCase class InstalledRepositoryManagerTestCase(BaseToolBoxTestCase): diff --git a/test/unit/shed_unit/test_tool_panel_manager.py b/test/unit/shed_unit/test_tool_panel_manager.py index e6cab3a53cc..c3ab72fb727 100644 --- a/test/unit/shed_unit/test_tool_panel_manager.py +++ b/test/unit/shed_unit/test_tool_panel_manager.py @@ -4,7 +4,8 @@ from galaxy.util import parse_xml from tool_shed.galaxy_install.tools import tool_panel_manager from tool_shed.tools import tool_version_manager -from tools.test_toolbox import ( + +from ..tools.test_toolbox import ( BaseToolBoxTestCase, SimplifiedToolBox ) diff --git a/test/unit/tools/test_actions.py b/test/unit/tools/test_actions.py index 1978f6dd109..5d0e0d83db1 100644 --- a/test/unit/tools/test_actions.py +++ b/test/unit/tools/test_actions.py @@ -2,7 +2,6 @@ import string import unittest from xml.etree.ElementTree import XML -import tools_support from galaxy import model from galaxy.tools.actions import ( DefaultToolAction, @@ -11,6 +10,8 @@ from galaxy.tools.actions import ( ) from galaxy.tools.parser.output_objects import ToolOutput +from .. import tools_support + TEST_HANDLER_NAME = "test_handler_1" diff --git a/test/unit/tools/test_collect_primary_datasets.py b/test/unit/tools/test_collect_primary_datasets.py index b5317e1b358..9c04b3301e0 100644 --- a/test/unit/tools/test_collect_primary_datasets.py +++ b/test/unit/tools/test_collect_primary_datasets.py @@ -2,7 +2,6 @@ import json import os import unittest -import tools_support from galaxy import ( model, util @@ -10,6 +9,7 @@ from galaxy import ( from galaxy.tools.parameters.output_collect import LegacyToolProvidedMetadata, NullToolProvidedMetadata from galaxy.tools.parser import output_collection_def +from .. import tools_support DEFAULT_TOOL_OUTPUT = "out1" DEFAULT_EXTRA_NAME = "test1" diff --git a/test/unit/tools/test_column_parameters.py b/test/unit/tools/test_column_parameters.py index ab40fd07851..9df583cdfcc 100644 --- a/test/unit/tools/test_column_parameters.py +++ b/test/unit/tools/test_column_parameters.py @@ -1,12 +1,11 @@ """ Tests for tool parameters, more tests exist in test_data_parameters.py and test_select_parameters.py. """ - from galaxy import model from galaxy.util import bunch -from tools_support import datatypes_registry from .test_parameter_parsing import BaseParameterTestCase +from ..tools_support import datatypes_registry class DataColumnParameterTestCase(BaseParameterTestCase): diff --git a/test/unit/tools/test_data_parameters.py b/test/unit/tools/test_data_parameters.py index 561dff9e205..9fd0f521042 100644 --- a/test/unit/tools/test_data_parameters.py +++ b/test/unit/tools/test_data_parameters.py @@ -1,14 +1,8 @@ -import os -import sys - from galaxy import model from galaxy.util import bunch -unit_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) -sys.path.insert(1, unit_root) -from unittest_utils import galaxy_mock - from .test_parameter_parsing import BaseParameterTestCase +from ..unittest_utils import galaxy_mock class DataToolParameterTestCase(BaseParameterTestCase): diff --git a/test/unit/tools/test_dataset_matcher.py b/test/unit/tools/test_dataset_matcher.py index 98c2a7d30bd..fe5b231eeb5 100644 --- a/test/unit/tools/test_dataset_matcher.py +++ b/test/unit/tools/test_dataset_matcher.py @@ -1,7 +1,6 @@ from unittest import TestCase from xml.etree.ElementTree import XML -import tools_support from galaxy import model from galaxy.tools.parameters import ( basic, @@ -10,9 +9,10 @@ from galaxy.tools.parameters import ( from galaxy.util import bunch from .test_data_parameters import MockHistoryDatasetAssociation +from ..tools_support import UsesApp -class DatasetMatcherTestCase(TestCase, tools_support.UsesApp): +class DatasetMatcherTestCase(TestCase, UsesApp): def test_hda_accessible(self): # Cannot access errored or discard datasets. diff --git a/test/unit/tools/test_evaluation.py b/test/unit/tools/test_evaluation.py index be7c9db772e..d0e38fc561c 100644 --- a/test/unit/tools/test_evaluation.py +++ b/test/unit/tools/test_evaluation.py @@ -27,8 +27,9 @@ from galaxy.tools.parameters.grouping import ( ) from galaxy.tools.parser.output_objects import ToolOutput from galaxy.util.bunch import Bunch + # Test fixtures for Galaxy infrastructure. -from tools_support import UsesApp +from ..tools_support import UsesApp # To Test: # - param_file handling. diff --git a/test/unit/tools/test_execution.py b/test/unit/tools/test_execution.py index f28606ff989..3182a271995 100644 --- a/test/unit/tools/test_execution.py +++ b/test/unit/tools/test_execution.py @@ -5,11 +5,12 @@ from unittest import TestCase from paste import httpexceptions import galaxy.model -import tools_support from galaxy.tools.parameters import params_to_incoming from galaxy.util.bunch import Bunch from galaxy.util.odict import odict +from .. import tools_support + BASE_REPEAT_TOOL_CONTENTS = ''' echo "$param1" #for $r in $repeat# "$r.param2" #end for# < $out1 diff --git a/test/unit/tools/test_history_imp_exp.py b/test/unit/tools/test_history_imp_exp.py index 599cad1d37e..02a6f967e3f 100644 --- a/test/unit/tools/test_history_imp_exp.py +++ b/test/unit/tools/test_history_imp_exp.py @@ -1,5 +1,4 @@ import os -import sys import tarfile from shutil import rmtree from tempfile import mkdtemp @@ -8,9 +7,7 @@ from galaxy import model from galaxy.exceptions import MalformedContents from galaxy.tools.imp_exp import JobImportHistoryArchiveWrapper, unpack_tar_gz_archive -unit_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) -sys.path.insert(1, unit_root) -from unittest_utils.galaxy_mock import MockApp +from ..unittest_utils.galaxy_mock import MockApp # good enough for the very specific tests we're writing as of now... diff --git a/test/unit/tools/test_parameter_parsing.py b/test/unit/tools/test_parameter_parsing.py index 32da6acb482..e6b3939b0b1 100644 --- a/test/unit/tools/test_parameter_parsing.py +++ b/test/unit/tools/test_parameter_parsing.py @@ -1,13 +1,14 @@ from unittest import TestCase from xml.etree.ElementTree import XML -import tools_support from galaxy import model from galaxy.tools.parameters import basic from galaxy.util import bunch +from ..tools_support import UsesApp -class BaseParameterTestCase(TestCase, tools_support.UsesApp): + +class BaseParameterTestCase(TestCase, UsesApp): def setUp(self): self.setup_app() diff --git a/test/unit/tools/test_toolbox.py b/test/unit/tools/test_toolbox.py index cf941b22e1b..d9366a8dab6 100644 --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -7,7 +7,6 @@ import unittest import routes from six import string_types -import tools_support from galaxy import model from galaxy.model import tool_shed_install from galaxy.model.tool_shed_install import mapping @@ -15,11 +14,12 @@ from galaxy.tools import ToolBox from galaxy.tools.cache import ToolCache from galaxy.webapps.galaxy.config_watchers import ConfigWatchers -from .test_toolbox_filters import mock_trans from .test_tool_loader import ( SIMPLE_MACRO, SIMPLE_TOOL_WITH_MACRO ) +from .test_toolbox_filters import mock_trans +from ..tools_support import UsesApp, UsesTools CONFIG_TEST_TOOL_VERSION_TEMPLATE = string.Template( @@ -37,7 +37,7 @@ CONFIG_TEST_TOOL_VERSION_1 = CONFIG_TEST_TOOL_VERSION_TEMPLATE.safe_substitute(d CONFIG_TEST_TOOL_VERSION_2 = CONFIG_TEST_TOOL_VERSION_TEMPLATE.safe_substitute(dict(version="2")) -class BaseToolBoxTestCase(unittest.TestCase, tools_support.UsesApp, tools_support.UsesTools): +class BaseToolBoxTestCase(unittest.TestCase, UsesApp, UsesTools): @property def integerated_tool_panel_path(self): diff --git a/test/unit/tools/test_toolbox_filters.py b/test/unit/tools/test_toolbox_filters.py index e334faa7c1e..b1fcf1da3bb 100644 --- a/test/unit/tools/test_toolbox_filters.py +++ b/test/unit/tools/test_toolbox_filters.py @@ -1,5 +1,4 @@ from galaxy.tools.toolbox.filters import FilterFactory - from galaxy.util.bunch import Bunch @@ -50,7 +49,7 @@ def filter_factory(config_dict=None): tool_label_filters=["filtermod:filter_label_1", "filtermod:filter_label_2"], ) config = Bunch(**config_dict) - config.toolbox_filter_base_modules = "galaxy.tools.filters,tools.filter_modules" + config.toolbox_filter_base_modules = "galaxy.tools.filters,unit.tools.filter_modules" app = Bunch(config=config) toolbox = Bunch(app=app) return FilterFactory(toolbox) diff --git a/test/unit/visualizations/plugins/test_VisualizationPlugin.py b/test/unit/visualizations/plugins/test_VisualizationPlugin.py index 96460fabe21..eae02765c60 100644 --- a/test/unit/visualizations/plugins/test_VisualizationPlugin.py +++ b/test/unit/visualizations/plugins/test_VisualizationPlugin.py @@ -1,8 +1,6 @@ """ Test lib/galaxy/visualization/plugins/plugin. """ -import os -import sys import unittest from six import string_types @@ -13,12 +11,9 @@ from galaxy.visualization.plugins import ( utils as vis_utils ) -unit_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) -sys.path.insert(1, unit_root) -from unittest_utils import galaxy_mock, utility +from ...unittest_utils import galaxy_mock, utility -# ----------------------------------------------------------------------------- class VisualizationsPlugin_TestCase(unittest.TestCase): plugin_class = vis_plugin.VisualizationPlugin diff --git a/test/unit/visualizations/plugins/test_VisualizationsRegistry.py b/test/unit/visualizations/plugins/test_VisualizationsRegistry.py index dc00310ced2..7ed3b8af10a 100644 --- a/test/unit/visualizations/plugins/test_VisualizationsRegistry.py +++ b/test/unit/visualizations/plugins/test_VisualizationsRegistry.py @@ -3,7 +3,6 @@ Test lib/galaxy/visualization/plugins/registry. """ import os import re -import sys import unittest from six import string_types @@ -12,11 +11,8 @@ from galaxy import model from galaxy.visualization.plugins import plugin from galaxy.visualization.plugins.registry import VisualizationsRegistry -unit_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) -sys.path.insert(1, unit_root) -from unittest_utils import galaxy_mock, utility +from ...unittest_utils import galaxy_mock, utility -# ----------------------------------------------------------------------------- glx_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, os.pardir)) template_cache_dir = os.path.join(glx_dir, 'database', 'compiled_templates') addtional_templates_dir = os.path.join(glx_dir, 'config', 'plugins', 'visualizations', 'common', 'templates') @@ -41,7 +37,6 @@ config1 = """\ """ -# ----------------------------------------------------------------------------- class VisualizationsRegistry_TestCase(unittest.TestCase): def test_plugin_load_from_repo(self): diff --git a/test/unit/web/framework/test_webapp.py b/test/unit/web/framework/test_webapp.py index d6efd9bfb45..319b980498c 100644 --- a/test/unit/web/framework/test_webapp.py +++ b/test/unit/web/framework/test_webapp.py @@ -2,17 +2,13 @@ Unit tests for ``galaxy.web.framework.webapp`` """ import logging -import os import re -import sys import unittest import galaxy.config from galaxy.web.framework import webapp as Webapp -unit_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) -sys.path.insert(1, unit_root) -from unittest_utils import galaxy_mock +from ...unittest_utils import galaxy_mock log = logging.getLogger(__name__) diff --git a/test/unit/workflows/workflow_support.py b/test/unit/workflows/workflow_support.py index ee04ded4fc9..6f4f567cb66 100644 --- a/test/unit/workflows/workflow_support.py +++ b/test/unit/workflows/workflow_support.py @@ -75,7 +75,7 @@ class TestToolbox(object): def yaml_to_model(has_dict, id_offset=100): if isinstance(has_dict, str): - has_dict = yaml.load(has_dict) + has_dict = yaml.safe_load(has_dict) workflow = model.Workflow() workflow.steps = []