Merge branch 'dev' into replace_visualization_edit_form

2026-10-30 18:13:23 +08:00 · 2017-09-26 15:40:29 -04:00
parent daa973a5d9 0dd7f01008
commit b482ac162b
149 changed files with 702 additions and 855 deletions
@@ -1,9 +1,6 @@
 client/galaxy/style/source_material/circle.py
 contrib/
-cron/add_manual_builds.py
-cron/cleanup_datasets.py
-cron/parse_builds_3_sites.py
-cron/parse_builds.py
+cron/
 doc/parse_gx_xsd.py
 lib/galaxy/actions/
 lib/galaxy/auth/
@@ -77,6 +74,7 @@ lib/galaxy/visualization/genome/
 lib/galaxy/visualization/genomes.py
 lib/galaxy/visualization/__init__.py
 lib/galaxy/visualization/plugins/__init__.py
+lib/galaxy/visualization/plugins/plugin.py
 lib/galaxy/visualization/plugins/utils.py
 lib/galaxy/visualization/tracks/
 lib/galaxy/web/base/
@@ -152,182 +150,7 @@ lib/log_tempfile.py
 lib/mimeparse.py
 lib/psyco_full.py
 lib/tool_shed/
-scripts/api/common.py
-scripts/api/copy_hda_to_library_folder.py
-scripts/api/create.py
-scripts/api/delete.py
-scripts/api/example_watch_folder.py
-scripts/api/filter_failed_datasets_from_collection.py
-scripts/api/form_create_from_xml.py
-scripts/api/history_create_history.py
-scripts/api/history_delete_history.py
-scripts/api/import_library_dataset_to_history.py
-scripts/api/import_workflows_from_installed_tool_shed_repository.py
-scripts/api/install_tool_shed_repositories.py
-scripts/api/library_create_folder.py
-scripts/api/library_create_library.py
-scripts/api/library_upload_dir.py
-scripts/api/library_upload_from_import_dir.py
-scripts/api/load_data_with_metadata.py
-scripts/api/repair_tool_shed_repository.py
-scripts/api/requests_update_state.py
-scripts/api/request_type_create_from_xml.py
-scripts/api/reset_metadata_on_installed_repositories.py
-scripts/api/sample_dataset_update_status.py
-scripts/api/sample_update_state.py
-scripts/api/sequencer_configuration_create.py
-scripts/api/update.py
-scripts/api/upload_to_history.py
-scripts/api/workflow_delete.py
-scripts/api/workflow_execute_parameters.py
-scripts/api/workflow_execute.py
-scripts/api/workflow_import_from_file_rpark.py
-scripts/api/workflow_import.py
-scripts/auth/
-scripts/bootstrap_history.py
-scripts/build_toolbox.py
-scripts/build_universe_config.py
-scripts/check_galaxy.py
-scripts/check_python.py
-scripts/cleanup_datasets/
-scripts/communication/
-scripts/create_db.py
-scripts/data_libraries/
-scripts/db_shell.py
-scripts/drmaa_external_killer.py
-scripts/drmaa_external_runner.py
-scripts/external_chown_script.py
-scripts/extract_dataset_part.py
-scripts/extract_toolbox_sections.py
-scripts/functional_tests.py
-scripts/get_platforms.py
-scripts/helper.py
-scripts/__init__.py
-scripts/loc_files/
-scripts/manage_db.py
-scripts/manage_tools.py
-scripts/metagenomics/
-scripts/microbes/create_bacteria_loc_file.py
-scripts/microbes/create_bacteria_table.py
-scripts/microbes/create_nib_seq_loc_file.py
-scripts/microbes/get_builds_lengths.py
-scripts/microbes/ncbi_to_ucsc.py
-scripts/microbes/util.py
-scripts/migrate_tools/
-scripts/nosetests.py
-scripts/others/
-scripts/runtime_stats.py
-scripts/secret_decoder_ring.py
-scripts/set_dataset_sizes.py
-scripts/set_user_disk_usage.py
-scripts/sync_reports_config.py
-scripts/tool_shed/api/add_repository_registry_entry.py
-scripts/tool_shed/api/common.py
-scripts/tool_shed/api/create_categories.py
-scripts/tool_shed/api/create_users.py
-scripts/tool_shed/api/import_capsule.py
-scripts/tool_shed/api/remove_repository_registry_entry.py
-scripts/tool_shed/api/reset_metadata_on_repositories.py
-scripts/tool_shed/api/tool_shed_repository_revision_update.py
-scripts/tool_shed/build_ts_whoosh_index.py
-scripts/tool_shed/check_download_urls.py
-scripts/tool_shed/check_filesystem_for_empty_tool_dependency_installation_paths.py
-scripts/tool_shed/clean_up_tool_dependency_directory.py
-scripts/tool_shed/migrate_tools_to_repositories.py
-scripts/tool_shed/show_tool_dependency_installation_dir_contents.py
-scripts/tools/re_escape_output.py
-test/api/__init__.py
-test/api/test_dataset_collections.py
-test/api/test_datasets.py
-test/api/test_framework.py
-test/api/test_history_contents_provenance.py
-test/api/test_jobs.py
-test/api/test_libraries.py
-test/api/test_page_revisions.py
-test/api/test_search.py
-test/api/test_tool_data.py
-test/api/test_tours.py
-test/api/test_workflow_extraction.py
-test/api/test_workflows_from_yaml.py
-test/base/api_asserts.py
-test/base/api.py
-test/base/api_util.py
-test/base/__init__.py
-test/base/instrument.py
-test/base/integration_util.py
-test/base/interactor.py
-test/base/nose_util.py
-test/base/test_logging.py
-test/base/tool_shed_util.py
-test/base/twilltestcase.py
-test/base/workflows_format_2/
-test/casperjs/
-test/functional/database_contexts.py
-test/functional/__init__.py
-test/functional/test_data_managers.py
-test/functional/tool-data/
-test/functional/tools/
-test/integration/
-test/manual/__init__.py
-test/shed_functional/base/common.py
-test/shed_functional/base/__init__.py
-test/shed_functional/functional/
-test/shed_functional/functional_tests.py
-test/shed_functional/__init__.py
-test/unit/dataset_collections/
-test/unit/datatypes/dataproviders/__init__.py
-test/unit/datatypes/dataproviders/test_line_dataproviders.py
-test/unit/datatypes/__init__.py
-test/unit/datatypes/test_data.py
-test/unit/jobs/dynamic_tool_destination/
-test/unit/jobs/__init__.py
-test/unit/jobs/test_command_factory.py
-test/unit/jobs/test_datasets.py
-test/unit/jobs/test_job_configuration.py
-test/unit/jobs/test_job_output_checker.py
-test/unit/jobs/test_mapper.py
-test/unit/jobs/test_rule_helper.py
-test/unit/jobs/test_rules/
-test/unit/jobs/test_runner_params.py
-test/unit/managers/__init__.py
-test/unit/managers/test_CollectionManager.py
-test/unit/managers/test_DatasetManager.py
-test/unit/managers/test_HDAManager.py
-test/unit/managers/test_HDCAManager.py
-test/unit/managers/test_HistoryContentsManager.py
-test/unit/managers/test_UserManager.py
-test/unit/shed_unit/__init__.py
-test/unit/shed_unit/test_fabric_util.py
-test/unit/shed_unit/test_td_common_util.py
-test/unit/test_galaxy_transactions.py
-test/unit/test_lazy_process.py
-test/unit/test_objectstore.py
-test/unit/test_routes.py
-test/unit/test_security_helper.py
-test/unit/test_sockets.py
-test/unit/test_sqlite_utils.py
-test/unit/test_topsort.py
-test/unit/tools/filter_modules/
-test/unit/tools/__init__.py
-test/unit/tools/test_citations.py
-test/unit/tools/test_conda_resolution.py
-test/unit/tools/test_parsing.py
-test/unit/tools/test_select_parameters.py
-test/unit/tools/test_toolbox_filters.py
-test/unit/tools/test_tool_dependency_description.py
-test/unit/tools/test_tool_deps.py
-test/unit/tools/test_tool_external_files.py
-test/unit/tools/test_tool_loader.py
-test/unit/tools/test_tool_panel.py
-test/unit/tools/test_watcher.py
-test/unit/tools/test_wrappers.py
-test/unit/tools_support.py
-test/unit/unittest_utils/
-test/unit/visualizations/__init__.py
-test/unit/visualizations/plugins/__init__.py
-test/unit/web/base/__init__.py
-test/unit/web/framework/__init__.py
-test/unit/web/__init__.py
-test/unit/workflows/
+scripts/
+test/
 tool_list.py
 tools/
@@ -1,54 +1,31 @@
-If you use or extend Galaxy in your published work, please cite each of the
-following publications:
-
- Goecks, J, Nekrutenko, A, Taylor, J and The Galaxy Team. "Galaxy: a
-  comprehensive approach for supporting accessible, reproducible, and
-  transparent computational research in the life sciences." 
-  Genome Biol. 2010 Aug 25;11(8):R86.
-
- Blankenberg D, Von Kuster G, Coraor N, Ananda G, Lazarus R, Mangan M,
-  Nekrutenko A, Taylor J. "Galaxy: a web-based genome analysis tool for
-  experimentalists". Current Protocols in Molecular Biology. 
-  2010 Jan; Chapter 19:Unit 19.10.1-21.
-
- Giardine B, Riemer C, Hardison RC, Burhans R, Elnitski L, Shah P, Zhang Y,
-  Blankenberg D, Albert I, Taylor J, Miller W, Kent WJ, Nekrutenko A. "Galaxy:
-  a platform for interactive large-scale genome analysis." 
-  Genome Research. 2005 Oct; 15(10):1451-5.
-
-See also: https://galaxyproject.org/citing-galaxy
-
+If you use or extend Galaxy in your published work, please cite this
+publication:

+- Afgan E, Baker D, van den Beek M, Blankenberg D, Bouvier D, Čech M, Chilton
+  J, Clements D, Coraor N, Eberhard C, Grüning B, Guerler A, Hillman-Jackson
+  J, Von Kuster G, Rasche E, Soranzo N, Turaga N, Taylor J, Nekrutenko A, Goecks
+  J. "The Galaxy platform for accessible, reproducible and collaborative
+  biomedical analyses: 2016 update"
+  Nucleic Acids Research (2016) 44(W1):W3-W10 doi:10.1093/nar/gkw343

 BibTeX format:

-@article{goecks2010galaxy,
-  title={Galaxy: a comprehensive approach for supporting accessible, reproducible, and transparent computational research in the life sciences},
-  author={Goecks, Jeremy and Nekrutenko, Anton and Taylor, James and The Galaxy Team},
-  journal={Genome Biol},
-  volume={11},
-  number={8},
-  pages={R86},
-  year={2010}
+@article{afgan2016galaxy,
+  title={The {Galaxy} platform for accessible, reproducible and collaborative biomedical analyses: 2016 update},
+  author={Afgan, Enis and Baker, Dannon and van den Beek, Marius and Blankenberg, Daniel and Bouvier, Dave and \v{C}ech, Martin and Chilton, John and Clements, Dave and Coraor, Nate and Eberhard, Carl and Gr\"uning, Bj\"orn and Guerler, Aysam and Hillman-Jackson, Jennifer and Von Kuster, Greg and Rasche, Eric and Soranzo, Nicola and Turaga, Nitesh and Taylor, James and Nekrutenko, Anton and Goecks, Jeremy},
+  journal={Nucleic Acids Res.},
+  volume={44},
+  number={W1},
+  pages={W3-W10},
+  year={2016},
+  publisher={Oxford University Press}
 }

-@article{blankenberg2010galaxy,
-  title={Galaxy: A Web-Based Genome Analysis Tool for Experimentalists},
-  author={Blankenberg, Daniel and Kuster, Gregory Von and Coraor, Nathaniel and Ananda, Guruprasad and Lazarus, Ross and Mangan, Mary and Nekrutenko, Anton and Taylor, James},
-  journal={Current protocols in molecular biology},
-  pages={19--10},
-  year={2010},
-  publisher={John Wiley \& Sons, Inc.}
-}

-@article{giardine2005galaxy,
-  title={Galaxy: a platform for interactive large-scale genome analysis},
-  author={Giardine, Belinda and Riemer, Cathy and Hardison, Ross C and Burhans, Richard and Elnitski, Laura and Shah, Prachi and Zhang, Yi and Blankenberg, Daniel and Albert, Istvan and Taylor, James and Miller, Webb C and Kent, W James and Nekrutenko, Anton},
-  journal={Genome research},
-  volume={15},
-  number={10},
-  pages={1451--1455},
-  year={2005},
-  publisher={Cold Spring Harbor Lab}
-}
+More publications
+-----------------

+To cite specific Galaxy components/features or other Galaxy project
+publications, see
+
+https://galaxyproject.org/citing-galaxy
@@ -17,10 +17,10 @@ import fileinput
 import os
 import sys

+import requests
 from six.moves.urllib.parse import urlencode
-from six.moves.urllib.request import urlopen

-import parse_builds
+import parse_builds  # noqa: I100


 def getchrominfo(url, db):
@@ -36,8 +36,8 @@ def getchrominfo(url, db):
        "hgta_regionType": "",
        "position": "",
        "hgta_doTopSubmit": "get info"})
-    page = urlopen(URL)
-    for line in page:
+    page = requests.get(URL).text
+    for line in page.split('\n'):
        line = line.rstrip("\r\n")
        if line.startswith("#"):
            continue
@@ -9,18 +9,17 @@ from __future__ import print_function
 import sys
 import xml.etree.ElementTree as ElementTree

-from six.moves.urllib.request import urlopen
+import requests


 def getbuilds(url):
    try:
-        page = urlopen(url)
+        text = requests.get(url).text
    except:
        print("#Unable to open " + url)
        print("?\tunspecified (?)")
        sys.exit(1)

-    text = page.read()
    try:
        tree = ElementTree.fromstring(text)
    except:
@@ -6,7 +6,7 @@ from __future__ import print_function

 import xml.etree.ElementTree as ElementTree

-from six.moves.urllib.request import urlopen
+import requests

 sites = ['http://genome.ucsc.edu/cgi-bin/',
         'http://archaea.ucsc.edu/cgi-bin/',
@@ -20,11 +20,11 @@ def main():
        trackurl = sites[i] + "hgTracks?"
        builds = []
        try:
-            page = urlopen(site)
+            text = requests.get(site).text
        except:
            print("#Unable to connect to " + site)
            continue
-        text = page.read()
+
        try:
            tree = ElementTree.fromstring(text)
        except:
@@ -306,7 +306,7 @@ def parse_containers_config(containers_config_file):
    conf = DEFAULT_CONF.copy()
    try:
        with open(containers_config_file) as fh:
-            c = yaml.load(fh)
+            c = yaml.safe_load(fh)
            conf.update(c.get('containers', {}))
    except (OSError, IOError) as exc:
        if exc.errno == errno.ENOENT:
@@ -252,12 +252,8 @@ class Bam(Binary):
            message = 'Attempting to use functionality requiring samtools, but it cannot be located on Galaxy\'s PATH.'
            raise Exception(message)

-        # Get the version of samtools via --version-only, if available
-        p = subprocess.Popen(['samtools', '--version-only'],
-                             stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE)
+        p = subprocess.Popen(['samtools', '--version-only'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        output, error = p.communicate()
-
        # --version-only is available
        # Format is <version x.y.z>+htslib-<a.b.c>
        if p.returncode == 0:
@@ -294,10 +290,8 @@ class Bam(Binary):

    def _is_coordinate_sorted(self, file_name):
        """See if the input BAM file is sorted from the header information."""
-        params = ["samtools", "view", "-H", file_name]
-        output = subprocess.Popen(params, stderr=subprocess.PIPE, stdout=subprocess.PIPE).communicate()[0]
-        # find returns -1 if string is not found
-        return output.find("SO:coordinate") != -1 or output.find("SO:sorted") != -1
+        output = subprocess.check_output(["samtools", "view", "-H", file_name])
+        return 'SO:coordinate' in output or 'SO:sorted' in output

    def dataset_content_needs_grooming(self, file_name):
        """See if file_name is a sorted BAM file"""
@@ -322,8 +316,7 @@ class Bam(Binary):
                return False
            index_name = tempfile.NamedTemporaryFile(prefix="bam_index").name
            stderr_name = tempfile.NamedTemporaryFile(prefix="bam_index_stderr").name
-            command = 'samtools index %s %s' % (file_name, index_name)
-            proc = subprocess.Popen(args=command, shell=True, stderr=open(stderr_name, 'wb'))
+            proc = subprocess.Popen(['samtools', 'index', file_name, index_name], stderr=open(stderr_name, 'wb'))
            proc.wait()
            stderr = open(stderr_name).read().strip()
            if stderr:
@@ -366,8 +359,8 @@ class Bam(Binary):
        tmp_sorted_dataset_file_name_prefix = os.path.join(tmp_dir, 'sorted')
        stderr_name = tempfile.NamedTemporaryFile(dir=tmp_dir, prefix="bam_sort_stderr").name
        samtools_created_sorted_file_name = "%s.bam" % tmp_sorted_dataset_file_name_prefix  # samtools accepts a prefix, not a filename, it always adds .bam to the prefix
-        command = "samtools sort %s %s" % (file_name, tmp_sorted_dataset_file_name_prefix)
-        proc = subprocess.Popen(args=command, shell=True, cwd=tmp_dir, stderr=open(stderr_name, 'wb'))
+        proc = subprocess.Popen(['samtools', 'sort', file_name, tmp_sorted_dataset_file_name_prefix],
+                                cwd=tmp_dir, stderr=open(stderr_name, 'wb'))
        exit_code = proc.wait()
        # Did sort succeed?
        stderr = open(stderr_name).read().strip()
@@ -1309,11 +1302,8 @@ class ExcelXls(Binary):
    edam_format = "format_3468"

    def sniff(self, filename):
-        mime_type = subprocess.check_output("file --mime-type '{}'".format(filename), shell=True).rstrip()
-        if mime_type.find("application/vnd.ms-excel") != -1:
-            return True
-        else:
-            return False
+        mime_type = subprocess.check_output(['file', '--mime-type', filename])
+        return "application/vnd.ms-excel" in mime_type

    def get_mime(self):
        """Returns the mime type of the datatype"""
@@ -133,8 +133,9 @@ if __name__ == "__main__":
    # Sort through a tempfile first
    temp_file = tempfile.NamedTemporaryFile(mode="r")
    environ['LC_ALL'] = 'POSIX'
-    commandline = "sort -f -n -k %d -k %d -k %d -o %s %s" % (chr_col_1 + 1, start_col_1 + 1, end_col_1 + 1, temp_file.name, in_fname)
-    subprocess.check_call(commandline, shell=True)
+    subprocess.check_call([
+        'sort', '-f', '-n', '-k', chr_col_1 + 1, '-k', start_col_1 + 1, '-k', end_col_1 + 1, '-o', temp_file.name, in_fname
+    ])

    coverage = CoverageWriter(out_stream=open(out_fname, "a"),
                              chromCol=chr_col_2, positionCol=position_col_2,
@@ -72,9 +72,9 @@ def rgConv(inpedfilepath, outhtmlname, outfilepath, plink):
    if not missval:
        print('### lped_to_pbed_converter.py cannot identify missing value in %s' % pedf)
        missval = '0'
-    cl = '%s --noweb --file %s --make-bed --out %s --missing-genotype %s' % (plink, inpedfilepath, outroot, missval)
-    p = subprocess.Popen(cl, shell=True, cwd=outfilepath)
-    p.wait()  # run plink
+    subprocess.check_call([plink, '--noweb', '--file', inpedfilepath,
+                           '--make-bed', '--out', outroot,
+                           '--missing-genotype', missval], cwd=outfilepath)


 def main():
@@ -41,8 +41,7 @@ def pruneLD(plinktasks=[], cd='./', vclbase=[]):
    for task in plinktasks:  # each is a list
        vcl = vclbase + task
        with open(plog, 'w') as sto:
-            x = subprocess.Popen(' '.join(vcl), shell=True, stdout=sto, stderr=sto, cwd=cd)
-            x.wait()
+            subprocess.check_call(vcl, stdout=sto, stderr=sto, cwd=cd)
        try:
            lplog = open(plog, 'r').readlines()
            lplog = [elem for elem in lplog if elem.find('Pruning SNP') == -1]
@@ -40,9 +40,7 @@ def rgConv(inpedfilepath, outhtmlname, outfilepath, plink):
    """
    basename = os.path.split(inpedfilepath)[-1]  # get basename
    outroot = os.path.join(outfilepath, basename)
-    cl = '%s --noweb --bfile %s --recode --out %s ' % (plink, inpedfilepath, outroot)
-    p = subprocess.Popen(cl, shell=True, cwd=outfilepath)
-    p.wait()  # run plink
+    subprocess.check_call([plink, '--noweb', '--bfile', inpedfilepath, '--recode', '--out', outroot], cwd=outfilepath)


 def main():
@@ -34,11 +34,8 @@ def _get_samtools_version():
    if not cmd_exists('samtools'):
        raise Exception('This tool needs samtools, but it is not on PATH.')
    # Get the version of samtools via --version-only, if available
-    p = subprocess.Popen(['samtools', '--version-only'],
-                         stdout=subprocess.PIPE,
-                         stderr=subprocess.PIPE)
+    p = subprocess.Popen(['samtools', '--version-only'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    output, error = p.communicate()
-
    # --version-only is available
    # Format is <version x.y.z>+htslib-<a.b.c>
    if p.returncode == 0:
@@ -359,7 +359,7 @@ class Registry(object):
            build_sites_config_file = getattr(self.config, "build_sites_config_file", None)
            if build_sites_config_file and os.path.exists(build_sites_config_file):
                with open(build_sites_config_file, "r") as f:
-                    build_sites_config = yaml.load(f)
+                    build_sites_config = yaml.safe_load(f)
                if not isinstance(build_sites_config, list):
                    self.log.exception("Build sites configuration YAML file does not declare list of sites.")
                    return
@@ -7,6 +7,7 @@ import logging
 import os
 import re
 import string
+import subprocess
 import sys
 from cgi import escape
 from itertools import islice
@@ -693,8 +694,7 @@ class BaseFastq (Sequence):
        else:
            commands = Sequence.get_split_commands_sequential(is_gzip(input_name), input_name, output_name, start_sequence, sequence_count)
        for cmd in commands:
-            if 0 != os.system(cmd):
-                raise Exception("Executing '%s' failed" % cmd)
+            subprocess.check_call(cmd, shell=True)
        return True
    process_split_file = staticmethod(process_split_file)

@@ -521,15 +521,12 @@ class Sam(Tabular):
        Multiple SAM files may each have headers. Since the headers should all be the same, remove
        the headers from files 1-n, keeping them in the first file only
        """
-        cmd = 'mv %s %s' % (split_files[0], output_file)
-        result = os.system(cmd)
-        if result != 0:
-            raise Exception('Result %s from %s' % (result, cmd))
+        shutil.move(split_files[0], output_file)
+
        if len(split_files) > 1:
-            cmd = 'egrep -v -h "^@" %s >> %s' % (' '.join(split_files[1:]), output_file)
-        result = os.system(cmd)
-        if result != 0:
-            raise Exception('Result %s from %s' % (result, cmd))
+            cmd = ['egrep', '-v', '-h', '^@'] + split_files[1:] + ['>>', output_file]
+            subprocess.check_call(cmd, shell=True)
+
    merge = staticmethod(merge)

    # Dataproviders
@@ -10,6 +10,8 @@ import re
 import subprocess
 import tempfile

+from six.moves import shlex_quote
+
 from galaxy.datatypes.data import get_file_peek, Text
 from galaxy.datatypes.metadata import MetadataElement, MetadataParameter
 from galaxy.datatypes.sniff import iter_headers
@@ -148,13 +150,12 @@ class Ipynb(Json):
            ofilename = ofile_handle.name
            ofile_handle.close()
            try:
-                cmd = 'jupyter nbconvert --to html --template full %s --output %s' % (dataset.file_name, ofilename)
-                log.info("Calling command %s" % cmd)
-                subprocess.call(cmd, shell=True)
+                cmd = ['jupyter', 'nbconvert', '--to', 'html', '--template', 'full', dataset.file_name, '--output', ofilename]
+                subprocess.check_call(cmd)
                ofilename = '%s.html' % ofilename
-            except:
+            except subprocess.CalledProcessError:
                ofilename = dataset.file_name
-                log.exception('Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.', cmd)
+                log.exception('Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.', ' '.join(map(shlex_quote, cmd)))
            return open(ofilename)

    def set_meta(self, dataset, **kwd):
@@ -57,7 +57,7 @@ anyjson==0.3.3

 # Pulsar requirements
 psutil==4.1.0
-pulsar-galaxy-lib==0.7.0.dev5
+pulsar-galaxy-lib==0.8.0

 # sqlalchemy-migrate and dependencies
 sqlalchemy-migrate==0.10.0
@@ -1,6 +1,6 @@
 # Contains actions that are used in External Services
 import logging
-from urllib import urlopen
+import requests
 from galaxy.web import url_for
 from galaxy.util.template import fill_template
 from result_handlers.basic import ExternalServiceActionResultHandler
@@ -104,7 +104,7 @@ class ExternalServiceWebAPIActionResult(ExternalServiceResult):
    @property
    def content(self):
        if self._content is None:
-            self._content = urlopen(self.url).read()
+            self._content = requests.get(self.url).text
        return self._content


@@ -884,6 +884,9 @@ class JobWrapper(object, HasResourceParameters):
        self.galaxy_lib_dir
        # Shell fragment to inject dependencies
        self.dependency_shell_commands = self.tool.build_dependency_shell_commands(job_directory=self.working_directory)
+        if self.tool.requires_galaxy_python_environment:
+            # These tools (upload, metadata, data_source) may need access to the datatypes registry.
+            self.app.datatypes_registry.to_xml_file(os.path.join(self.working_directory, 'registry.xml'))
        # We need command_line persisted to the db in order for Galaxy to re-queue the job
        # if the server was stopped and restarted before the job finished
        job.command_line = unicodify(self.command_line)
@@ -1568,6 +1571,9 @@ class JobWrapper(object, HasResourceParameters):
                    paths.append(DatasetPath(da.id, real_path=real_path, false_path=false_path, mutable=False))
        return paths

+    def get_output_basenames(self):
+        return map(os.path.basename, map(str, self.get_output_fnames()))
+
    def get_output_fnames(self):
        if self.output_paths is None:
            self.compute_outputs()
@@ -1645,7 +1651,8 @@ class JobWrapper(object, HasResourceParameters):

    def setup_external_metadata(self, exec_dir=None, tmp_dir=None,
                                dataset_files_path=None, config_root=None,
-                                config_file=None, resolve_metadata_dependencies=False,
+                                config_file=None, datatypes_config=None,
+                                resolve_metadata_dependencies=False,
                                set_extension=True, **kwds):
        # extension could still be 'auto' if this is the upload tool.
        job = self.get_job()
@@ -1664,8 +1671,9 @@ class JobWrapper(object, HasResourceParameters):
            config_root = self.app.config.root
        if config_file is None:
            config_file = self.app.config.config_file
-        datatypes_config = os.path.join(self.working_directory, 'registry.xml')
-        self.app.datatypes_registry.to_xml_file(path=datatypes_config)
+        if datatypes_config is None:
+            datatypes_config = os.path.join(self.working_directory, 'registry.xml')
+            self.app.datatypes_registry.to_xml_file(path=datatypes_config)
        command = self.external_output_metadata.setup_external_metadata([output_dataset_assoc.dataset for
                                                                         output_dataset_assoc in
                                                                         job.output_datasets + job.output_library_datasets],
@@ -1974,7 +1982,7 @@ class TaskWrapper(JobWrapper):
        pass

    def setup_external_metadata(self, exec_dir=None, tmp_dir=None, dataset_files_path=None,
-                                config_root=None, config_file=None,
+                                config_root=None, config_file=None, datatypes_config=None,
                                set_extension=True, **kwds):
        # There is no metadata setting for tasks.  This is handled after the merge, at the job level.
        return ""
@@ -1994,6 +2002,10 @@ class ComputeEnvironment(object):
    compute server.
    """

+    @abstractmethod
+    def output_names(self):
+        """ Output unqualified filenames defined by job. """
+
    @abstractmethod
    def output_paths(self):
        """ Output DatasetPaths defined by job. """
@@ -2059,6 +2071,9 @@ class SharedComputeEnvironment(SimpleComputeEnvironment):
        self.job_wrapper = job_wrapper
        self.job = job

+    def output_names(self):
+        return self.job_wrapper.get_output_basenames()
+
    def output_paths(self):
        return self.job_wrapper.get_output_fnames()

@@ -308,7 +308,6 @@ class DeleteIntermediatesAction(DefaultJobAction):
                    safe_to_delete = True
                    for job_to_check in [d_j.job for d_j in input_dataset.dependent_jobs]:
                        if job_to_check != job and job_to_check.state not in [job.states.OK, job.states.DELETED]:
-                            log.debug("Workflow Intermediates cleanup attempted, but non-terminal state '%s' detected for job %s" % (job_to_check.state, job_to_check.id))
                            safe_to_delete = False
                    if safe_to_delete:
                        # Support purging here too.
@@ -17,19 +17,7 @@ log = getLogger(__name__)
 CAPTURE_RETURN_CODE = "return_code=$?"
 YIELD_CAPTURED_CODE = 'sh -c "exit $return_code"'
 SETUP_GALAXY_FOR_METADATA = """
-if [ "$GALAXY_LIB" != "None" ]; then
-    if [ -n "$PYTHONPATH" ]; then
-        PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"
-    else
-        PYTHONPATH="$GALAXY_LIB"
-    fi
-    export PYTHONPATH
-fi
-if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" \
-     -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then
-    . "$GALAXY_VIRTUAL_ENV/bin/activate"
-fi
-GALAXY_PYTHON=`command -v python`
+[ "$GALAXY_VIRTUAL_ENV" = "None" ] && GALAXY_VIRTUAL_ENV="$_GALAXY_VIRTUAL_ENV"; _galaxy_setup_environment True
 """


@@ -198,6 +186,7 @@ def __handle_metadata(commands_builder, job_wrapper, runner, remote_command_para
    output_fnames = metadata_kwds.get('output_fnames', job_wrapper.get_output_fnames())
    config_root = metadata_kwds.get('config_root', None)
    config_file = metadata_kwds.get('config_file', None)
+    datatypes_config = metadata_kwds.get('datatypes_config', None)
    compute_tmp_dir = metadata_kwds.get('compute_tmp_dir', None)
    resolve_metadata_dependencies = job_wrapper.commands_in_new_shell
    metadata_command = job_wrapper.setup_external_metadata(
@@ -208,6 +197,7 @@ def __handle_metadata(commands_builder, job_wrapper, runner, remote_command_para
        set_extension=False,
        config_root=config_root,
        config_file=config_file,
+        datatypes_config=datatypes_config,
        compute_tmp_dir=compute_tmp_dir,
        resolve_metadata_dependencies=resolve_metadata_dependencies,
        kwds={'overwrite': False}
@@ -2,11 +2,10 @@
 Module for managing jobs in Pacific Bioscience's SMRT Portal and automatically transferring files
 produced by SMRT Portal.
 """
-import json
 import logging
 from string import Template

-from six.moves.urllib.request import urlopen
+import requests

 from .data_transfer import DataTransfer

@@ -88,8 +87,8 @@ class SMRTPortalPlugin(DataTransfer):
            if self._missing_params(job.params, ['smrt_host', 'smrt_job_id']):
                return self.job_states.INVALID
            url = 'http://' + job.params['smrt_host'] + self.api_path + '/Jobs/' + job.params['smrt_job_id'] + '/Status'
-            r = urlopen(url)
-            status = json.loads(r.read())
+            r = requests.get(url)
+            status = r.json()
            # TODO: error handling: unexpected json or bad response, bad url, etc.
            if status['Code'] == 'Completed':
                log.debug("SMRT Portal job '%s' is Completed.  Initiating transfer." % job.params['smrt_job_id'])
@@ -7,6 +7,7 @@ from __future__ import absolute_import  # Need to import pulsar_client absolutel
 import errno
 import logging
 import os
+import subprocess
 from distutils.version import LooseVersion
 from time import sleep

@@ -51,7 +52,10 @@ __all__ = (
    'PulsarEmbeddedJobRunner',
 )

-MINIMUM_PULSAR_VERSION = LooseVersion("0.7.0.dev3")
+MINIMUM_PULSAR_VERSIONS = {
+    '_default_': LooseVersion("0.7.0.dev3"),
+    'remote_metadata': LooseVersion("0.8.0"),
+}

 NO_REMOTE_GALAXY_FOR_METADATA_MESSAGE = "Pulsar misconfiguration - Pulsar client configured to set metadata remotely, but remote Pulsar isn't properly configured with a galaxy_home directory."
 NO_REMOTE_DATATYPES_CONFIG = "Pulsar client is configured to use remote datatypes configuration when setting metadata externally, but Pulsar is not configured with this information. Defaulting to datatypes_conf.xml."
@@ -59,7 +63,7 @@ GENERIC_REMOTE_ERROR = "Failed to communicate with remote job server."
 FAILED_REMOTE_ERROR = "Remote job server indicated a problem running or monitoring this job."
 LOST_REMOTE_ERROR = "Remote job server could not determine this job's state."

-UPGRADE_PULSAR_ERROR = "Galaxy is misconfigured, please contact administrator. The target Pulsar server is unsupported, this version of Galaxy requires Pulsar version %s or newer." % MINIMUM_PULSAR_VERSION
+UPGRADE_PULSAR_ERROR = "Galaxy is misconfigured, please contact administrator. The target Pulsar server is unsupported, this version of Galaxy requires Pulsar version %s or newer."

 # Is there a good way to infer some default for this? Can only use
 # url_for from web threads. https://gist.github.com/jmchilton/9098762
@@ -217,7 +221,7 @@ class PulsarJobRunner(AsynchronousJobRunner):
        else:
            log.info("Loading Pulsar app configuration from %s" % pulsar_conf_path)
            with open(pulsar_conf_path, "r") as f:
-                conf.update(yaml.load(f) or {})
+                conf.update(yaml.safe_load(f) or {})
        if "job_metrics_config_file" not in conf:
            conf["job_metrics"] = self.app.job_metrics
        if "staging_directory" not in conf:
@@ -277,8 +281,10 @@ class PulsarJobRunner(AsynchronousJobRunner):
            dependencies_description = PulsarJobRunner.__dependencies_description(client, job_wrapper)
            rewrite_paths = not PulsarJobRunner.__rewrite_parameters(client)
            unstructured_path_rewrites = {}
+            output_names = []
            if compute_environment:
                unstructured_path_rewrites = compute_environment.unstructured_path_rewrites
+                output_names = compute_environment.output_names()

            client_job_description = ClientJobDescription(
                command_line=command_line,
@@ -292,6 +298,7 @@ class PulsarJobRunner(AsynchronousJobRunner):
                env=client.env,
                rewrite_paths=rewrite_paths,
                arbitrary_files=unstructured_path_rewrites,
+                touch_outputs=output_names,
            )
            job_id = pulsar_submit_job(client, client_job_description, remote_job_config)
            log.info("Pulsar job submitted with job_id %s" % job_id)
@@ -310,6 +317,11 @@ class PulsarJobRunner(AsynchronousJobRunner):
        pulsar_job_state.job_destination = job_destination
        self.monitor_job(pulsar_job_state)

+    def __needed_features(self, client):
+            return {
+                'remote_metadata': PulsarJobRunner.__remote_metadata(client),
+            }
+
    def __prepare_job(self, job_wrapper, job_destination):
        """Build command-line and Pulsar client for this job."""
        command_line = None
@@ -319,8 +331,9 @@ class PulsarJobRunner(AsynchronousJobRunner):
        try:
            client = self.get_client_from_wrapper(job_wrapper)
            tool = job_wrapper.tool
-            remote_job_config = client.setup(tool.id, tool.version)
-            PulsarJobRunner.check_job_config(remote_job_config)
+            remote_job_config = client.setup(tool.id, tool.version, tool.requires_galaxy_python_environment)
+            needed_features = self.__needed_features(client)
+            PulsarJobRunner.check_job_config(remote_job_config, check_features=needed_features)
            rewrite_parameters = PulsarJobRunner.__rewrite_parameters(client)
            prepare_kwds = {}
            if rewrite_parameters:
@@ -382,8 +395,7 @@ class PulsarJobRunner(AsynchronousJobRunner):
        prepare_input_files_cmds = getattr(job_wrapper, 'prepare_input_files_cmds', None)
        if prepare_input_files_cmds is not None:
            for cmd in prepare_input_files_cmds:  # run the commands to stage the input files
-                if 0 != os.system(cmd):
-                    raise Exception('Error running file staging command: %s' % cmd)
+                subprocess.check_call(cmd, shell=True)
            job_wrapper.prepare_input_files_cmds = None  # prevent them from being used in-line

    def _populate_parameter_defaults(self, job_destination):
@@ -593,12 +605,17 @@ class PulsarJobRunner(AsynchronousJobRunner):
        return client_outputs

    @staticmethod
-    def check_job_config(remote_job_config):
+    def check_job_config(remote_job_config, check_features=None):
+        check_features = check_features or {}
        # 0.6.0 was newest Pulsar version that did not report it's version.
        pulsar_version = LooseVersion(remote_job_config.get('pulsar_version', "0.6.0"))
+        needed_version = LooseVersion("0.0.0")
        log.info("pulsar_version is %s" % pulsar_version)
-        if pulsar_version < MINIMUM_PULSAR_VERSION:
-            raise UnsupportedPulsarException()
+        for feature in list(check_features.keys()) + ['_default_']:
+            if pulsar_version < MINIMUM_PULSAR_VERSIONS[feature]:
+                needed_version = max(needed_version, MINIMUM_PULSAR_VERSIONS[feature])
+        if pulsar_version < needed_version:
+            raise UnsupportedPulsarException(needed_version)

    @staticmethod
    def __dependencies_description(pulsar_client, job_wrapper):
@@ -656,7 +673,6 @@ class PulsarJobRunner(AsynchronousJobRunner):
                raise Exception(NO_REMOTE_GALAXY_FOR_METADATA_MESSAGE)
            metadata_kwds['exec_dir'] = remote_galaxy_home
            outputs_directory = remote_job_config['outputs_directory']
-            configs_directory = remote_job_config['configs_directory']
            working_directory = remote_job_config['working_directory']
            metadata_directory = remote_job_config['metadata_directory']
            # For metadata calculation, we need to build a list of of output
@@ -687,7 +703,7 @@ class PulsarJobRunner(AsynchronousJobRunner):
                    remote_datatypes_config = os.path.join(remote_galaxy_home, 'datatypes_conf.xml')
                metadata_kwds['datatypes_config'] = remote_datatypes_config
            else:
-                datatypes_config = os.path.join(configs_directory, 'registry.xml')
+                datatypes_config = os.path.join(job_wrapper.working_directory, 'registry.xml')
                self.app.datatypes_registry.to_xml_file(path=datatypes_config)
                # Ensure this file gets pushed out to the remote config dir.
                job_wrapper.extra_filenames.append(datatypes_config)
@@ -783,6 +799,10 @@ class PulsarComputeEnvironment(ComputeEnvironment):
            version_path = new_version_path
        self._version_path = version_path

+    def output_names(self):
+        # Maybe this should use the path mapper, but the path mapper just uses basenames
+        return self.job_wrapper.get_output_basenames()
+
    def output_paths(self):
        local_output_paths = self._wrapper_output_paths

@@ -854,5 +874,5 @@ class PulsarComputeEnvironment(ComputeEnvironment):

 class UnsupportedPulsarException(Exception):

-    def __init__(self):
-        super(UnsupportedPulsarException, self).__init__(UPGRADE_PULSAR_ERROR)
+    def __init__(self, needed):
+        super(UnsupportedPulsarException, self).__init__(UPGRADE_PULSAR_ERROR % needed)
@@ -1,25 +1,33 @@
 #!$shell

 $headers
+
+_galaxy_setup_environment() {
+    local _use_framework_galaxy="$1"
+    if [ "$GALAXY_LIB" != "None" -a "$_use_framework_galaxy" = "True" ]; then
+        if [ -n "$PYTHONPATH" ]; then
+            PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"
+        else
+            PYTHONPATH="$GALAXY_LIB"
+        fi
+        export PYTHONPATH
+    fi
+    $env_setup_commands
+    if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" \
+         -a "`command -v python`" != "$GALAXY_VIRTUAL_ENV/bin/python" ]; then
+        . "$GALAXY_VIRTUAL_ENV/bin/activate"
+    fi
+}
+
 $integrity_injection
 $slots_statement
 export GALAXY_SLOTS
+GALAXY_VIRTUAL_ENV="$galaxy_virtual_env"
+_GALAXY_VIRTUAL_ENV="$galaxy_virtual_env"
 PRESERVE_GALAXY_ENVIRONMENT="$preserve_python_environment"
 GALAXY_LIB="$galaxy_lib"
-if [ "$GALAXY_LIB" != "None" -a "$PRESERVE_GALAXY_ENVIRONMENT" = "True" ]; then
-    if [ -n "$PYTHONPATH" ]; then
-        PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"
-    else
-        PYTHONPATH="$GALAXY_LIB"
-    fi
-    export PYTHONPATH
-fi
-$env_setup_commands
-GALAXY_VIRTUAL_ENV="$galaxy_virtual_env"
-if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" \
-     -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" -a "$PRESERVE_GALAXY_ENVIRONMENT" = "True" ]; then
-    . "$GALAXY_VIRTUAL_ENV/bin/activate"
-fi
+_galaxy_setup_environment "$PRESERVE_GALAXY_ENVIRONMENT"
+GALAXY_PYTHON=`command -v python`
 $instrument_pre_commands
 cd $working_directory
 $command
@@ -119,9 +119,8 @@ def _handle_script_integrity(path, config):
    sleep_amt = getattr(config, "check_job_script_integrity_sleep", DEFAULT_INTEGRITY_SLEEP)
    for i in range(count):
        try:
-            proc = subprocess.Popen([path], shell=True, env={"ABC_TEST_JOB_SCRIPT_INTEGRITY_XYZ": "1"})
-            proc.wait()
-            if proc.returncode == 42:
+            returncode = subprocess.call([path], env={"ABC_TEST_JOB_SCRIPT_INTEGRITY_XYZ": "1"})
+            if returncode == 42:
                script_integrity_verified = True
                break

@@ -1,6 +1,6 @@
 import os
+import subprocess
 from platform import system
-from subprocess import Popen
 from time import sleep

 try:
@@ -41,8 +41,8 @@ def _stock_kill_pid(pid):

 def __kill_windows(pid):
    try:
-        Popen("taskkill /F /T /PID %i" % pid, shell=True)
-    except Exception:
+        subprocess.check_call(['taskkill', '/F', '/T', '/PID', pid])
+    except subprocess.CalledProcessError:
        pass


@@ -9,6 +9,8 @@ import socket
 import subprocess
 import threading

+from six.moves import shlex_quote
+
 from galaxy.util import listify, sleeper
 from galaxy.util.json import jsonrpc_request, validate_jsonrpc_response

@@ -23,7 +25,7 @@ class TransferManager(object):
    def __init__(self, app):
        self.app = app
        self.sa_session = app.model.context.current
-        self.command = 'python %s' % os.path.abspath(os.path.join(os.getcwd(), 'scripts', 'transfer.py'))
+        self.command = ['python', os.path.abspath(os.path.join(os.getcwd(), 'scripts', 'transfer.py'))]
        if app.config.get_bool('enable_job_recovery', True):
            # Only one Galaxy server process should be able to recover jobs! (otherwise you'll have nasty race conditions)
            self.running = True
@@ -68,9 +70,9 @@ class TransferManager(object):
            # The transfer script should daemonize fairly quickly - if this is
            # not the case, this process will need to be moved to a
            # non-blocking method.
-            cmd = '%s %s' % (self.command, tj.id)
-            log.debug('Transfer command is: %s' % cmd)
-            p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+            cmd = self.command + [tj.id]
+            log.debug('Transfer command is: %s', ' '.join(map(shlex_quote, cmd)))
+            p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            p.wait()
            output = p.stdout.read(32768)
            if p.returncode != 0:
@@ -1,6 +1,6 @@
 import functools
 import os
-import urllib2
+import requests

 from beaker.cache import CacheManager
 from beaker.util import parse_cache_config_options
@@ -47,10 +47,8 @@ class DoiCache(object):
    def _raw_get_bibtex(self, doi):
        dx_url = "http://dx.doi.org/" + doi
        headers = {'Accept': 'text/bibliography; style=bibtex, application/x-bibtex'}
-        req = urllib2.Request(dx_url, data="", headers=headers)
-        response = urllib2.urlopen(req)
-        bibtex = response.read()
-        return bibtex
+        req = requests.get(dx_url, headers=headers)
+        return req.text

    def get_bibtex(self, doi):
        createfunc = functools.partial(self._raw_get_bibtex, doi)
@@ -836,7 +836,7 @@ class JobExternalOutputMetadataWrapper(object):
                sa_session.add(metadata_files)
                sa_session.flush()
            metadata_files_list.append(metadata_files)
-        args = '"%s" "%s" %s %s' % (datatypes_config,
+        args = '"%s" "%s" %s %s' % (metadata_path_on_compute(datatypes_config),
                                    job_metadata,
                                    " ".join(map(__metadata_files_list_to_cmd_line, metadata_files_list)),
                                    max_metadata_value_size)
@@ -18,6 +18,7 @@ from galaxy.util import (
    safe_relpath,
    string_as_bool,
    umask_fix_perms,
+    which,
 )
 from galaxy.util.sleeper import Sleeper

@@ -67,10 +68,9 @@ class S3ObjectStore(ObjectStore):
            self.cache_monitor_thread.start()
            log.info("Cache cleaner manager started")
        # Test if 'axel' is available for parallel download and pull the key into cache
-        try:
-            subprocess.call('axel')
+        if which('axel'):
            self.use_axel = True
-        except OSError:
+        else:
            self.use_axel = False

    def _configure_connection(self):
@@ -341,7 +341,7 @@ class S3ObjectStore(ObjectStore):
                log.debug("Parallel pulled key '%s' into cache to %s", rel_path, self._get_cache_path(rel_path))
                ncores = multiprocessing.cpu_count()
                url = key.generate_url(7200)
-                ret_code = subprocess.call("axel -a -n %s '%s'" % (ncores, url))
+                ret_code = subprocess.call(['axel', '-a', '-n', ncores, url])
                if ret_code == 0:
                    return True
            else:
@@ -16,7 +16,7 @@ import time
 from glob import glob
 from tempfile import NamedTemporaryFile

-from six.moves.urllib.request import urlopen
+import requests

 from galaxy import util
 from galaxy.util.dictifiable import Dictifiable
@@ -340,7 +340,7 @@ class TabularToolDataTable(ToolDataTable, Dictifiable):
                if filename:
                    tmp_file = NamedTemporaryFile(prefix='TTDT_URL_%s-' % self.name)
                    try:
-                        tmp_file.write(urlopen(filename, timeout=url_timeout).read())
+                        tmp_file.write(requests.get(filename, timeout=url_timeout).text)
                    except Exception as e:
                        log.error('Error loading Data Table URL "%s": %s', filename, e)
                        continue
@@ -61,7 +61,7 @@ def _render_jinja2(recipe_dir):

@_Memoized
 def yamlize(data):
-    res = yaml.load(data)
+    res = yaml.safe_load(data)
    # ensure the result is a dict
    if res is None:
        res = {}
@@ -3,6 +3,7 @@
 import collections
 import logging
 import os
+import subprocess

 import six

@@ -11,7 +12,6 @@ from ..container_resolvers import (
 )
 from ..docker_util import build_docker_images_command
 from ..mulled.mulled_build import (
-    check_output,
    DEFAULT_CHANNELS,
    ensure_installed,
    InvolucroContext,
@@ -54,8 +54,8 @@ CachedV2MulledImageMultiTarget.package_hash = _package_hash

 def list_docker_cached_mulled_images(namespace=None, hash_func="v2"):
    command = build_docker_images_command(truncate=True, sudo=False)
-    command = "%s | tail -n +2 | tr -s ' ' | cut -d' ' -f1,2" % command
-    images_and_versions = check_output(command)
+    images_and_versions = subprocess.check_output(command).strip().split('\n')
+    images_and_versions = [line.split()[0:2] for line in images_and_versions[1:]]
    name_filter = get_filter(namespace)

    def output_line_to_image(line):
@@ -116,17 +116,12 @@ def get_affected_packages(args):
    """
    recipes_dir = args.recipes_dir
    hours = args.diff_hours
-    cmd = """cd '%s' && git log --diff-filter=ACMRTUXB --name-only --pretty="" --since="%s hours ago" | grep -E '^recipes/.*/meta.yaml' | sort | uniq""" % (recipes_dir, hours)
-    pkg_list = check_output(cmd, shell=True)
-    ret = list()
-    for pkg in pkg_list.strip().split('\n'):
+    cmd = ['git', 'log', '--diff-filter=ACMRTUXB', '--name-only', '--pretty=""', '--since="%s hours ago"' % hours]
+    changed_files = subprocess.check_output(cmd, cwd=recipes_dir).strip().split('\n')
+    pkg_list = set([x for x in changed_files if x.startswith('recipes/') and x.endswith('meta.yaml')])
+    for pkg in pkg_list:
        if pkg and os.path.exists(os.path.join(recipes_dir, pkg)):
-            ret.append((get_pkg_name(args, pkg), get_tests(args, pkg)))
-    return ret
-
-
-def check_output(cmd, shell=True):
-    return subprocess.check_output(cmd, shell=shell)
+            yield (get_pkg_name(args, pkg), get_tests(args, pkg))


 def conda_versions(pkg_name, file_name):
@@ -19,6 +19,7 @@ See recent changes that would be built with:
 from __future__ import print_function

 import os
+import subprocess
 import sys
 import time

@@ -27,7 +28,6 @@ from .mulled_build import (
    add_build_arguments,
    args_to_mull_targets_kwds,
    build_target,
-    check_output,
    conda_versions,
    get_affected_packages,
    mull_targets,
@@ -42,7 +42,13 @@ def _fetch_repo_data(args):
        repo_data = "%s-repodata.json" % channel
    if not os.path.exists(repo_data):
        platform_tag = 'osx-64' if sys.platform == 'darwin' else 'linux-64'
-        check_output("wget --quiet https://conda.anaconda.org/%s/%s/repodata.json.bz2 -O '%s.bz2' && bzip2 -d '%s.bz2'" % (channel, platform_tag, repo_data, repo_data))
+        subprocess.check_call([
+            'wget', '--quiet', 'https://conda.anaconda.org/%s/%s/repodata.json.bz2' % (channel, platform_tag),
+            '-O', '%s.bz2' % repo_data
+        ])
+        subprocess.check_call([
+            'bzip2', '-d', '%s.bz2' % repo_data
+        ])
    return repo_data


@@ -55,8 +61,7 @@ def _new_versions(quay, conda):

 def run_channel(args, build_last_n_versions=1):
    """Build list of involucro commands (as shell snippet) to run."""
-    pkgs = get_affected_packages(args)
-    for pkg_name, pkg_tests in pkgs:
+    for pkg_name, pkg_tests in get_affected_packages(args):
        repo_data = _fetch_repo_data(args)
        c = conda_versions(pkg_name, repo_data)
        # only package the most recent N versions
@@ -97,7 +97,7 @@ class MappableDependencyResolver:
    @staticmethod
    def _mapping_file_to_list(mapping_file):
        with open(mapping_file, "r") as f:
-            raw_mapping = yaml.load(f) or []
+            raw_mapping = yaml.safe_load(f) or []
        return map(RequirementMapping.from_dict, raw_mapping)

    def _expand_mappings(self, requirement):
@@ -173,6 +173,14 @@ class ModuleDependency(Dependency):
        self.module_version = module_version
        self._exact = exact

+    @property
+    def name(self):
+        return self.module_name
+
+    @property
+    def version(self):
+        return self.module_version
+
    @property
    def exact(self):
        return self._exact
@@ -15,7 +15,7 @@ import tarfile
 import tempfile
 from base64 import b64decode

-from six.moves.urllib.request import urlopen
+import requests

 # Set max size of archive/file that will be handled to be 100 GB. This is
 # arbitrary and should be adjusted as needed.
@@ -27,18 +27,16 @@ def url_to_file(url, dest_file):
    Transfer a file from a remote URL to a temporary file.
    """
    try:
-        url_reader = urlopen(url)
+        url_reader = requests.get(url, stream=True)
        CHUNK = 10 * 1024  # 10k
        total = 0
        fp = open(dest_file, 'wb')
-        while True:
-            chunk = url_reader.read(CHUNK)
-            if not chunk:
-                break
-            fp.write(chunk)
-            total += CHUNK
-            if total > MAX_SIZE:
-                break
+        for chunk in url_reader.iter_content(chunk_size=CHUNK):
+            if chunk:
+                fp.write(chunk)
+                total += CHUNK
+                if total > MAX_SIZE:
+                    break
        fp.close()
        return dest_file
    except Exception as e:
@@ -57,7 +57,7 @@ class _Ga4ghToolClient(object):
        if as_string:
            return descriptor_str
        else:
-            return yaml.load(descriptor_str)
+            return yaml.safe_load(descriptor_str)

    @property
    def _requests(self):
@@ -83,7 +83,7 @@ class FilterFactory(object):
            try:
                __import__(full_module_name)
            except ImportError:
-                # log.debug("Failed to load module.", exc_info=True)
+                # log.debug("Failed to load module %s using sys.path %s", full_module_name, sys.path, exc_info=True)
                continue
            module = sys.modules[full_module_name]
            if hasattr(module, function_name):
@@ -59,7 +59,7 @@ class YamlToolConfSource(ToolConfSource):

    def __init__(self, config_filename):
        with open(config_filename, "r") as f:
-            as_dict = yaml.load(f)
+            as_dict = yaml.safe_load(f)
        self.as_dict = as_dict

    def parse_tool_path(self):
@@ -68,7 +68,7 @@ class ToursRegistry(object):
        tour_id = os.path.splitext(filename)[0]
        try:
            with open(tour_path) as handle:
-                conf = yaml.load(handle)
+                conf = yaml.safe_load(handle)
                tour = tour_loader(conf)
                self.tours[tour_id] = tour_loader(conf)
                log.info("Loaded tour '%s'" % tour_id)
@@ -83,4 +83,4 @@ def __read_yaml(path):
        raise ImportError("Attempting to read YAML configuration file - but PyYAML dependency unavailable.")

    with open(path, "rb") as f:
-        return yaml.load(f)
+        return yaml.safe_load(f)
@@ -67,7 +67,7 @@ def load_app_properties(
                config_section = "galaxy"

            with open(config_file, "r") as f:
-                raw_properties = yaml.load(f)
+                raw_properties = yaml.safe_load(f)
            properties = raw_properties[config_section] or {}

    override_prefix = "%sOVERRIDE_" % config_prefix
@@ -111,7 +111,7 @@ class InteractiveEnvironmentRequest(object):
                raise Exception("[{0}] Could not find allowed_images.yml, or image tag in {0}.ini file for ".format(self.attr.viz_id))

        with open(fn, 'r') as handle:
-            self.allowed_images = [x['image'] for x in yaml.load(handle)]
+            self.allowed_images = [x['image'] for x in yaml.safe_load(handle)]

            if len(self.allowed_images) == 0:
                raise Exception("No allowed images specified for " + self.attr.viz_id)
@@ -373,7 +373,7 @@ class InteractiveEnvironmentRequest(object):
        p = Popen(raw_cmd, stdout=PIPE, stderr=PIPE, close_fds=True, shell=True)
        stdout, stderr = p.communicate()
        if p.returncode != 0:
-            log.error("%s\n%s" % (stdout, stderr))
+            log.error("Container Launch error\n\n%s\n%s" % (stdout, stderr))
            return None
        else:
            container_id = stdout.strip()
@@ -484,7 +484,7 @@ class InteractiveEnvironmentRequest(object):
        p = Popen(command, stdout=PIPE, stderr=PIPE, close_fds=True, shell=True)
        stdout, stderr = p.communicate()
        if p.returncode != 0:
-            log.error("%s\n%s" % (stdout, stderr))
+            log.error("Container Launch error\n\n%s\n%s" % (stdout, stderr))
            return None

        inspect_data = json.loads(stdout)
@@ -2,17 +2,18 @@
 Visualization plugins: instantiate/deserialize data and models
 from a query string and render a webpage based on those data.
 """
-
-import logging
 import copy
-import mako
+import logging
 import os

-from galaxy.managers import api_keys
+import mako.lookup

-from galaxy.visualization.plugins import interactive_environments
-from galaxy.visualization.plugins import resource_parser
-from galaxy.visualization.plugins import utils
+from galaxy.managers import api_keys
+from galaxy.visualization.plugins import (
+    interactive_environments,
+    resource_parser,
+    utils
+)

 log = logging.getLogger(__name__)

@@ -8,9 +8,10 @@ from galaxy.util import sockets
 from galaxy.util.lazy_process import LazyProcess, NoOpLazyProcess
 from galaxy.util import sqlite
 from galaxy.util import unique_id
-import urllib2
 import time

+import requests
+
 log = logging.getLogger(__name__)


@@ -301,21 +302,17 @@ class RestGolangProxyIpc(object):
            'ContainerIds': container_ids,
        }

-        req = urllib2.Request(self.api_url)
-        req.add_header('Content-Type', 'application/json')
-
        # Sometimes it takes our poor little proxy a second or two to get
        # going, so if this fails, re-call ourselves with an increased timeout.
        try:
-            urllib2.urlopen(req, json.dumps(values))
-        except urllib2.URLError as err:
-            log.debug(err)
+            requests.get(self.api_url, headers={'Content-Type': 'application/json'}, data=json.dumps(values))
+        except requests.exceptions.ConnectionError as err:
+            log.exception(err)
            if sleep > 5:
                excp = "Could not contact proxy after %s seconds" % sum(range(sleep + 1))
                raise Exception(excp)
            time.sleep(sleep)
-            self.handle_requests(authentication, proxy_requests, route_name, container_ids, sleep=sleep + 1)
-        pass
+            self.handle_requests(authentication, proxy_requests, route_name, container_ids, container_interface, sleep=sleep + 1)


 ProxyMapping = namedtuple('ProxyMapping', ['host', 'port', 'container_ids', 'container_interface'])
@@ -11,8 +11,8 @@ import string
 import sys
 import tempfile
 from textwrap import TextWrapper
-import urllib2

+import requests
 import six
 from six import StringIO

@@ -307,9 +307,8 @@ def _write_option_rst(args, rst, key, heading_level, option_value):


 def _build_uwsgi_schema(args, app_desc):
-    req = urllib2.Request('https://raw.githubusercontent.com/unbit/uwsgi-docs/master/Options.rst')
-    response = urllib2.urlopen(req)
-    rst_options = response.read()
+    req = requests.get('https://raw.githubusercontent.com/unbit/uwsgi-docs/master/Options.rst')
+    rst_options = req.text
    last_line = None
    current_opt = None

@@ -282,7 +282,7 @@ class UserAPIController(BaseAPIController, UsesTagsMixin, CreatesUsersMixin, Cre
        path = trans.app.config.user_preferences_extra_config_file
        try:
            with open(path, 'r') as stream:
-                config = yaml.load(stream)
+                config = yaml.safe_load(stream)
        except:
            log.warning('Config file (%s) could not be found or is malformed.' % path)
            return {}
@@ -2,9 +2,13 @@
 Upload class
 """

+from __future__ import absolute_import
+
 import logging
 import urllib

+import requests
+
 from galaxy import jobs, web
 from galaxy.util import Params
 from galaxy.util.hash_util import hmac_new
@@ -162,8 +166,7 @@ class ASync(BaseUIController):
                url = "%s%s%s" % (url, url_join_char, urllib.urlencode(params.flatten()))
                log.debug("connecting to -> %s" % url)
                trans.log_event("Async connecting to -> %s" % url)
-                text = urllib.urlopen(url).read(-1)
-                text = text.strip()
+                text = requests.get(url).text.strip()
                if not text.endswith('OK'):
                    raise Exception(text)
                data.state = data.blurb = data.states.RUNNING
@@ -1,4 +1,7 @@
+from __future__ import absolute_import
+
 import glob
+import json
 import logging
 import operator
 import os
@@ -8,10 +11,9 @@ import sys
 import tarfile
 import tempfile
 import urllib
-import urllib2
 import zipfile
-from json import dumps, loads

+import requests
 from markupsafe import escape
 from sqlalchemy import and_, false
 from sqlalchemy.orm import eagerload_all
@@ -555,7 +557,7 @@ class LibraryCommon(BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMeta
            if len(em_string):
                payload = None
                try:
-                    payload = loads(em_string)
+                    payload = json.loads(em_string)
                except Exception:
                    message = 'Invalid JSON input'
                    status = 'error'
@@ -1122,8 +1124,8 @@ class LibraryCommon(BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMeta
        json_file_path = upload_common.create_paramfile(trans, uploaded_datasets)
        data_list = [ud.data for ud in uploaded_datasets]
        job_params = {}
-        job_params['link_data_only'] = dumps(kwd.get('link_data_only', 'copy_files'))
-        job_params['uuid'] = dumps(kwd.get('uuid', None))
+        job_params['link_data_only'] = json.dumps(kwd.get('link_data_only', 'copy_files'))
+        job_params['uuid'] = json.dumps(kwd.get('uuid', None))
        job, output = upload_common.create_job(trans, tool_params, tool, json_file_path, data_list, folder=library_bunch.folder, job_params=job_params)
        trans.sa_session.add(job)
        trans.sa_session.flush()
@@ -2762,9 +2764,7 @@ def lucene_search(trans, cntrller, search_term, search_url, **kwd):
    message = escape(kwd.get('message', ''))
    status = kwd.get('status', 'done')
    full_url = "%s/find?%s" % (search_url, urllib.urlencode({"kwd" : search_term}))
-    response = urllib2.urlopen(full_url)
-    ldda_ids = loads(response.read())["ids"]
-    response.close()
+    ldda_ids = requests.get(full_url).json()['ids']
    lddas = [trans.sa_session.query(trans.app.model.LibraryDatasetDatasetAssociation).get(ldda_id) for ldda_id in ldda_ids]
    return status, message, get_sorted_accessible_library_items(trans, cntrller, lddas, 'name')

@@ -1,10 +1,12 @@
 """
 Contains the main interface in the Universe class
 """
+from __future__ import absolute_import
+
 import cgi
 import os
-import urllib

+import requests
 from paste.httpexceptions import HTTPNotFound, HTTPBadGateway

 from galaxy import web
@@ -470,8 +472,8 @@ class RootController(controller.JSAppLauncher, UsesAnnotations):
    def bucket_proxy(self, trans, bucket=None, **kwd):
        if bucket:
            trans.response.set_content_type('text/xml')
-            b_list_xml = urllib.urlopen('http://s3.amazonaws.com/%s/' % bucket)
-            return b_list_xml.read()
+            b_list_xml = requests.get('http://s3.amazonaws.com/%s/' % bucket)
+            return b_list_xml.text
        raise Exception("You must specify a bucket")

    # ---- Debug methods ----------------------------------------------------
@@ -931,7 +931,7 @@ class VisualizationController(BaseUIController, SharableMixin, UsesVisualization
                        continue

                    with open(image_file, 'r') as handle:
-                        self.gie_image_map[gie] = yaml.load(handle)
+                        self.gie_image_map[gie] = yaml.safe_load(handle)

        return trans.fill_template_mako(
            "visualization/gie.mako",
@@ -1,10 +1,12 @@
+from __future__ import absolute_import
+
 import base64
 import httplib
 import json
 import logging
 import os
 import sgmllib
-import urllib2
+import requests

 from sqlalchemy import and_
 from sqlalchemy.orm import joinedload
@@ -763,7 +765,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
                # Load workflow from external URL
                # NOTE: blocks the web thread.
                try:
-                    workflow_data = urllib2.urlopen(url).read()
+                    workflow_data = requests.get(url).text
                except Exception as e:
                    message = "Failed to open URL: %s. Exception: %s" % (escape(url), escape(str(e)))
                    status = 'error'
@@ -64,7 +64,7 @@ class Configuration(object):

    def check(self):
        # Check that required directories exist
-        for path in self.root, self.file_path, self.template_path:
+        for path in self.root, self.template_path:
            if not os.path.isdir(path):
                raise ConfigurationError("Directory does not exist: %s" % path)

@@ -1,5 +1,6 @@
 import logging
 import os
+import subprocess
 from datetime import datetime, timedelta
 from decimal import Decimal

@@ -148,12 +149,11 @@ class System(BaseUIController):
                                   message=message)

    def get_disk_usage(self, file_path):
-        df_cmd = 'df -h ' + file_path
        is_sym_link = os.path.islink(file_path)
        file_system = disk_size = disk_used = disk_avail = disk_cap_pct = mount = None
-        df_file = os.popen(df_cmd)
-        while True:
-            df_line = df_file.readline()
+        df_output = subprocess.check_output(['df', '-h', file_path])
+
+        for df_line in df_output:
            df_line = df_line.strip()
            if df_line:
                df_line = df_line.lower()
@@ -176,7 +176,6 @@ class System(BaseUIController):
                        pass
            else:
                break  # EOF
-        df_file.close()
        return (file_system, disk_size, disk_used, disk_avail, disk_cap_pct, mount)

    @web.expose
@@ -3,7 +3,8 @@ import os
 import shutil
 import tarfile
 import tempfile
-import urllib
+
+import requests

 from galaxy import util
 from galaxy import web
@@ -74,7 +75,7 @@ class UploadController(BaseUIController):
            elif url:
                valid_url = True
                try:
-                    stream = urllib.urlopen(url)
+                    stream = requests.get(url, stream=True)
                except Exception as e:
                    valid_url = False
                    message = 'Error uploading file via http: %s' % str(e)
@@ -83,11 +84,9 @@ class UploadController(BaseUIController):
                if valid_url:
                    fd, uploaded_file_name = tempfile.mkstemp()
                    uploaded_file = open(uploaded_file_name, 'wb')
-                    while 1:
-                        chunk = stream.read(util.CHUNK_SIZE)
-                        if not chunk:
-                            break
-                        uploaded_file.write(chunk)
+                    for chunk in stream.iter_content(chunk_size=util.CHUNK_SIZE):
+                        if chunk:
+                            uploaded_file.write(chunk)
                    uploaded_file.flush()
                    uploaded_file_filename = url.split('/')[-1]
                    isempty = os.path.getsize(os.path.abspath(uploaded_file_name)) == 0
@@ -64,7 +64,7 @@ class WebhooksRegistry(object):
    def load_webhook_from_config(self, config_dir, config_file):
        try:
            with open(os.path.join(config_dir, config_file)) as file:
-                config = yaml.load(file)
+                config = yaml.safe_load(file)
                path = os.path.normpath(os.path.join(config_dir, '..'))
                webhook = Webhook(
                    config['name'],
@@ -7,7 +7,7 @@ import tempfile
 import threading
 from time import gmtime, strftime

-from six.moves.urllib.request import urlopen
+import requests
 from sqlalchemy import and_, false

 import tool_shed.repository_types.util as rt_util
@@ -809,24 +809,20 @@ class ImportRepositoryManager(object):
                           uploaded_file=None,
                           capsule_file_name=None)
        if url:
-            valid_url = True
            try:
-                stream = urlopen(url)
+                stream = requests.get(url, stream=True)
            except Exception as e:
-                valid_url = False
                return_dict['error_message'] = 'Error importing file via http: %s' % str(e)
                return_dict['status'] = 'error'
                return return_dict
-            if valid_url:
-                fd, uploaded_file_name = tempfile.mkstemp()
-                uploaded_file = open(uploaded_file_name, 'wb')
-                while 1:
-                    chunk = stream.read(CHUNK_SIZE)
-                    if not chunk:
-                        break
+
+            fd, uploaded_file_name = tempfile.mkstemp()
+            uploaded_file = open(uploaded_file_name, 'wb')
+            for chunk in stream.iter_content(chunk_size=CHUNK_SIZE):
+                if chunk:
                    uploaded_file.write(chunk)
-                uploaded_file.flush()
-                uploaded_file_filename = url.split('/')[-1]
+            uploaded_file.flush()
+            uploaded_file_filename = url.split('/')[-1]
        elif file_data not in ('', None):
            uploaded_file = file_data.file
            uploaded_file_name = uploaded_file.name
@@ -4,11 +4,13 @@
 # Script makes the naive assumption that dbkey==sequence id, which in many cases is not true nor desired
 # *** This script is not recommended for use as-is on a production server ***
 from __future__ import print_function
+
 import optparse
 import time
+
 from six.moves.urllib.parse import urljoin

-from common import get, post
+from common import get, post  # noqa: I100

 DEFAULT_SLEEP_TIME = 3
 FETCH_GENOME_TOOL_ID = 'testtoolshed.g2.bx.psu.edu/repos/blankenberg/data_manager_fetch_genome_all_fasta/data_manager_fetch_genome_all_fasta/0.0.1'
@@ -6,7 +6,7 @@ import sys

 from six.moves.urllib.error import URLError

-from common import display
+from common import display  # noqa: I100

 try:
    display(*sys.argv[1:3])
@@ -2,10 +2,12 @@
 Sample script for Galaxy Search API
 """
 from __future__ import print_function
+
 import json
-import requests
 import sys

+import requests
+

 class RemoteGalaxy(object):

@@ -1,75 +0,0 @@
-#!/usr/bin/env python
-"""
-Creates a Python binary-compatibility.cfg as described here:
-
-    https://mail.python.org/pipermail/distutils-sig/2015-July/026617.html
-"""
-from __future__ import print_function
-
-# Vaguely Python 2.6 compatibile ArgumentParser import
-try:
-    from argparse import ArgumentParser
-except ImportError:
-    from optparse import OptionParser
-
-    class ArgumentParser(OptionParser):
-
-        def __init__(self, **kwargs):
-            self.delegate = OptionParser(**kwargs)
-
-        def add_argument(self, *args, **kwargs):
-            if "required" in kwargs:
-                del kwargs["required"]
-            return self.delegate.add_option(*args, **kwargs)
-
-        def parse_args(self, args=None):
-            (options, args) = self.delegate.parse_args(args)
-            return options
-
-import sys
-import json
-
-from pip.pep425tags import get_supported
-from pip.platform import get_specific_platform
-
-
-compatible_platforms = {
-    'centos': 'rhel',
-    'scientific': 'rhel',
-}
-
-
-def install_compat():
-    spec_plat = get_specific_platform()
-    if spec_plat is None:
-        return None
-    this_plat = spec_plat[0]
-    compat_plat = compatible_platforms.get(this_plat, None)
-    rval = {}
-    if compat_plat:
-        print('{0} is binary compatible with {1} (and can install {2} wheels)'
-              .format(this_plat, compat_plat, compat_plat),
-              file=sys.stderr)
-        for py, abi, plat in get_supported():
-            if this_plat in plat:
-                rval[plat] = {'install': [plat.replace(this_plat, compat_plat)]}
-    return rval
-
-
-def main():
-    arg_parser = ArgumentParser()
-    arg_parser.add_argument('-o', '--output', default=None, help='Output to file')
-    args = arg_parser.parse_args()
-
-    compat = install_compat()
-
-    if compat:
-        if args.output is not None:
-            with open(args.output, 'w') as out:
-                json.dump(compat, out)
-        else:
-            print(json.dumps(install_compat()))
-
-
-if __name__ == '__main__':
-    main()
@@ -139,9 +139,9 @@ if [ $REPLACE_PIP -eq 1 ]; then
 fi

 if [ $FETCH_WHEELS -eq 1 ]; then
-    pip install -r requirements.txt --index-url "${GALAXY_WHEELS_INDEX_URL}"
+    pip install -r requirements.txt --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url https://pypi.python.org/pypi
    GALAXY_CONDITIONAL_DEPENDENCIES=$(PYTHONPATH=lib python -c "import galaxy.dependencies; print '\n'.join(galaxy.dependencies.optional('$GALAXY_CONFIG_FILE'))")
-    [ -z "$GALAXY_CONDITIONAL_DEPENDENCIES" ] || echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}"
+    [ -z "$GALAXY_CONDITIONAL_DEPENDENCIES" ] || echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url https://pypi.python.org/pypi
 fi

 if [ $FETCH_WHEELS -eq 1 -a $DEV_WHEELS -eq 1 ]; then
@@ -1,6 +1,8 @@
 #!/bin/sh

-uwsgi_args="--master --pythonpath=lib"
+: ${GALAXY_STATIC_DIRECTORY:=static}
+: ${GALAXY_STYLE_DIRECTORY:=static/style/blue}
+uwsgi_args="--master --pythonpath=lib --static-map /static=$GALAXY_STATIC_DIRECTORY --static-map /static/style=$GALAXY_STYLE_DIRECTORY"

 parse_common_args() {
    INITIALIZE_TOOL_DEPENDENCIES=1
@@ -14,7 +14,8 @@ import csv
 import os
 import sys
 import urllib
-import urllib2
+
+import requests

 sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))

@@ -39,9 +40,7 @@ def main(ini_file):

 def build_index(search_url, dataset_file):
    url = "%s/index?%s" % (search_url, urllib.urlencode({"docfile": dataset_file}))
-    request = urllib2.Request(url)
-    request.get_method = lambda: "PUT"
-    urllib2.urlopen(request)
+    requests.put(url)


 def create_dataset_file(dataset_iter):
@@ -2,8 +2,8 @@
 import argparse
 import datetime
 import os
-import time
 import sys
+import time

 sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'test')))

@@ -16,13 +16,14 @@ from __future__ import print_function

 import os
 import sys
-import urllib2
 from xml import etree

+import requests
+
 sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))

-import galaxy.model
 import galaxy.datatypes.registry
+import galaxy.model

 SCRIPTS_DIR = os.path.dirname(__file__)
 PROJECT_DIR = os.path.abspath(os.path.join(SCRIPTS_DIR, os.pardir))
@@ -35,7 +36,7 @@ EDAM_OWL_URL = "http://data.bioontology.org/ontologies/EDAM/submissions/25/downl


 if not os.path.exists("/tmp/edam.owl"):
-    open("/tmp/edam.owl", "w").write(urllib2.urlopen(EDAM_OWL_URL).read())
+    open("/tmp/edam.owl", "w").write(requests.get(EDAM_OWL_URL).text)


 owl_xml_tree = etree.ElementTree.parse("/tmp/edam.owl")
@@ -1,7 +1,12 @@
 #!/usr/bin/env python
 from __future__ import print_function
+
 from os import pardir
-from os.path import join, abspath, dirname
+from os.path import (
+    abspath,
+    dirname,
+    join
+)
 from sys import exit

 msg = """
@@ -4,25 +4,24 @@
 See doc/source/admin/grt.rst for more detailed usage information.
 """
 import argparse
-import tarfile
 import json
+import logging
 import os
-import sqlalchemy as sa
 import subprocess
 import sys
+import tarfile
 import time
-import yaml
-import logging
-
 from collections import defaultdict

-sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))
+import sqlalchemy as sa
+import yaml

-from galaxy.util.properties import load_app_properties
+sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))
 import galaxy
 import galaxy.config
-from galaxy.objectstore import build_object_store_from_config
 from galaxy.model import mapping
+from galaxy.objectstore import build_object_store_from_config
+from galaxy.util.properties import load_app_properties

 sample_config = os.path.abspath(os.path.join(os.path.dirname(__file__), 'grt.yml.sample'))
 default_config = os.path.abspath(os.path.join(os.path.dirname(__file__), 'grt.yml'))
@@ -191,11 +190,11 @@ def main(argv):
    annotate('init_start', 'Loading GRT configuration...')
    try:
        with open(args.config) as handle:
-            config = yaml.load(handle)
+            config = yaml.safe_load(handle)
    except Exception:
        logging.info('Using default GRT configuration')
        with open(sample_config) as handle:
-            config = yaml.load(handle)
+            config = yaml.safe_load(handle)
    annotate('init_end')

    REPORT_DIR = args.report_directory
@@ -4,11 +4,12 @@
 See doc/source/admin/grt.rst for more detailed usage information.
 """
 import argparse
+import logging
 import os
 import sys
-import yaml
-import logging
+
 import requests
+import yaml

 sample_config = os.path.abspath(os.path.join(os.path.dirname(__file__), 'grt.yml.sample'))
 default_config = os.path.abspath(os.path.join(os.path.dirname(__file__), 'grt.yml'))
@@ -28,11 +29,11 @@ def main(argv):
    logging.info('Loading GRT configuration...')
    try:
        with open(args.config) as handle:
-            config = yaml.load(handle)
+            config = yaml.safe_load(handle)
    except Exception:
        logging.info('Using default GRT configuration')
        with open(sample_config) as handle:
-            config = yaml.load(handle)
+            config = yaml.safe_load(handle)

    REPORT_DIR = args.report_directory
    GRT_URL = config['grt']['url'].rstrip('/') + '/'
@@ -4,7 +4,6 @@ import sys

 sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))

-from galaxy.util.properties import find_config_file, load_app_properties
 from galaxy.config import (
    configure_logging,
    find_path,
@@ -12,6 +11,7 @@ from galaxy.config import (
    parse_dependency_options,
 )
 from galaxy.tools.deps import CachedDependencyManager, DependencyManager, NullDependencyManager
+from galaxy.util.properties import find_config_file, load_app_properties

 DESCRIPTION = "Script to manage tool dependencies (with focus on a Conda environments)."

@@ -45,12 +45,15 @@ from __future__ import generators

 import codecs
 import re
+import sgmllib
 import string
 import sys
 import types
-import sgmllib
 from htmlentitydefs import name2codepoint
-from sgmllib import SGMLParser, SGMLParseError
+from sgmllib import (
+    SGMLParseError,
+    SGMLParser
+)

 __author__ = "Leonard Richardson (crummy.com)"
 __contributors__ = ["Sam Ruby (intertwingly.net)",
@@ -8,11 +8,16 @@ import os
 import sys
 import time
 from ftplib import FTP
-from urllib2 import urlopen
 from urllib import urlretrieve

+import requests
 from BeautifulSoup import BeautifulSoup
-from util import get_bed_from_genbank, get_bed_from_glimmer3, get_bed_from_GeneMarkHMM, get_bed_from_GeneMark
+from util import (
+    get_bed_from_genbank,
+    get_bed_from_GeneMark,
+    get_bed_from_GeneMarkHMM,
+    get_bed_from_glimmer3
+)

 assert sys.version_info[:2] >= (2, 4)

@@ -26,7 +31,7 @@ desired_ftp_files = {'GeneMark': {'ext': 'GeneMark-2.5f', 'parser': 'process_Gen

 # number, name, chroms, kingdom, group, genbank, refseq, info_url, ftp_url
 def iter_genome_projects(url="http://www.ncbi.nlm.nih.gov/genomes/lproks.cgi?view=1", info_url_base="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=genomeprj&cmd=Retrieve&dopt=Overview&list_uids="):
-    for row in BeautifulSoup(urlopen(url)).findAll(name='tr', bgcolor=["#EEFFDD", "#E8E8DD"]):
+    for row in BeautifulSoup(requests.get(url).text).findAll(name='tr', bgcolor=["#EEFFDD", "#E8E8DD"]):
        row = str(row).replace("\n", "").replace("\r", "")

        fields = row.split("</td>")
@@ -65,7 +70,7 @@ def get_chroms_by_project_id(org_num, base_url="http://www.ncbi.nlm.nih.gov/entr
        html_count += 1
        url = "%s%s" % (base_url, org_num)
        try:
-            html = urlopen(url)
+            html = requests.get(url).text
        except:
            print "GENOME PROJECT FAILED:", html_count, "org:", org_num, url
            html = None
@@ -11,7 +11,7 @@ sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pa

 from galaxy.util.pastescript import serve

-from check_python import check_python
+from check_python import check_python  # noqa: I100

 # ensure supported version
 try:
@@ -16,8 +16,10 @@ try:
 except:
    pass

-from docutils.core import publish_cmdline, default_description
-
+from docutils.core import (
+    default_description,
+    publish_cmdline
+)

 description = ('Generates (X)HTML documents from standalone reStructuredText '
               'sources.  ' + default_description)
@@ -1,8 +1,8 @@
 """Script to parse timings out of a Galaxy log and summarize."""
 from __future__ import print_function

-from argparse import ArgumentParser
 import re
+from argparse import ArgumentParser

 import numpy

@@ -11,12 +11,13 @@ import argparse
 import os
 import sys
 import tempfile
-import urllib2
+
+import requests

 sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 'lib'))
 from tool_shed.util import basic_util

-from common import display, submit
+from common import display, submit  # noqa: I100

 CAPSULE_FILENAME = 'capsule'
 CAPSULE_WITH_DEPENDENCIES_FILENAME = 'capsule_with_dependencies'
@@ -98,24 +99,11 @@ def main(options):
            download_url = export_dict['download_url']
            download_dir = os.path.abspath(options.download_dir)
            file_path = os.path.join(download_dir, repositories_archive_filename)
-            src = None
-            dst = None
-            try:
-                src = urllib2.urlopen(download_url)
-                dst = open(file_path, 'wb')
-                while True:
-                    chunk = src.read(CHUNK_SIZE)
+            src = requests.get(download_url, stream=True)
+            with open(file_path, 'wb') as dst:
+                for chunk in src.iter_content(chunk_size=CHUNK_SIZE):
                    if chunk:
                        dst.write(chunk)
-                    else:
-                        break
-            except:
-                raise
-            finally:
-                if src:
-                    src.close()
-                if dst:
-                    dst.close()
            print "Successfully exported revision ", options.changeset_revision, " of repository ", options.name, " owned by ", options.owner
            print "to location ", file_path
    else:
@@ -28,7 +28,7 @@ sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
 from galaxy.util import asbool
 from tool_shed.util import hg_util

-from common import get_api_url, get_repository_dict, json_from_url
+from common import get_api_url, get_repository_dict, json_from_url  # noqa: I100


 def main(options):
@@ -1,6 +1,6 @@
 #!/usr/bin/python
-import optparse
 import ConfigParser
+import optparse
 import os
 import sys

@@ -2,10 +2,10 @@

 import ConfigParser
 import logging
+import optparse
 import os
 import re
 import sys
-import optparse

 sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 'lib'))
 sys.path.insert(1, os.path.join(os.path.dirname(__file__)))
@@ -13,7 +13,8 @@ sys.path.insert(1, os.path.join(os.path.dirname(__file__)))
 import galaxy.webapps.tool_shed.config as tool_shed_config
 from galaxy.web import security
 from galaxy.webapps.tool_shed.model import mapping
-from bootstrap_util import admin_user_info
+
+from bootstrap_util import admin_user_info  # noqa: I100

 log = logging.getLogger(__name__)

@@ -8,8 +8,8 @@ import sys
 import textwrap
 import time
 from datetime import datetime, timedelta
-from time import strftime
 from optparse import OptionParser
+from time import strftime

 import sqlalchemy as sa
 from sqlalchemy import and_, distinct, false, not_
@@ -18,8 +18,10 @@ sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,

 import galaxy.webapps.tool_shed.config as tool_shed_config
 import galaxy.webapps.tool_shed.model.mapping
-from galaxy.util import send_mail as galaxy_send_mail
-from galaxy.util import build_url
+from galaxy.util import (
+    build_url,
+    send_mail as galaxy_send_mail
+)

 log = logging.getLogger()
 log.setLevel(10)
@@ -1,10 +1,12 @@
 # Dan Blankenberg
 # This script checks maf_index.loc file for inconsistencies between what is listed as available and what is really available.
 # Make sure that required dependencies (e.g. galaxy_root/lib) are included in your PYTHONPATH
-import bx.align.maf
-from galaxy.tools.util import maf_utilities
 import sys

+import bx.align.maf
+
+from galaxy.tools.util import maf_utilities
+
 assert sys.version_info[:2] >= (2, 4)


@@ -215,6 +215,7 @@ def transfer(app, transfer_job_id):
 def http_transfer(transfer_job):
    """Plugin" for handling http(s) transfers."""
    url = transfer_job.params['url']
+    assert url.startswith('http://') or url.startswith('https://')
    try:
        f = urllib2.urlopen(url)
    except urllib2.URLError as e:
@@ -117,10 +117,10 @@
    ## Get URL to other published items owned by user that owns this item.
    <%
        ##TODO: is there a better way to create this URL? Can't use 'f-username' as a key b/c it's not a valid identifier.
-        controller_name = get_controller_name( item )
+        modern_route = modern_route_for_controller(get_controller_name(item))
        item_plural = get_item_plural( item )
-        href_to_all_items = h.url_for( controller='/' + controller_name, action='list_published')
-        href_to_user_items = h.url_for( controller='/' + controller_name, action='list_published', xxx=item.user.username)
+        href_to_all_items = h.url_for( controller='/' + modern_route, action='list_published')
+        href_to_user_items = h.url_for( controller='/' + modern_route, action='list_published', xxx=item.user.username)
        href_to_user_items = href_to_user_items.replace( 'xxx', 'f-username')
    %>

@@ -162,10 +162,10 @@

    <%
        ## FIXME: duplicated from above for now
-        controller_name = get_controller_name( item )
+        modern_route = modern_route_for_controller(get_controller_name(item))
        item_plural = get_item_plural( item )
-        href_to_all_items = h.url_for( controller='/' + controller_name, action='list_published')
-        href_to_user_items = h.url_for( controller='/' + controller_name, action='list_published', xxx=item.user.username)
+        href_to_all_items = h.url_for( controller='/' + modern_route , action='list_published')
+        href_to_user_items = h.url_for( controller='/' + modern_route, action='list_published', xxx=item.user.username)
        href_to_user_items = href_to_user_items.replace( 'xxx', 'f-username')
    %>

@@ -111,6 +111,21 @@
    %>
 </%def>

+<%def name="modern_route_for_controller( controller )">
+    <%
+        if controller == "history":
+            return "histories"
+        elif controller == "workflow":
+            return "workflows"
+        elif controller == "dataset":
+            return "datasets"
+        elif controller == "page":
+            return "pages"
+        else:
+            return controller
+    %>
+</%def>
+
 ## Returns item user/owner.
 <%def name="get_item_user( item )">
    <%
@@ -2,7 +2,7 @@ import json

 from requests import post

-from base import api
+from base import api  # noqa: I100


 class ApiBatchTestCase(api.ApiTestCase):
@@ -2,7 +2,7 @@ import base64

 from requests import get

-from base import api
+from base import api  # noqa: I100

 TEST_USER_EMAIL = "auth_user_test@bx.psu.edu"
 TEST_USER_PASSWORD = "testpassword1"
@@ -2,7 +2,7 @@ import time

 from requests import put

-from base import api
+from base import api  # noqa: I100

 HIDDEN_DURING_UPLOAD_DATATYPE = "fli"

@@ -1,14 +1,16 @@
 # -*- coding: utf-8 -*-
-
 from requests import (
    get,
    post,
    put
 )

-from base import api
-
-from base.populators import DatasetPopulator, DatasetCollectionPopulator, wait_on
+from base import api  # noqa: I100
+from base.populators import (  # noqa: I100
+    DatasetCollectionPopulator,
+    DatasetPopulator,
+    wait_on
+)


 class HistoriesApiTestCase(api.ApiTestCase):
@@ -4,8 +4,12 @@ import json

 from requests import delete, put

-from base import api
-from base.populators import DatasetCollectionPopulator, LibraryPopulator, TestsDatasets
+from base import api  # noqa: I100
+from base.populators import (  # noqa: I100
+    DatasetCollectionPopulator,
+    LibraryPopulator,
+    TestsDatasets
+)


 # TODO: Test anonymous access.
@@ -1,6 +1,6 @@
 from requests import delete

-from base import api
+from base import api  # noqa: I100
 from galaxy.exceptions import error_codes


@@ -6,8 +6,8 @@ from base.populators import (
    DatasetCollectionPopulator,
    DatasetPopulator,
    LibraryPopulator,
-    skip_without_tool,
    skip_without_datatype,
+    skip_without_tool,
 )
 from galaxy.tools.verify.test_data import TestDataResolver

@@ -1,8 +1,11 @@
 import json

-from requests import put, get
+from requests import (
+    get,
+    put
+)

-from base import api
+from base import api  # noqa: I100

 TEST_USER_EMAIL = "user_for_users_index_test@bx.psu.edu"

@@ -5,8 +5,8 @@ from requests import (
    put
 )

-from base import api
-from base.api_asserts import assert_has_keys
+from base import api  # noqa: I100
+from base.api_asserts import assert_has_keys  # noqa: I100

 INDEX_KEYS = [
    'id', 'title', 'type', 'dbkey', 'url'
@@ -8,8 +8,8 @@ from uuid import uuid4
 import yaml
 from requests import delete, put

-from base import api
-from base.populators import (
+from base import api  # noqa: I100
+from base.populators import (  # noqa: I100
    DatasetCollectionPopulator,
    DatasetPopulator,
    skip_without_tool,
@@ -172,7 +172,7 @@ class BaseWorkflowsApiTestCase(api.ApiTestCase):
        )
        if jobs_descriptions is None:
            assert source_type != "path"
-            jobs_descriptions = yaml.load(has_workflow)
+            jobs_descriptions = yaml.safe_load(has_workflow)

        test_data = jobs_descriptions.get("test_data", {})

@@ -55,11 +55,11 @@ steps:
        assert tool_count['random_lines1'] == 1
        assert tool_count['cat1'] == 2

-# FIXME:  This test fails on some machines due to (we're guessing) yaml loading
+# FIXME:  This test fails on some machines due to (we're guessing) yaml.safe_loading
 # order being not guaranteed and inconsistent across platforms.  The workflow
-# yaml loader probably needs to enforce order using something like the
+# yaml.safe_loader probably needs to enforce order using something like the
 # approach described here:
-# https://stackoverflow.com/questions/13297744/pyyaml-control-ordering-of-items-called-by-yaml-load
+# https://stackoverflow.com/questions/13297744/pyyaml-control-ordering-of-items-called-by-yaml.safe_load
 #     def test_multiple_input( self ):
 #         history_id = self.dataset_populator.new_history()
 #         self._run_jobs("""
@@ -1,17 +1,15 @@
 import contextlib
 import json
 import time
-
 from functools import wraps
 from operator import itemgetter

 import requests
-
 from pkg_resources import resource_string
 from six import StringIO

-from base import api_asserts
-from base.workflows_format_2 import (
+from . import api_asserts
+from .workflows_format_2 import (
    convert_and_import_workflow,
    ImporterGalaxyInterface,
 )
@@ -32,7 +32,7 @@ RUN_ACTIONS_TO_STEPS = {

 def yaml_to_workflow(has_yaml, galaxy_interface, workflow_directory):
    """Convert a Format 2 workflow into standard Galaxy format from supplied stream."""
-    as_python = yaml.load(has_yaml)
+    as_python = yaml.safe_load(has_yaml)
    return python_to_workflow(as_python, galaxy_interface, workflow_directory)


@@ -109,7 +109,7 @@ def _python_to_workflow(as_python, conversion_context):
                run_action_path = run_action["@import"]
                runnable_path = os.path.join(conversion_context.workflow_directory, run_action_path)
                with open(runnable_path, "r") as f:
-                    runnable_description = yaml.load(f)
+                    runnable_description = yaml.safe_load(f)
                    run_action = runnable_description

            run_class = run_action["class"]
@@ -20,7 +20,7 @@ def convert_and_import_workflow(has_workflow, **kwds):
        if workflow_directory is None:
            workflow_directory = os.path.dirname(has_workflow)
        with open(workflow_path, "r") as f:
-            has_workflow = yaml.load(f)
+            has_workflow = yaml.safe_load(f)

    if workflow_directory is not None:
        workflow_directory = os.path.abspath(workflow_directory)
--- a/Show More
+++ b/Show More