From 32b85ecb591d01acbbbb9ed1bedd1a48804a0837 Mon Sep 17 00:00:00 2001 From: E Rasche Date: Wed, 13 Sep 2017 06:03:55 +0000 Subject: [PATCH] Only permit yaml.safe_loading of data Event trusted data, belt + suspenders method. --- lib/galaxy/containers/__init__.py | 2 +- lib/galaxy/datatypes/registry.py | 2 +- lib/galaxy/jobs/runners/pulsar.py | 2 +- lib/galaxy/tools/deps/conda_compat.py | 2 +- lib/galaxy/tools/deps/resolvers/__init__.py | 2 +- lib/galaxy/tools/locations/dockstore.py | 2 +- lib/galaxy/tools/parser/factory.py | 2 +- lib/galaxy/tools/toolbox/parser.py | 2 +- lib/galaxy/tours/__init__.py | 2 +- lib/galaxy/util/plugin_config.py | 2 +- lib/galaxy/util/properties.py | 2 +- .../visualization/plugins/interactive_environments.py | 2 +- lib/galaxy/webapps/config_manage.py | 4 ++-- lib/galaxy/webapps/galaxy/api/users.py | 2 +- lib/galaxy/webapps/galaxy/controllers/visualization.py | 2 +- lib/galaxy/webhooks/__init__.py | 2 +- scripts/grt/export.py | 4 ++-- scripts/grt/upload.py | 4 ++-- test/api/test_workflows.py | 2 +- test/api/test_workflows_from_yaml.py | 6 +++--- test/base/workflows_format_2/converter.py | 4 ++-- test/base/workflows_format_2/main.py | 2 +- test/galaxy_selenium/data.py | 2 +- test/galaxy_selenium/navigates_galaxy.py | 2 +- test/unit/workflows/workflow_support.py | 2 +- 25 files changed, 31 insertions(+), 31 deletions(-) diff --git a/lib/galaxy/containers/__init__.py b/lib/galaxy/containers/__init__.py index 4b33dbe8d5a..6cebdd82e14 100644 --- a/lib/galaxy/containers/__init__.py +++ b/lib/galaxy/containers/__init__.py @@ -306,7 +306,7 @@ def parse_containers_config(containers_config_file): conf = DEFAULT_CONF.copy() try: with open(containers_config_file) as fh: - c = yaml.load(fh) + c = yaml.safe_load(fh) conf.update(c.get('containers', {})) except (OSError, IOError) as exc: if exc.errno == errno.ENOENT: diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 31902b14513..f11b8498719 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -359,7 +359,7 @@ class Registry(object): build_sites_config_file = getattr(self.config, "build_sites_config_file", None) if build_sites_config_file and os.path.exists(build_sites_config_file): with open(build_sites_config_file, "r") as f: - build_sites_config = yaml.load(f) + build_sites_config = yaml.safe_load(f) if not isinstance(build_sites_config, list): self.log.exception("Build sites configuration YAML file does not declare list of sites.") return diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 9722ea32736..9fee6ef7f81 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -217,7 +217,7 @@ class PulsarJobRunner(AsynchronousJobRunner): else: log.info("Loading Pulsar app configuration from %s" % pulsar_conf_path) with open(pulsar_conf_path, "r") as f: - conf.update(yaml.load(f) or {}) + conf.update(yaml.safe_load(f) or {}) if "job_metrics_config_file" not in conf: conf["job_metrics"] = self.app.job_metrics if "staging_directory" not in conf: diff --git a/lib/galaxy/tools/deps/conda_compat.py b/lib/galaxy/tools/deps/conda_compat.py index c45573ef49d..10527bf890d 100644 --- a/lib/galaxy/tools/deps/conda_compat.py +++ b/lib/galaxy/tools/deps/conda_compat.py @@ -61,7 +61,7 @@ def _render_jinja2(recipe_dir): @_Memoized def yamlize(data): - res = yaml.load(data) + res = yaml.safe_load(data) # ensure the result is a dict if res is None: res = {} diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py index e8188e0964a..5c3811024e0 100644 --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -97,7 +97,7 @@ class MappableDependencyResolver: @staticmethod def _mapping_file_to_list(mapping_file): with open(mapping_file, "r") as f: - raw_mapping = yaml.load(f) or [] + raw_mapping = yaml.safe_load(f) or [] return map(RequirementMapping.from_dict, raw_mapping) def _expand_mappings(self, requirement): diff --git a/lib/galaxy/tools/locations/dockstore.py b/lib/galaxy/tools/locations/dockstore.py index b379c23c38f..3eb96e55e24 100644 --- a/lib/galaxy/tools/locations/dockstore.py +++ b/lib/galaxy/tools/locations/dockstore.py @@ -57,7 +57,7 @@ class _Ga4ghToolClient(object): if as_string: return descriptor_str else: - return yaml.load(descriptor_str) + return yaml.safe_load(descriptor_str) @property def _requests(self): diff --git a/lib/galaxy/tools/parser/factory.py b/lib/galaxy/tools/parser/factory.py index 52a11ad7b57..b6655be63d5 100644 --- a/lib/galaxy/tools/parser/factory.py +++ b/lib/galaxy/tools/parser/factory.py @@ -62,7 +62,7 @@ def ordered_load(stream): yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping) - return yaml.load(stream, OrderedLoader) + return yaml.safe_load(stream, OrderedLoader) def get_input_source(content): diff --git a/lib/galaxy/tools/toolbox/parser.py b/lib/galaxy/tools/toolbox/parser.py index 3c79a8b4d7b..db1b582404c 100644 --- a/lib/galaxy/tools/toolbox/parser.py +++ b/lib/galaxy/tools/toolbox/parser.py @@ -59,7 +59,7 @@ class YamlToolConfSource(ToolConfSource): def __init__(self, config_filename): with open(config_filename, "r") as f: - as_dict = yaml.load(f) + as_dict = yaml.safe_load(f) self.as_dict = as_dict def parse_tool_path(self): diff --git a/lib/galaxy/tours/__init__.py b/lib/galaxy/tours/__init__.py index 0511aa29e83..c35d147759c 100644 --- a/lib/galaxy/tours/__init__.py +++ b/lib/galaxy/tours/__init__.py @@ -68,7 +68,7 @@ class ToursRegistry(object): tour_id = os.path.splitext(filename)[0] try: with open(tour_path) as handle: - conf = yaml.load(handle) + conf = yaml.safe_load(handle) tour = tour_loader(conf) self.tours[tour_id] = tour_loader(conf) log.info("Loaded tour '%s'" % tour_id) diff --git a/lib/galaxy/util/plugin_config.py b/lib/galaxy/util/plugin_config.py index 133998f210f..7f6866d6f2d 100644 --- a/lib/galaxy/util/plugin_config.py +++ b/lib/galaxy/util/plugin_config.py @@ -83,4 +83,4 @@ def __read_yaml(path): raise ImportError("Attempting to read YAML configuration file - but PyYAML dependency unavailable.") with open(path, "rb") as f: - return yaml.load(f) + return yaml.safe_load(f) diff --git a/lib/galaxy/util/properties.py b/lib/galaxy/util/properties.py index 2ca160c27f7..8c22e0aaf59 100644 --- a/lib/galaxy/util/properties.py +++ b/lib/galaxy/util/properties.py @@ -67,7 +67,7 @@ def load_app_properties( config_section = "galaxy" with open(config_file, "r") as f: - raw_properties = yaml.load(f) + raw_properties = yaml.safe_load(f) properties = raw_properties[config_section] or {} override_prefix = "%sOVERRIDE_" % config_prefix diff --git a/lib/galaxy/visualization/plugins/interactive_environments.py b/lib/galaxy/visualization/plugins/interactive_environments.py index 161882e9409..602da1257e1 100644 --- a/lib/galaxy/visualization/plugins/interactive_environments.py +++ b/lib/galaxy/visualization/plugins/interactive_environments.py @@ -111,7 +111,7 @@ class InteractiveEnvironmentRequest(object): raise Exception("[{0}] Could not find allowed_images.yml, or image tag in {0}.ini file for ".format(self.attr.viz_id)) with open(fn, 'r') as handle: - self.allowed_images = [x['image'] for x in yaml.load(handle)] + self.allowed_images = [x['image'] for x in yaml.safe_load(handle)] if len(self.allowed_images) == 0: raise Exception("No allowed images specified for " + self.attr.viz_id) diff --git a/lib/galaxy/webapps/config_manage.py b/lib/galaxy/webapps/config_manage.py index 07d0bc03658..4cad182179e 100644 --- a/lib/galaxy/webapps/config_manage.py +++ b/lib/galaxy/webapps/config_manage.py @@ -670,7 +670,7 @@ def _ordered_load(stream): def include(self, node): filename = os.path.join(self._root, self.construct_scalar(node)) with open(filename, 'r') as f: - return yaml.load(f, OrderedLoader) + return yaml.safe_load(f, OrderedLoader) def construct_mapping(loader, node): loader.flatten_mapping(node) @@ -681,7 +681,7 @@ def _ordered_load(stream): construct_mapping) OrderedLoader.add_constructor('!include', OrderedLoader.include) - return yaml.load(stream, OrderedLoader) + return yaml.safe_load(stream, OrderedLoader) def _ordered_dump(data, stream=None, Dumper=yaml.Dumper, **kwds): diff --git a/lib/galaxy/webapps/galaxy/api/users.py b/lib/galaxy/webapps/galaxy/api/users.py index 67ec64351f3..63c9e3a78a0 100644 --- a/lib/galaxy/webapps/galaxy/api/users.py +++ b/lib/galaxy/webapps/galaxy/api/users.py @@ -282,7 +282,7 @@ class UserAPIController(BaseAPIController, UsesTagsMixin, CreatesUsersMixin, Cre path = trans.app.config.user_preferences_extra_config_file try: with open(path, 'r') as stream: - config = yaml.load(stream) + config = yaml.safe_load(stream) except: log.warning('Config file (%s) could not be found or is malformed.' % path) return {} diff --git a/lib/galaxy/webapps/galaxy/controllers/visualization.py b/lib/galaxy/webapps/galaxy/controllers/visualization.py index f134ebe48b4..9c7c1e5424e 100644 --- a/lib/galaxy/webapps/galaxy/controllers/visualization.py +++ b/lib/galaxy/webapps/galaxy/controllers/visualization.py @@ -961,7 +961,7 @@ class VisualizationController(BaseUIController, SharableMixin, UsesVisualization continue with open(image_file, 'r') as handle: - self.gie_image_map[gie] = yaml.load(handle) + self.gie_image_map[gie] = yaml.safe_load(handle) return trans.fill_template_mako( "visualization/gie.mako", diff --git a/lib/galaxy/webhooks/__init__.py b/lib/galaxy/webhooks/__init__.py index 7e9c9482433..6cc2c34d9d9 100644 --- a/lib/galaxy/webhooks/__init__.py +++ b/lib/galaxy/webhooks/__init__.py @@ -64,7 +64,7 @@ class WebhooksRegistry(object): def load_webhook_from_config(self, config_dir, config_file): try: with open(os.path.join(config_dir, config_file)) as file: - config = yaml.load(file) + config = yaml.safe_load(file) path = os.path.normpath(os.path.join(config_dir, '..')) webhook = Webhook( config['name'], diff --git a/scripts/grt/export.py b/scripts/grt/export.py index 4619c6929c6..da8048522f2 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -191,11 +191,11 @@ def main(argv): annotate('init_start', 'Loading GRT configuration...') try: with open(args.config) as handle: - config = yaml.load(handle) + config = yaml.safe_load(handle) except Exception: logging.info('Using default GRT configuration') with open(sample_config) as handle: - config = yaml.load(handle) + config = yaml.safe_load(handle) annotate('init_end') REPORT_DIR = args.report_directory diff --git a/scripts/grt/upload.py b/scripts/grt/upload.py index 332cbe89e6d..a93efb86a62 100644 --- a/scripts/grt/upload.py +++ b/scripts/grt/upload.py @@ -28,11 +28,11 @@ def main(argv): logging.info('Loading GRT configuration...') try: with open(args.config) as handle: - config = yaml.load(handle) + config = yaml.safe_load(handle) except Exception: logging.info('Using default GRT configuration') with open(sample_config) as handle: - config = yaml.load(handle) + config = yaml.safe_load(handle) REPORT_DIR = args.report_directory GRT_URL = config['grt']['url'].rstrip('/') + '/' diff --git a/test/api/test_workflows.py b/test/api/test_workflows.py index 79a0c1c2218..f3c45c03368 100644 --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -172,7 +172,7 @@ class BaseWorkflowsApiTestCase(api.ApiTestCase): ) if jobs_descriptions is None: assert source_type != "path" - jobs_descriptions = yaml.load(has_workflow) + jobs_descriptions = yaml.safe_load(has_workflow) test_data = jobs_descriptions.get("test_data", {}) diff --git a/test/api/test_workflows_from_yaml.py b/test/api/test_workflows_from_yaml.py index 9beddf7aa2e..2b39628c27d 100644 --- a/test/api/test_workflows_from_yaml.py +++ b/test/api/test_workflows_from_yaml.py @@ -55,11 +55,11 @@ steps: assert tool_count['random_lines1'] == 1 assert tool_count['cat1'] == 2 -# FIXME: This test fails on some machines due to (we're guessing) yaml loading +# FIXME: This test fails on some machines due to (we're guessing) yaml.safe_loading # order being not guaranteed and inconsistent across platforms. The workflow -# yaml loader probably needs to enforce order using something like the +# yaml.safe_loader probably needs to enforce order using something like the # approach described here: -# https://stackoverflow.com/questions/13297744/pyyaml-control-ordering-of-items-called-by-yaml-load +# https://stackoverflow.com/questions/13297744/pyyaml-control-ordering-of-items-called-by-yaml.safe_load # def test_multiple_input( self ): # history_id = self.dataset_populator.new_history() # self._run_jobs(""" diff --git a/test/base/workflows_format_2/converter.py b/test/base/workflows_format_2/converter.py index 05a9a319c49..c35ec2797c0 100644 --- a/test/base/workflows_format_2/converter.py +++ b/test/base/workflows_format_2/converter.py @@ -32,7 +32,7 @@ RUN_ACTIONS_TO_STEPS = { def yaml_to_workflow(has_yaml, galaxy_interface, workflow_directory): """Convert a Format 2 workflow into standard Galaxy format from supplied stream.""" - as_python = yaml.load(has_yaml) + as_python = yaml.safe_load(has_yaml) return python_to_workflow(as_python, galaxy_interface, workflow_directory) @@ -109,7 +109,7 @@ def _python_to_workflow(as_python, conversion_context): run_action_path = run_action["@import"] runnable_path = os.path.join(conversion_context.workflow_directory, run_action_path) with open(runnable_path, "r") as f: - runnable_description = yaml.load(f) + runnable_description = yaml.safe_load(f) run_action = runnable_description run_class = run_action["class"] diff --git a/test/base/workflows_format_2/main.py b/test/base/workflows_format_2/main.py index 70d59a8db8d..a4af30e8da6 100644 --- a/test/base/workflows_format_2/main.py +++ b/test/base/workflows_format_2/main.py @@ -20,7 +20,7 @@ def convert_and_import_workflow(has_workflow, **kwds): if workflow_directory is None: workflow_directory = os.path.dirname(has_workflow) with open(workflow_path, "r") as f: - has_workflow = yaml.load(f) + has_workflow = yaml.safe_load(f) if workflow_directory is not None: workflow_directory = os.path.abspath(workflow_directory) diff --git a/test/galaxy_selenium/data.py b/test/galaxy_selenium/data.py index a3d969f8a4f..ef15e27197b 100644 --- a/test/galaxy_selenium/data.py +++ b/test/galaxy_selenium/data.py @@ -3,4 +3,4 @@ from pkg_resources import resource_string import yaml data_yaml = resource_string(__name__, 'navigation-data.yml').decode("UTF-8") -NAVIGATION_DATA = yaml.load(data_yaml) +NAVIGATION_DATA = yaml.safe_load(data_yaml) diff --git a/test/galaxy_selenium/navigates_galaxy.py b/test/galaxy_selenium/navigates_galaxy.py index cff486af9a7..1a60e7ada62 100644 --- a/test/galaxy_selenium/navigates_galaxy.py +++ b/test/galaxy_selenium/navigates_galaxy.py @@ -798,7 +798,7 @@ class NavigatesGalaxy(HasDriver): self.home() with open(path, "r") as f: - tour_dict = yaml.load(f) + tour_dict = yaml.safe_load(f) steps = tour_dict["steps"] for i, step in enumerate(steps): title = step.get("title", None) diff --git a/test/unit/workflows/workflow_support.py b/test/unit/workflows/workflow_support.py index ee04ded4fc9..6f4f567cb66 100644 --- a/test/unit/workflows/workflow_support.py +++ b/test/unit/workflows/workflow_support.py @@ -75,7 +75,7 @@ class TestToolbox(object): def yaml_to_model(has_dict, id_offset=100): if isinstance(has_dict, str): - has_dict = yaml.load(has_dict) + has_dict = yaml.safe_load(has_dict) workflow = model.Workflow() workflow.steps = []