From e50e7239b58ef80f62c537096bdf642198aaa9cb Mon Sep 17 00:00:00 2001 From: E Rasche Date: Mon, 21 Aug 2017 12:02:50 +0200 Subject: [PATCH] Implement IP address whitelisting --- config/galaxy.ini.sample | 9 ++ lib/galaxy/config.py | 7 ++ lib/galaxy/tools/actions/upload.py | 2 +- lib/galaxy/tools/actions/upload_common.py | 84 ++++++++++++++++++- .../galaxy/controllers/library_common.py | 2 +- 5 files changed, 100 insertions(+), 4 deletions(-) diff --git a/config/galaxy.ini.sample b/config/galaxy.ini.sample index 4896f40797a..b72f7b1569d 100644 --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -1069,6 +1069,15 @@ use_interactive = True #expose_user_name = False #expose_user_email = False +# Whitelist for local network addresses for "Upload from URL" dialog. +# By default, Galaxy will deny access to the local network address space, to +# prevent users making requests to services which the administrator did not +# intend to expose. Previously, you could request any network service that +# Galaxy might have had access to, even if the user could not normally access it. +# It should be a comma separated list of IP addresses or IP address/mask, e.g. +# 10.10.10.10,10.0.1.0/24,fd00::/8 +#fetch_url_whitelist= + # -- Beta features # Enable new run workflow form diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 4408dabef24..692985e2b40 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -226,6 +226,13 @@ class Configuration(object): self.remote_user_logout_href = kwargs.get("remote_user_logout_href", None) self.remote_user_secret = kwargs.get("remote_user_secret", None) self.require_login = string_as_bool(kwargs.get("require_login", "False")) + self.fetch_url_whitelist_ips = [ + ipaddress.ip_network(ip.strip()) # If it has a slash, assume 127.0.0.1/24 notation + if '/' in ip else + ipaddress.ip_address(ip.strip()) # Otherwise interpret it as an ip address. + for ip in kwargs.get("fetch_url_whitelist", "").split(',') + if len(ip.strip()) > 0 + ] self.allow_user_creation = string_as_bool(kwargs.get("allow_user_creation", "True")) self.allow_user_deletion = string_as_bool(kwargs.get("allow_user_deletion", "False")) self.allow_user_dataset_purge = string_as_bool(kwargs.get("allow_user_dataset_purge", "True")) diff --git a/lib/galaxy/tools/actions/upload.py b/lib/galaxy/tools/actions/upload.py index 4ce32706795..089205c41a2 100644 --- a/lib/galaxy/tools/actions/upload.py +++ b/lib/galaxy/tools/actions/upload.py @@ -19,7 +19,7 @@ class UploadToolAction(ToolAction): persisting_uploads_timer = ExecutionTimer() precreated_datasets = upload_common.get_precreated_datasets(trans, incoming, trans.app.model.HistoryDatasetAssociation) - incoming = upload_common.persist_uploads(incoming) + incoming = upload_common.persist_uploads(incoming, trans) log.debug("Persisted uploads %s" % persisting_uploads_timer) # We can pass an empty string as the cntrller here since it is used to check whether we # are in an admin view, and this tool is currently not used there. diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index 2c46a0813af..bad9880f3ad 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -1,6 +1,8 @@ +import ipaddress import logging import os import shlex +import socket import subprocess import tempfile from cgi import FieldStorage @@ -8,16 +10,91 @@ from json import dumps from six import StringIO from sqlalchemy.orm import eagerload_all +try: + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse from galaxy import datatypes, util from galaxy.exceptions import ObjectInvalid from galaxy.managers import tags +from galaxy.util import unicodify from galaxy.util.odict import odict log = logging.getLogger(__name__) -def persist_uploads(params): +def validate_url(url, ip_whitelist): + # Extract hostname component + parsed_url = urlparse(url).netloc + # If credentials are in this URL, we need to strip those. + if parsed_url.count('@') > 0: + # credentials. + parsed_url = parsed_url[parsed_url.rindex('@') + 1:] + # Percent encoded colons and other characters will not be resolved as such + # so we don't have to either. + + # Sometimes the netloc will contain the port which is not desired, so we + # need to extract that. + port = None + # However, it could ALSO be an IPv6 address they've supplied. + if ':' in parsed_url: + # IPv6 addresses have colons in them already (it seems like always more than two) + if parsed_url.count(':') >= 2: + # Since IPv6 already use colons extensively, they wrap it in + # brackets when there is a port, e.g. http://[2001:db8:1f70::999:de8:7648:6e8]:100/ + # However if it ends with a ']' then there is no port after it and + # they've wrapped it in brackets just for fun. + if ']' in parsed_url and not parsed_url.endswith(']'): + port = parsed_url[parsed_url.rindex(':') + 1:] + # If that +1 throws a range error, we don't care, their url + # shouldn't end with a colon. + else: + # Plain ipv6 without port + pass + else: + # This should finally be ipv4 with port. It cannot be IPv6 as that + # was caught by earlier cases, and it cannot be due to credentials. + port = parsed_url[parsed_url.rindex(':') + 1:] + + # Call getaddrinfo to resolve hostname into tuples containing IPs. + addrinfo = socket.getaddrinfo(parsed_url, port) + # Get the IP addresses that this entry resolves to (uniquely) + # We drop: + # AF_* family: It will resolve to AF_INET or AF_INET6, getaddrinfo(3) doesn't even mention AF_UNIX, + # socktype: We don't care if a stream/dgram/raw protocol + # protocol: we don't care if it is tcp or udp. + addrinfo_results = set([info[4][0] for info in addrinfo]) + # There may be multiple (e.g. IPv4 + IPv6 or DNS round robin). Any one of these + # could resolve to a local addresses (and could be returned by chance), + # therefore we must check them all. + for raw_ip in addrinfo_results: + # Convert to an IP object so we can tell if it is in private space. + ip = ipaddress.ip_address(unicodify(raw_ip)) + # If this is a private address + if ip.is_private: + results = [] + # If this IP is not anywhere in the whitelist + for whitelisted in ip_whitelist: + # If it's an IP address range (rather than a single one...) + if hasattr(whitelisted, 'subnets'): + results.append(ip in whitelisted) + else: + results.append(ip == whitelisted) + + if any(results): + # If we had any True, then THIS (and ONLY THIS) IP address that + # that specific DNS entry resolved to is in whitelisted and + # safe to access. But we cannot exit here, we must ensure that + # all IPs that that DNS entry resolves to are likewise safe. + pass + else: + # Otherwise, we deny access. + raise Exception("Access to this address in not permitted by server configuration") + return url + + +def persist_uploads(params, trans): """ Turn any uploads in the submitted form to persisted files. """ @@ -35,7 +112,10 @@ def persist_uploads(params): elif type(f) == dict and 'local_filename' not in f: raise Exception('Uploaded file was encoded in a way not understood by Galaxy.') if upload_dataset['url_paste'] and upload_dataset['url_paste'].strip() != '': - upload_dataset['url_paste'], is_multi_byte = datatypes.sniff.stream_to_file(StringIO(upload_dataset['url_paste']), prefix="strio_url_paste_") + upload_dataset['url_paste'], is_multi_byte = datatypes.sniff.stream_to_file( + StringIO(validate_url(upload_dataset['url_paste'], trans.app.config.fetch_url_whitelist_ips)), + prefix="strio_url_paste_" + ) else: upload_dataset['url_paste'] = None new_files.append(upload_dataset) diff --git a/lib/galaxy/webapps/galaxy/controllers/library_common.py b/lib/galaxy/webapps/galaxy/controllers/library_common.py index 935a3aa6e58..365dd48c6b9 100644 --- a/lib/galaxy/webapps/galaxy/controllers/library_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/library_common.py @@ -1096,7 +1096,7 @@ class LibraryCommon(BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMeta if response_code == 200: precreated_datasets = upload_common.get_precreated_datasets(trans, tool_params, trans.app.model.LibraryDatasetDatasetAssociation, controller=cntrller) if upload_option == 'upload_file': - tool_params = upload_common.persist_uploads(tool_params) + tool_params = upload_common.persist_uploads(tool_params, trans) uploaded_datasets = upload_common.get_uploaded_datasets(trans, cntrller, tool_params, precreated_datasets, dataset_upload_inputs, library_bunch=library_bunch) elif upload_option == 'upload_directory': uploaded_datasets, response_code, message = self.get_server_dir_uploaded_datasets(trans, cntrller, kwd, full_dir, import_dir_desc, library_bunch, response_code, message)