Implement IP address whitelisting

config/galaxy.ini.sample

@@ -1069,6 +1069,15 @@ use_interactive = True
 #expose_user_name = False
 #expose_user_email = False
 
+# Whitelist for local network addresses for "Upload from URL" dialog.
+# By default, Galaxy will deny access to the local network address space, to
+# prevent users making requests to services which the administrator did not
+# intend to expose. Previously, you could request any network service that
+# Galaxy might have had access to, even if the user could not normally access it.
+# It should be a comma separated list of IP addresses or IP address/mask, e.g.
+# 10.10.10.10,10.0.1.0/24,fd00::/8
+#fetch_url_whitelist=
+
 # -- Beta features
 
 # Enable new run workflow form

lib/galaxy/config.py

@@ -226,6 +226,13 @@ class Configuration(object):
         self.remote_user_logout_href = kwargs.get("remote_user_logout_href", None)
         self.remote_user_secret = kwargs.get("remote_user_secret", None)
         self.require_login = string_as_bool(kwargs.get("require_login", "False"))
+        self.fetch_url_whitelist_ips = [
+            ipaddress.ip_network(ip.strip()) # If it has a slash, assume 127.0.0.1/24 notation
+            if '/' in ip else
+            ipaddress.ip_address(ip.strip()) # Otherwise interpret it as an ip address.
+            for ip in kwargs.get("fetch_url_whitelist", "").split(',')
+            if len(ip.strip()) > 0
+        ]
         self.allow_user_creation = string_as_bool(kwargs.get("allow_user_creation", "True"))
         self.allow_user_deletion = string_as_bool(kwargs.get("allow_user_deletion", "False"))
         self.allow_user_dataset_purge = string_as_bool(kwargs.get("allow_user_dataset_purge", "True"))

lib/galaxy/tools/actions/upload.py

@@ -19,7 +19,7 @@ class UploadToolAction(ToolAction):
 
         persisting_uploads_timer = ExecutionTimer()
         precreated_datasets = upload_common.get_precreated_datasets(trans, incoming, trans.app.model.HistoryDatasetAssociation)
-        incoming = upload_common.persist_uploads(incoming)
+        incoming = upload_common.persist_uploads(incoming, trans)
         log.debug("Persisted uploads %s" % persisting_uploads_timer)
         # We can pass an empty string as the cntrller here since it is used to check whether we
         # are in an admin view, and this tool is currently not used there.

lib/galaxy/tools/actions/upload_common.py

@@ -1,6 +1,8 @@
+import ipaddress
 import logging
 import os
 import shlex
+import socket
 import subprocess
 import tempfile
 from cgi import FieldStorage
@@ -8,16 +10,91 @@ from json import dumps
 
 from six import StringIO
 from sqlalchemy.orm import eagerload_all
+try:
+    from urlparse import urlparse
+except ImportError:
+    from urllib.parse import urlparse
 
 from galaxy import datatypes, util
 from galaxy.exceptions import ObjectInvalid
 from galaxy.managers import tags
+from galaxy.util import unicodify
 from galaxy.util.odict import odict
 
 log = logging.getLogger(__name__)
 
 
-def persist_uploads(params):
+def validate_url(url, ip_whitelist):
+    # Extract hostname component
+    parsed_url = urlparse(url).netloc
+    # If credentials are in this URL, we need to strip those.
+    if parsed_url.count('@') > 0:
+        # credentials.
+        parsed_url = parsed_url[parsed_url.rindex('@') + 1:]
+    # Percent encoded colons and other characters will not be resolved as such
+    # so we don't have to either.
+
+    # Sometimes the netloc will contain the port which is not desired, so we
+    # need to extract that.
+    port = None
+    # However, it could ALSO be an IPv6 address they've supplied.
+    if ':' in parsed_url:
+        # IPv6 addresses have colons in them already (it seems like always more than two)
+        if parsed_url.count(':') >= 2:
+            # Since IPv6 already use colons extensively, they wrap it in
+            # brackets when there is a port, e.g. http://[2001:db8:1f70::999:de8:7648:6e8]:100/
+            # However if it ends with a ']' then there is no port after it and
+            # they've wrapped it in brackets just for fun.
+            if ']' in parsed_url and not parsed_url.endswith(']'):
+                port = parsed_url[parsed_url.rindex(':') + 1:]
+                # If that +1 throws a range error, we don't care, their url
+                # shouldn't end with a colon.
+            else:
+                # Plain ipv6 without port
+                pass
+        else:
+            # This should finally be ipv4 with port. It cannot be IPv6 as that
+            # was caught by earlier cases, and it cannot be due to credentials.
+            port = parsed_url[parsed_url.rindex(':') + 1:]
+
+    # Call getaddrinfo to resolve hostname into tuples containing IPs.
+    addrinfo = socket.getaddrinfo(parsed_url, port)
+    # Get the IP addresses that this entry resolves to (uniquely)
+    # We drop:
+    #   AF_* family: It will resolve to AF_INET or AF_INET6, getaddrinfo(3) doesn't even mention AF_UNIX,
+    #   socktype: We don't care if a stream/dgram/raw protocol
+    #   protocol: we don't care if it is tcp or udp.
+    addrinfo_results = set([info[4][0] for info in addrinfo])
+    # There may be multiple (e.g. IPv4 + IPv6 or DNS round robin). Any one of these
+    # could resolve to a local addresses (and could be returned by chance),
+    # therefore we must check them all.
+    for raw_ip in addrinfo_results:
+        # Convert to an IP object so we can tell if it is in private space.
+        ip = ipaddress.ip_address(unicodify(raw_ip))
+        # If this is a private address
+        if ip.is_private:
+            results = []
+            # If this IP is not anywhere in the whitelist
+            for whitelisted in ip_whitelist:
+                # If it's an IP address range (rather than a single one...)
+                if hasattr(whitelisted, 'subnets'):
+                    results.append(ip in whitelisted)
+                else:
+                    results.append(ip == whitelisted)
+
+            if any(results):
+                # If we had any True, then THIS (and ONLY THIS) IP address that
+                # that specific DNS entry resolved to is in whitelisted and
+                # safe to access. But we cannot exit here, we must ensure that
+                # all IPs that that DNS entry resolves to are likewise safe.
+                pass
+            else:
+                # Otherwise, we deny access.
+                raise Exception("Access to this address in not permitted by server configuration")
+    return url
+
+
+def persist_uploads(params, trans):
     """
     Turn any uploads in the submitted form to persisted files.
     """
@@ -35,7 +112,10 @@ def persist_uploads(params):
             elif type(f) == dict and 'local_filename' not in f:
                 raise Exception('Uploaded file was encoded in a way not understood by Galaxy.')
             if upload_dataset['url_paste'] and upload_dataset['url_paste'].strip() != '':
-                upload_dataset['url_paste'], is_multi_byte = datatypes.sniff.stream_to_file(StringIO(upload_dataset['url_paste']), prefix="strio_url_paste_")
+                upload_dataset['url_paste'], is_multi_byte = datatypes.sniff.stream_to_file(
+                    StringIO(validate_url(upload_dataset['url_paste'], trans.app.config.fetch_url_whitelist_ips)),
+                    prefix="strio_url_paste_"
+                )
             else:
                 upload_dataset['url_paste'] = None
             new_files.append(upload_dataset)

lib/galaxy/webapps/galaxy/controllers/library_common.py

@@ -1096,7 +1096,7 @@ class LibraryCommon(BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMeta
         if response_code == 200:
             precreated_datasets = upload_common.get_precreated_datasets(trans, tool_params, trans.app.model.LibraryDatasetDatasetAssociation, controller=cntrller)
             if upload_option == 'upload_file':
-                tool_params = upload_common.persist_uploads(tool_params)
+                tool_params = upload_common.persist_uploads(tool_params, trans)
                 uploaded_datasets = upload_common.get_uploaded_datasets(trans, cntrller, tool_params, precreated_datasets, dataset_upload_inputs, library_bunch=library_bunch)
             elif upload_option == 'upload_directory':
                 uploaded_datasets, response_code, message = self.get_server_dir_uploaded_datasets(trans, cntrller, kwd, full_dir, import_dir_desc, library_bunch, response_code, message)