Fix tool shed bootstrapping script

scripts/tool_shed/api/common.py

@@ -1,60 +1,4 @@
-import json
-import os
-import sys
-from urllib.error import HTTPError
-from urllib.request import (
-    build_opener,
-    HTTPRedirectHandler,
-    install_opener,
-    Request,
-    urlopen
-)
-
-sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 'lib'))
-
-from galaxy import util
-from tool_shed.util import hg_util
-
-
-class HTTPRedirectWithDataHandler(HTTPRedirectHandler):
-
-    def __init__(self, method):
-        '''
-        Upon first inspection, it would seem that this shouldn't be necessary, but for some reason
-        not having a constructor explicitly set the request method breaks PUT requests.
-        '''
-        self.valid_methods = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE']
-        self.redirect_codes = ['301', '302', '303', '307']
-        self.method = method
-
-    def redirect_request(self, request, fp, code, msg, headers, new_url):
-        request_method = request.get_method()
-        if str(code) in self.redirect_codes and request_method in self.valid_methods:
-            new_url = new_url.replace(' ', '%20')
-            request = Request(new_url,
-                              data=request.data,
-                              headers=request.headers,
-                              origin_req_host=request.get_origin_req_host(),
-                              unverifiable=True)
-            if self.method in self.valid_methods:
-                if request.get_method() != self.method:
-                    request.get_method = lambda: self.method
-            return request
-        else:
-            HTTPRedirectHandler.redirect_request(request, fp, code, msg, headers, new_url)
-
-
-def build_request_with_data(url, data, api_key, method):
-    """Build a request with the received method."""
-    http_redirect_with_data_handler = HTTPRedirectWithDataHandler(method=method)
-    opener = build_opener(http_redirect_with_data_handler)
-    install_opener(opener)
-    url = make_url(url, api_key=api_key, args=None)
-    request = Request(url, headers={'Content-Type': 'application/json'}, data=json.dumps(data))
-    request_method = request.get_method()
-    if request_method != method:
-        request.get_method = lambda: method
-    return opener, request
+import requests
 
 
 def delete(api_key, url, data, return_formatted=True):
@@ -62,17 +6,12 @@ def delete(api_key, url, data, return_formatted=True):
     Sends an API DELETE request and acts as a generic formatter for the JSON response.  The
     'data' will become the JSON payload read by the Tool Shed.
     """
-    try:
-        opener, request = build_request_with_data(url, data, api_key, 'DELETE')
-        delete_request = opener.open(request)
-        response = json.loads(delete_request.read())
-    except HTTPError as e:
-        if return_formatted:
-            print(e)
-            print(e.read(1024))
-            sys.exit(1)
-        else:
-            return dict(status='error', message=str(e.read(1024)))
+    headers = {}
+    if api_key:
+        headers['x-api-key'] = api_key
+    response = requests.delete(url, headers=headers)
+    response.raise_for_status()
+    response = response.json()
     if return_formatted:
         print('Response')
         print('--------')
@@ -83,16 +22,7 @@ def delete(api_key, url, data, return_formatted=True):
 
 def display(url, api_key=None, return_formatted=True):
     """Sends an API GET request and acts as a generic formatter for the JSON response."""
-    try:
-        r = get(url, api_key=api_key)
-    except HTTPError as e:
-        print(e)
-        # Only return the first 1K of errors.
-        print(e.read(1024))
-        sys.exit(1)
-    if isinstance(r, str):
-        print('error: %s' % r)
-        return None
+    r = get(url, api_key=api_key)
     if not return_formatted:
         return r
     elif isinstance(r, list):
@@ -122,119 +52,32 @@ def display(url, api_key=None, return_formatted=True):
 
 def get(url, api_key=None):
     """Do the GET."""
-    url = make_url(url, api_key=api_key, args=None)
-    try:
-        return json.loads(urlopen(url).read())
-    except ValueError:
-        sys.exit("URL did not return JSON data")
-
-
-def get_api_url(base, parts, params=None):
-    """Compose and return a URL for the Tool Shed API."""
-    if 'api' in parts and parts.index('api') != 0:
-        parts.pop(parts.index('api'))
-        parts.insert(0, 'api')
-    elif 'api' not in parts:
-        parts.insert(0, 'api')
-    url = util.build_url(base, pathspec=parts, params=params)
-    return url
-
-
-def get_latest_downloadable_changeset_revision_via_api(url, name, owner):
-    """
-    Return the latest downloadable changeset revision for the repository defined by the received
-    name and owner.
-    """
-    error_message = ''
-    parts = ['api', 'repositories', 'get_ordered_installable_revisions']
-    params = dict(name=name, owner=owner)
-    api_url = get_api_url(base=url, parts=parts, params=params)
-    changeset_revisions, error_message = json_from_url(api_url)
-    if changeset_revisions is None or error_message:
-        return None, error_message
-    if len(changeset_revisions) >= 1:
-        return changeset_revisions[-1], error_message
-    return hg_util.INITIAL_CHANGELOG_HASH, error_message
-
-
-def get_repository_dict(url, repository_dict):
-    """
-    Send a request to the Tool Shed to get additional information about the repository defined
-    by the received repository_dict.  Add the information to the repository_dict and return it.
-    """
-    error_message = ''
-    if not isinstance(repository_dict, dict):
-        error_message = 'Invalid repository_dict received: %s' % str(repository_dict)
-        return None, error_message
-    repository_id = repository_dict.get('repository_id', None)
-    if repository_id is None:
-        error_message = 'Invalid repository_dict does not contain a repository_id entry: %s' % str(repository_dict)
-        return None, error_message
-    parts = ['api', 'repositories', repository_id]
-    api_url = get_api_url(base=url, parts=parts)
-    extended_dict, error_message = json_from_url(api_url)
-    if extended_dict is None or error_message:
-        return None, error_message
-    name = extended_dict.get('name', None)
-    owner = extended_dict.get('owner', None)
-    if name is not None and owner is not None:
-        name = str(name)
-        owner = str(owner)
-        latest_changeset_revision, error_message = get_latest_downloadable_changeset_revision_via_api(url, name, owner)
-        if latest_changeset_revision is None or error_message:
-            return None, error_message
-        extended_dict['latest_revision'] = str(latest_changeset_revision)
-        return extended_dict, error_message
-    else:
-        error_message = 'Invalid extended_dict does not contain name or owner entries: %s' % str(extended_dict)
-        return None, error_message
-
-
-def json_from_url(url):
-    """Send a request to the Tool Shed via the Tool Shed API and handle the response."""
-    error_message = ''
-    url_handle = urlopen(url)
-    url_contents = url_handle.read()
-    try:
-        parsed_json = json.loads(url_contents)
-    except Exception as e:
-        error_message = str(url_contents)
-        print('Error parsing JSON data in json_from_url():', e)
-        return None, error_message
-    return parsed_json, error_message
-
-
-def make_url(url, api_key=None, args=None):
-    """Adds the API Key to the URL if it's not already there."""
-    if args is None:
-        args = []
-    argsep = '&'
-    if '?' not in url:
-        argsep = '?'
+    headers = {}
     if api_key:
-        if '?key=' not in url and '&key=' not in url:
-            args.insert(0, ('key', api_key))
-    return url + argsep + '&'.join('='.join(t) for t in args)
+        headers['x-api-key'] = api_key
+    response = requests.get(url, headers=headers)
+    response.raise_for_status()
+    return response.json()
 
 
 def post(url, data, api_key=None):
     """Do the POST."""
-    try:
-        opener, request = build_request_with_data(url, data, api_key, 'POST')
-        post_request = opener.open(request)
-        return json.loads(post_request.read())
-    except HTTPError as e:
-        return dict(status='error', message=str(e.read(1024)))
+    headers = {}
+    if api_key:
+        headers['x-api-key'] = api_key
+    response = requests.post(url, data, headers=headers)
+    response.raise_for_status()
+    return response.json()
 
 
 def put(url, data, api_key=None):
     """Do the PUT."""
-    try:
-        opener, request = build_request_with_data(url, data, api_key, 'PUT')
-        put_request = opener.open(request)
-        return json.loads(put_request.read())
-    except HTTPError as e:
-        return dict(status='error', message=str(e.read(1024)))
+    headers = {}
+    if api_key:
+        headers['x-api-key'] = api_key
+    response = requests.put(url, data, headers=headers)
+    response.raise_for_status()
+    return response.json()
 
 
 def submit(url, data, api_key=None, return_formatted=True):
@@ -242,15 +85,7 @@ def submit(url, data, api_key=None, return_formatted=True):
     Sends an API POST request and acts as a generic formatter for the JSON response.  The
     'data' will become the JSON payload read by the Tool Shed.
     """
-    try:
-        response = post(url, data, api_key=api_key)
-    except HTTPError as e:
-        if return_formatted:
-            print(e)
-            print(e.read(1024))
-            sys.exit(1)
-        else:
-            return dict(status='error', message=str(e.read(1024)))
+    response = post(url, data, api_key)
     if not return_formatted:
         return response
     print('Response')
@@ -279,15 +114,7 @@ def update(api_key, url, data, return_formatted=True):
     Sends an API PUT request and acts as a generic formatter for the JSON response.  The
     'data' will become the JSON payload read by the Tool Shed.
     """
-    try:
-        response = put(url, data, api_key=api_key)
-    except HTTPError as e:
-        if return_formatted:
-            print(e)
-            print(e.read(1024))
-            sys.exit(1)
-        else:
-            return dict(status='error', message=str(e.read(1024)))
+    response = put(url, data, api_key=api_key)
     if return_formatted:
         print('Response')
         print('--------')

scripts/tool_shed/bootstrap_tool_shed/bootstrap_tool_shed.sh

@@ -1,17 +1,25 @@
-#!/bin/bash
+#!/usr/bin/env bash
+
+set -x
 
 # Activate the virtualenv, if it exists.
 [ -f ./.venv/bin/activate ] && . ./.venv/bin/activate
 
-: ${TOOL_SHED_CONFIG_FILE:=config/tool_shed.yml.sample}
+: "${TOOL_SHED_CONFIG_FILE:=lib/galaxy/config/sample/tool_shed.yml.sample}"
+LOCAL_SHED_URL="${LOCAL_SHED_URL:-http://localhost:9009}"
+TOOL_SHED_PID=tool_shed_bootstrap.pid
+TOOL_SHED_LOG=tool_shed_bootstrap.log
+export TOOL_SHED_PID
+export TOOL_SHED_LOG
+export TOOL_SHED_CONFIG_FILE
 
 stop_err() {
-	echo $1
-	python ./scripts/paster.py serve ${TOOL_SHED_CONFIG_FILE} --pid-file=tool_shed_bootstrap.pid --log-file=tool_shed_bootstrap.log --stop-daemon
+	echo "$1"
+	kill "$(cat $TOOL_SHED_PID)"
 	exit 1
 }
 
-tool_shed=`./scripts/tool_shed/bootstrap_tool_shed/parse_run_sh_args.sh $@`
+tool_shed=$(./scripts/tool_shed/bootstrap_tool_shed/parse_run_sh_args.sh "$@")
 
 if [ $? -ne 0 ] ; then
 	exit 0
@@ -19,7 +27,7 @@ fi
 
 log_file="scripts/tool_shed/bootstrap_tool_shed/bootstrap.log"
 
-database_result=`python ./scripts/tool_shed/bootstrap_tool_shed/bootstrap_util.py --execute check_db --config_file ${TOOL_SHED_CONFIG_FILE}`
+database_result="$(python ./scripts/tool_shed/bootstrap_tool_shed/bootstrap_util.py --execute check_db --config_file ${TOOL_SHED_CONFIG_FILE})"
 
 if [ $? -ne 0 ] ; then
 	stop_err "Unable to bootstrap tool shed. $database_result"
@@ -36,13 +44,12 @@ else
 fi
 
 if [ $? -eq 0 ] ; then
-	user_auth=`python ./scripts/tool_shed/bootstrap_tool_shed/bootstrap_util.py --execute admin_user_info --config_file ${TOOL_SHED_CONFIG_FILE}`
-	local_shed_url=`python ./scripts/tool_shed/bootstrap_tool_shed/bootstrap_util.py --execute get_url --config_file ${TOOL_SHED_CONFIG_FILE}`
+	user_auth=$(python ./scripts/tool_shed/bootstrap_tool_shed/bootstrap_util.py --execute admin_user_info --config_file ${TOOL_SHED_CONFIG_FILE})
 fi
 
-admin_user_name=`echo $user_auth | awk 'BEGIN { FS="__SEP__" } ; { print \$1 }'`
-admin_user_email=`echo $user_auth | awk 'BEGIN { FS="__SEP__" } ; { print \$2 }'`
-admin_user_password=`echo $user_auth | awk 'BEGIN { FS="__SEP__" } ; { print \$3 }'`
+admin_user_name=$(echo "$user_auth" | awk 'BEGIN { FS="__SEP__" } ; { print $1 }')
+admin_user_email=$(echo "$user_auth" | awk 'BEGIN { FS="__SEP__" } ; { print $2 }')
+admin_user_password=$(echo "$user_auth" | awk 'BEGIN { FS="__SEP__" } ; { print $3 }')
 
 echo -n "Creating user '$admin_user_name' with email address '$admin_user_email'..."
 
@@ -50,16 +57,15 @@ python ./scripts/tool_shed/bootstrap_tool_shed/create_user_with_api_key.py -c ${
 
 echo " done."
 
-sed -i"bak" "s/#admin_users = user1@example.org,user2@example.org/admin_users = $admin_user_email/" "${TOOL_SHED_CONFIG_FILE}"
+export TOOL_SHED_CONFIG_ADMIN_USERS=$admin_user_email
 echo -n "Starting tool shed in order to populate users and categories... "
 
-if [ -f tool_shed_bootstrap.pid ] ; then
+
+if [ -f "$TOOL_SHED_PID" ] ; then
 	stop_err "A bootstrap process is already running."
 fi
 
-python ./scripts/paster.py serve ${TOOL_SHED_CONFIG_FILE} --pid-file=tool_shed_bootstrap.pid --log-file=tool_shed_bootstrap.log --daemon > /dev/null
-
-shed_pid=`cat tool_shed_bootstrap.pid`
+./run_tool_shed.sh -bootstrap_from_tool_shed --daemon > /dev/null
 
 while : ; do
 	tail -n 1 tool_shed_bootstrap.log | grep -q "Removing PID file tool_shed_webapp.pid"
@@ -71,20 +77,20 @@ while : ; do
 		echo "========================================"
 		stop_err " "
 	fi
-	tail -n 2 tool_shed_bootstrap.log | grep -q "Starting server in PID $shed_pid"
+	tail -n 2 tool_shed_bootstrap.log | grep -q "Application startup complete."
 	if [ $? -eq 0 ] ; then
 		echo "done."
 		break
 	fi
 done
 
-echo -n "Retrieving admin user's API key from $local_shed_url..."
+echo -n "Retrieving admin user's API key from $LOCAL_SHED_URL..."
 
-curl_response=`curl -s --user $admin_user_email:$admin_user_password $local_shed_url/api/authenticate/baseauth/`
+curl_response=$(curl -s --user "$admin_user_email":"$admin_user_password" "$LOCAL_SHED_URL"/api/authenticate/baseauth/)
 # Gets an empty response only on first attempt for some reason?
 sleep 1
-curl_response=`curl -s --user $admin_user_email:$admin_user_password $local_shed_url/api/authenticate/baseauth/`
-api_key=`echo $curl_response | grep api_key | awk -F\" '{print $4}'`
+curl_response=$(curl -s --user "$admin_user_email":"$admin_user_password" "$LOCAL_SHED_URL"/api/authenticate/baseauth/)
+api_key=$(echo "$curl_response" | grep api_key | awk -F\" '{print $4}')
 
 if [[ -z $api_key && ${api_key+x} ]] ; then
 		stop_err "Error getting API key for user $admin_user_email. Response: $curl_response"
@@ -94,16 +100,16 @@ echo " done."
 
 if [ $? -eq 0 ] ; then
 	echo -n "Creating users... "
-	python scripts/tool_shed/api/create_users.py -a $api_key -f $tool_shed -t $local_shed_url >> $log_file
+	python scripts/tool_shed/api/create_users.py -a "$api_key" -f "$tool_shed" -t "$LOCAL_SHED_URL" >> "$log_file"
 	echo "done."
 	echo -n "Creating categories... "
-	python scripts/tool_shed/api/create_categories.py -a $api_key -f $tool_shed -t $local_shed_url >> $log_file
+	python scripts/tool_shed/api/create_categories.py -a "$api_key" -f "$tool_shed" -t "$LOCAL_SHED_URL" >> "$log_file"
 	echo "done."
 else
 	stop_err "Error getting API key from local tool shed."
 fi
 
 echo "Bootstrap complete, shutting down temporary tool shed process. A log has been saved to tool_shed_bootstrap.log"
-python ./scripts/paster.py serve ${TOOL_SHED_CONFIG_FILE} --pid-file=tool_shed_bootstrap.pid --log-file=tool_shed_bootstrap.log --stop-daemon
+kill "$(cat $TOOL_SHED_PID)"
 
 exit 0

scripts/tool_shed/bootstrap_tool_shed/bootstrap_util.py

@@ -11,18 +11,11 @@ sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
 
 import tool_shed.webapp.model.mapping as tool_shed_model
 from tool_shed.util import xml_util
+from tool_shed.webapp.config import ToolShedAppConfiguration
 
 
-def check_db(config_parser):
-    dburi = None
-
-    if config_parser.has_option('app:main', 'database_connection'):
-        dburi = config_parser.get('app:main', 'database_connection')
-    elif config_parser.has_option('app:main', 'database_file'):
-        db_file = config_parser.get('app:main', 'database_file')
-        dburi = "sqlite:///%s?isolation_level=IMMEDIATE" % db_file
-    else:
-        sys.exit('The database configuration setting is missing from the tool_shed.ini file.  Add this setting before attempting to bootstrap.')
+def check_db(config: ToolShedAppConfiguration):
+    dburi = config.database_connection
 
     sa_session = None
 
@@ -30,7 +23,9 @@ def check_db(config_parser):
     database_exists_message += 'Create a new database that has not been migrated before attempting to bootstrap.'
 
     try:
-        model = tool_shed_model.init(config_parser.get('app:main', 'file_path'), dburi, engine_options={}, create_tables=False)
+        model = tool_shed_model.init(
+            config.file_path, dburi, engine_options={}, create_tables=False
+        )
         sa_session = model.context.current
         sys.exit(database_exists_message)
     except ProgrammingError:
@@ -48,10 +43,10 @@ def check_db(config_parser):
     except ProgrammingError:
         pass
 
-    if config_parser.has_option('app:main', 'hgweb_config_dir'):
+    if config.hgweb_config_dir:
         hgweb_config_parser = ConfigParser()
-        hgweb_dir = config_parser.get('app:main', 'hgweb_config_dir')
-        hgweb_config_file = os.path.join(hgweb_dir, 'hgweb.config')
+        hgweb_dir = config.hgweb_config_dir
+        hgweb_config_file = os.path.join(hgweb_dir, "hgweb.config")
         if not os.path.exists(hgweb_config_file):
             sys.exit(0)
         hgweb_config_parser.read(hgweb_config_file)
@@ -63,8 +58,6 @@ def check_db(config_parser):
             sys.exit(message)
         else:
             sys.exit(0)
-    else:
-        sys.exit(0)
 
     sys.exit(0)
 
@@ -76,10 +69,9 @@ def admin_user_info():
     email = None
     password = None
     if tree is None:
-        print("The XML file ", user_info_config, " seems to be invalid, using defaults.")
-        email = 'admin@test.org'
-        password = 'testuser'
-        username = 'admin'
+        email = "admin@test.org"
+        password = "testuser"
+        username = "admin"
     else:
         root = tree.getroot()
         for elem in root:
@@ -92,40 +84,22 @@ def admin_user_info():
     return (username, email, password)
 
 
-def get_local_tool_shed_url(config_parser):
-    port = '9009'
-    if config_parser.has_section('server:main'):
-        if config_parser.has_option('server:main', 'port'):
-            port = config_parser.get('server:main', 'port')
-    host = '127.0.0.1'
-    print(f'http://{host}:{port}')
-    return 0
-
-
 def main(args):
-    config_parser = ConfigParser()
+    config = ToolShedAppConfiguration(config_file=args.config)
 
-    if os.path.exists(args.config):
-        config_parser.read(args.config)
-    else:
-        return 1
-
-    if args.method == 'check_db':
-        return check_db(config_parser)
-    elif args.method == 'admin_user_info':
+    if args.method == "check_db":
+        return check_db(config)
+    elif args.method == "admin_user_info":
         (username, email, password) = admin_user_info()
         print(f'{username}__SEP__{email}__SEP__{password}')
         return 0
-    elif args.method == 'get_url':
-        return get_local_tool_shed_url(config_parser)
     else:
         return 1
 
 
-parser = optparse.OptionParser()
-parser.add_option('-c', '--config_file', dest='config', action='store', default='config/tool_shed.yml.sample')
-parser.add_option('-e', '--execute', dest='method', action='store', default='check_db')
-(args, options) = parser.parse_args()
-
-if __name__ == '__main__':
+if __name__ == "__main__":
+    parser = optparse.OptionParser()
+    parser.add_option("-c", "--config_file", dest="config", action="store", default="config/tool_shed.yml.sample")
+    parser.add_option("-e", "--execute", dest="method", action="store", default="check_db")
+    (args, options) = parser.parse_args()
     sys.exit(main(args))

scripts/tool_shed/bootstrap_tool_shed/create_user_with_api_key.py

@@ -4,7 +4,6 @@ import logging
 import optparse
 import os
 import sys
-from configparser import ConfigParser
 
 sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 'lib'))
 sys.path.insert(1, os.path.join(os.path.dirname(__file__)))
@@ -39,9 +38,6 @@ class BootstrapApplication:
                                   engine_options={},
                                   create_tables=False)
         self.security = IdEncodingHelper(id_secret=self.config.id_secret)
-        self.hgweb_config_manager = self.model.hgweb_config_manager
-        self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir
-        print('Using hgweb.config file: ', self.hgweb_config_manager.hgweb_config)
 
     @property
     def sa_session(self):
@@ -89,17 +85,10 @@ def validate(email, password, username):
 
 
 if __name__ == "__main__":
-    parser = optparse.OptionParser(description='Create a user with API key.')
-    parser.add_option('-c', dest='config', action='store', help='.ini file to retrieve toolshed configuration from')
+    parser = optparse.OptionParser(description="Create a user with API key.")
+    parser.add_option("-c", dest="config", action="store", help="File to retrieve toolshed configuration from")
     (args, options) = parser.parse_args()
-    ini_file = args.config
-    config_parser = ConfigParser({'here': os.getcwd()})
-    print("Reading ini file: ", ini_file)
-    config_parser.read(ini_file)
-    config_dict = {}
-    for key, value in config_parser.items("app:main"):
-        config_dict[key] = value
-    config = tool_shed_config.Configuration(**config_dict)
+    config = tool_shed_config.Configuration(config_file=args.config)
     app = BootstrapApplication(config)
     user = create_user(app)
     if user is not None: