opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-10-24 12:13:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #9528 from mvdbeek/twill_work

Browse Source 
Make Toolshed Python 3 compatible & tests
This commit is contained in:
Nicola Soranzo
2020-04-12 20:52:29 +01:00
committed by GitHub
parent 169cabbaba dd63ce043d
commit c11dee5886
25 changed files with 197 additions and 339 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/toolshed.yaml
+37
View File
@@ -0,0 +1,37 @@
name: Toolshed
on: [push, pull_request]
env:
GALAXY_TEST_DBURI: 'postgres://postgres:postgres@localhost:5432/galaxy?client_encoding=utf8'
TOOL_SHED_TEST_DBURI: 'postgres://postgres:postgres@localhost:5432/toolshed?client_encoding=utf8'
jobs:
test:
name: Test
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.7]
services:
postgres:
image: postgres:11
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: postgres
ports:
- 5432:5432
steps:
- uses: actions/checkout@v2
with:
path: 'galaxy root'
- uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python-version }}
- name: Cache pip dir
uses: actions/cache@v1
id: pip-cache
with:
path: ~/.cache/pip
key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
- name: Run tests
run: './run_tests.sh -toolshed'
working-directory: 'galaxy root'
lib/galaxy/dependencies/pipfiles/default/Pipfile
+1 -1
View File
@@ -25,7 +25,7 @@ Sphinx = "*"
sphinx_markdown_tables = "*"
sphinx_rtd_theme = "*"
testfixtures = "*"
twill = {version = "==0.9.1", markers = "python_version < '3'"}
twill = "*"
watchdog = "*"
python-irodsclient = "*"
lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt
+1 -1
View File
@@ -57,7 +57,7 @@ sphinx-rtd-theme==0.4.3
sphinx==1.8.5
sphinxcontrib-websupport==1.1.2
testfixtures==6.14.0
twill==0.9.1 ; python_version < '3'
twill==2.0
typing==3.7.4.1 ; python_version < '3.5'
urllib3==1.25.8
watchdog==0.10.2
lib/galaxy/model/tool_shed_install/migrate/check.py
+9
View File
@@ -12,6 +12,10 @@ from sqlalchemy import (
Table
)
from sqlalchemy.exc import NoSuchTableError
from sqlalchemy_utils import (
create_database,
database_exists,
)
from galaxy.model.tool_shed_install import mapping
@@ -27,6 +31,11 @@ def create_or_verify_database(url, engine_options={}, app=None):
"""
"""
# Create engine and metadata
if not database_exists(url):
message = "Creating database for URI [%s]" % url
log.info(message)
create_database(url)
engine = create_engine(url, **engine_options)
def migrate():
lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/asynchronous_reader.py
+4 -2
View File
@@ -1,6 +1,8 @@
import logging
import threading
from galaxy.util import unicodify
log = logging.getLogger(__name__)
@@ -20,8 +22,8 @@ class AsynchronousReader(threading.Thread):
"""Read lines and put them on the queue."""
thread_lock = threading.Lock()
thread_lock.acquire()
for line in iter(self._fd.readline, ''):
stripped_line = line.rstrip()
for line in iter(self._fd.readline, b''):
stripped_line = unicodify(line).rstrip()
self.lines.append(stripped_line)
self._queue.put(stripped_line)
thread_lock.release()
lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py
+21 -19
View File
@@ -13,13 +13,16 @@ from fabric.operations import _AttributeString
from six.moves import queue
from galaxy.tool_shed.galaxy_install.tool_dependencies.recipe import asynchronous_reader
from galaxy.tool_shed.util.basic_util import INSTALLATION_LOG, NO_OUTPUT_TIMEOUT
from galaxy.tool_shed.util.basic_util import (
INSTALLATION_LOG,
NO_OUTPUT_TIMEOUT,
)
from galaxy.tool_shed.util.tool_dependency_util import set_tool_dependency_attributes
from galaxy.util import (
DATABASE_MAX_STRING_SIZE,
DATABASE_MAX_STRING_SIZE_PRETTY,
shrink_string_by_size,
unicodify
unicodify,
)
log = logging.getLogger(__name__)
@@ -78,13 +81,13 @@ class InstallEnvironment(object):
"""
stdout_logger = logging.getLogger('install_environment.STDOUT')
stderr_logger = logging.getLogger('install_environment.STDERR')
for line in iter(stdout.readline, ''):
output = line.rstrip()
for line in iter(stdout.readline, b''):
output = unicodify(line).rstrip()
stdout_logger.debug(output)
stdout_queue.put(output)
stdout_queue.put(None)
for line in iter(stderr.readline, ''):
output = line.rstrip()
for line in iter(stderr.readline, b''):
output = unicodify(line).rstrip()
stderr_logger.debug(output)
stderr_queue.put(output)
stderr_queue.put(None)
@@ -245,19 +248,18 @@ class InstallEnvironment(object):
def log_results(self, command, fabric_AttributeString, file_path):
"""Write attributes of fabric.operations._AttributeString to a specified log file."""
if os.path.exists(file_path):
logfile = open(file_path, 'ab')
else:
logfile = open(file_path, 'wb')
logfile.write("\n#############################################\n")
logfile.write('%s\nSTDOUT\n' % command)
logfile.write(str(fabric_AttributeString.stdout))
logfile.write("\n#############################################\n")
logfile.write("\n#############################################\n")
logfile.write('%s\nSTDERR\n' % command)
logfile.write(str(fabric_AttributeString.stderr))
logfile.write("\n#############################################\n")
logfile.close()
mode = 'a' if os.path.exists(file_path) else 'w'
with open(file_path, mode) as logfile:
logfile.write("\n#############################################\n")
logfile.write(command)
logfile.write('\nSTDOUT\n')
logfile.write(fabric_AttributeString.stdout)
logfile.write("\n#############################################\n")
logfile.write("\n#############################################\n")
logfile.write(command)
logfile.write('\nSTDERR\n')
logfile.write(fabric_AttributeString.stderr)
logfile.write("\n#############################################\n")
@contextmanager
def use_tmp_dir(self):
lib/galaxy/tool_shed/util/hg_util.py
+2 -2
View File
@@ -77,7 +77,7 @@ def get_ctx_file_path_from_manifest(filename, repo, changeset_revision):
for changeset in reversed_upper_bounded_changelog(repo, changeset_revision):
manifest_ctx = repo[changeset]
for ctx_file in manifest_ctx.files():
ctx_file_name = basic_util.strip_path(ctx_file)
ctx_file_name = basic_util.strip_path(unicodify(ctx_file))
if ctx_file_name == stripped_filename:
return manifest_ctx, ctx_file
return None, None
@@ -93,7 +93,7 @@ def get_file_context_from_ctx(ctx, filename):
deleted = False
filename = basic_util.strip_path(filename)
for ctx_file in ctx.files():
ctx_file_name = basic_util.strip_path(ctx_file)
ctx_file_name = basic_util.strip_path(unicodify(ctx_file))
if filename == ctx_file_name:
try:
# If the file was moved, its destination will be returned here.
lib/galaxy/webapps/galaxy/controllers/user.py
+1
View File
@@ -237,6 +237,7 @@ class User(BaseUIController, UsesFormDefinitionsMixin, CreatesApiKeysMixin):
# Since logging an event requires a session, we'll log prior to ending the session
trans.log_event("User logged out")
trans.handle_user_logout(logout_all=logout_all)
return {"message": "Success."}
@expose_api_anonymous_and_sessionless
def create(self, trans, payload={}, **kwd):
lib/tool_shed/metadata/repository_metadata_manager.py
+1 -1
View File
@@ -752,7 +752,7 @@ class RepositoryMetadataManager(metadata_generator.MetadataGenerator):
log.debug("Cloning repository changeset revision: %s", str(ctx.rev()))
cloned_ok, error_message = hg_util.clone_repository(self.repository_clone_url, work_dir, str(ctx.rev()))
if cloned_ok:
log.debug("Generating metadata for changset revision: %s", str(ctx.rev()))
log.debug("Generating metadata for changeset revision: %s", str(ctx.rev()))
self.set_changeset_revision(str(ctx))
self.set_repository_files_dir(work_dir)
self.generate_metadata_for_changeset_revision()
lib/tool_shed/repository_types/repository_suite_definition.py
+2 -1
View File
@@ -1,6 +1,7 @@
import logging
import tool_shed.repository_types.util as rt_util
from galaxy.util import unicodify
from tool_shed.repository_types.metadata import TipOnly
from tool_shed.util import basic_util
@@ -31,7 +32,7 @@ class RepositorySuiteDefinition(TipOnly):
# is named repository_dependencies.xml.
files_changed_in_changeset = ctx.files()
for file_path in files_changed_in_changeset:
file_name = basic_util.strip_path(file_path)
file_name = basic_util.strip_path(unicodify(file_path))
if file_name not in self.valid_file_names:
return False
return True
lib/tool_shed/repository_types/tool_dependency_definition.py
+2 -1
View File
@@ -1,6 +1,7 @@
import logging
import tool_shed.repository_types.util as rt_util
from galaxy.util import unicodify
from tool_shed.repository_types.metadata import TipOnly
from tool_shed.util import basic_util
@@ -30,7 +31,7 @@ class ToolDependencyDefinition(TipOnly):
# Inspect all files in the changeset (in sorted order) to make sure there is only one and it is named tool_dependencies.xml.
files_changed_in_changeset = ctx.files()
for file_path in files_changed_in_changeset:
file_name = basic_util.strip_path(file_path)
file_name = basic_util.strip_path(unicodify(file_path))
if file_name not in self.valid_file_names:
return False
return True
lib/tool_shed/test/base/twilltestcase.py
+55 -237
View File
@@ -7,22 +7,19 @@ import string
import tempfile
import time
from json import loads
from xml.etree import ElementTree
# Be sure to use Galaxy's vanilla pyparsing instead of the older version
# imported by twill.
import pyparsing # noqa: F401
import requests
import twill
import twill.commands as tc
from mercurial import commands, hg, ui
from six import string_types, StringIO
from six.moves.urllib.parse import (
quote_plus,
urlencode,
urlparse
)
from twill.other_packages._mechanize_dist import ClientForm
from twill.utils import ResultWrapper
import galaxy.model.tool_shed_install as galaxy_model
import galaxy.util
@@ -40,14 +37,10 @@ from . import common, test_db_util
# Set a 10 minute timeout for repository installation.
repository_installation_timeout = 600
# Force twill to log to a buffer -- FIXME: Should this go to stdout and be captured by nose?
buffer = StringIO()
twill.set_output(buffer)
tc.config('use_tidy', 0)
# Dial ClientCookie logging down (very noisy)
logging.getLogger("ClientCookie.cookies").setLevel(logging.WARNING)
log = logging.getLogger(__name__)
tc.options['equiv_refresh_interval'] = 0
class ShedTwillTestCase(FunctionalTestCase):
@@ -127,111 +120,33 @@ class ShedTwillTestCase(FunctionalTestCase):
invalid_username = False
try:
self.check_page_for_string("Created new user account")
except Exception:
except AssertionError:
try:
# May have created the account in a previous test run...
self.check_page_for_string("User with email '%s' already exists." % email)
previously_created = True
except Exception:
except AssertionError:
try:
self.check_page_for_string('Public name is taken; please choose another')
username_taken = True
except Exception:
except AssertionError:
# Note that we're only checking if the usr name is >< 4 chars here...
try:
# Note that we're only checking if the usr name is >< 4 chars here...
self.check_page_for_string('Public name must be at least 4 characters in length')
invalid_username = True
except Exception:
except AssertionError:
pass
return previously_created, username_taken, invalid_username
def get_all_history_ids_from_api(self):
return [history['id'] for history in self.json_from_url('/api/histories')]
def get_form_controls(self, form):
formcontrols = []
for i, control in enumerate(form.controls):
formcontrols.append("control %d: %s" % (i, str(control)))
return formcontrols
def get_hids_in_history(self, history_id):
"""Returns the list of hid values for items in a history"""
hids = []
api_url = '/api/histories/%s/contents' % history_id
hids = [history_item['hid'] for history_item in self.json_from_url(api_url)]
return hids
def get_history_as_data_list(self, show_deleted=False):
"""Returns the data elements of a history"""
tree = self.history_as_xml_tree(show_deleted=show_deleted)
data_list = [elem for elem in tree.findall("data")]
return data_list
def get_history_from_api(self, encoded_history_id=None, show_deleted=None, show_details=False):
if encoded_history_id is None:
history = self.get_latest_history()
encoded_history_id = history['id']
params = dict()
if show_deleted is not None:
params['deleted'] = show_deleted
api_url = '/api/histories/%s/contents' % encoded_history_id
json_data = self.json_from_url(api_url, params=params)
if show_deleted is not None:
hdas = []
for hda in json_data:
if show_deleted:
hdas.append(hda)
else:
if not hda['deleted']:
hdas.append(hda)
json_data = hdas
if show_details:
params['details'] = ','.join(hda['id'] for hda in json_data)
api_url = '/api/histories/%s/contents' % encoded_history_id
json_data = self.json_from_url(api_url, params=params)
return json_data
def get_latest_history(self):
return self.json_from_url('/api/histories')[0]
def get_running_datasets(self):
self.visit_url('/api/histories')
history_id = loads(self.last_page())[0]['id']
self.visit_url('/api/histories/%s' % history_id)
jsondata = loads(self.last_page())
return jsondata['state'] in ['queued', 'running']
def history_as_xml_tree(self, show_deleted=False):
"""Returns a parsed xml object of a history"""
params = {
'as_xml': True,
'show_deleted': show_deleted
}
self.visit_url('/history', params=params)
xml = self.last_page()
tree = ElementTree.fromstring(xml)
return tree
def is_history_empty(self):
"""
Uses history page JSON to determine whether this history is empty
(i.e. has no undeleted datasets).
"""
return len(self.get_history_from_api()) == 0
def json_from_url(self, url, params={}):
self.visit_url(url, params)
return loads(self.last_page())
def last_page(self):
"""
Return the last visited page (usually HTML, but can binary data as
well).
"""
return tc.browser.get_html()
return tc.browser.html
def last_url(self):
return tc.browser.get_url()
return tc.browser.url
def login(self, email='test@bx.psu.edu', password='testuser', username='admin-user', redirect='', logout_first=True):
# Clear cookies.
@@ -251,137 +166,35 @@ class ShedTwillTestCase(FunctionalTestCase):
self.submit_form('login', 'login_button', login=email, redirect=redirect, password=password)
def logout(self):
self.visit_url("%s/user/logout" % self.url)
self.visit_url("/user/logout")
self.check_page_for_string("You have been logged out")
tc.browser.cj.clear()
def showforms(self):
"""Shows form, helpful for debugging new tests"""
return tc.showforms()
return tc.browser.forms
def submit_form(self, form_no=0, button="runtool_btn", **kwd):
def submit_form(self, form_no=0, button="runtool_btn", form=None, **kwd):
"""Populates and submits a form from the keyword arguments."""
# An HTMLForm contains a sequence of Controls. Supported control classes are:
# TextControl, FileControl, ListControl, RadioControl, CheckboxControl, SelectControl,
# SubmitControl, ImageControl
for i, f in enumerate(self.showforms()):
if i == form_no:
break
# To help with debugging a tool, print out the form controls when the test fails
print("form '%s' contains the following controls ( note the values )" % f.name)
controls = {}
formcontrols = self.get_form_controls(f)
hc_prefix = '<HiddenControl('
for i, control in enumerate(f.controls):
if hc_prefix not in str(control):
try:
# check if a repeat element needs to be added
if control.name is not None:
if control.name not in kwd and control.name.endswith('_add'):
# control name doesn't exist, could be repeat
repeat_startswith = control.name[0:-4]
if repeat_startswith and not [c_name for c_name in controls.keys() if c_name.startswith(repeat_startswith)] and [c_name for c_name in kwd.keys() if c_name.startswith(repeat_startswith)]:
tc.browser.clicked(f, control)
tc.submit(control.name)
return self.submit_form(form_no=form_no, button=button, **kwd)
# Check for refresh_on_change attribute, submit a change if required
if hasattr(control, 'attrs') and 'refresh_on_change' in control.attrs.keys():
changed = False
# For DataToolParameter, control.value is the HDA id, but kwd contains the filename.
# This loop gets the filename/label for the selected values.
item_labels = [item.attrs['label'] for item in control.get_items() if item.selected]
for value in kwd[control.name]:
if value not in control.value and True not in [value in item_label for item_label in item_labels]:
changed = True
break
if changed:
# Clear Control and set to proper value
control.clear()
# kwd[control.name] should be a singlelist
for elem in kwd[control.name]:
tc.fv(f.name, control.name, str(elem))
# Create a new submit control, allows form to refresh, instead of going to next page
control = ClientForm.SubmitControl('SubmitControl', '___refresh_grouping___', {'name': 'refresh_grouping'})
control.add_to_form(f)
control.fixup()
# Submit for refresh
tc.submit('___refresh_grouping___')
return self.submit_form(form_no=form_no, button=button, **kwd)
except Exception:
log.exception("In submit_form, continuing, but caught exception.")
for formcontrol in formcontrols:
log.debug(formcontrol)
continue
controls[control.name] = control
# No refresh_on_change attribute found in current form, so process as usual
if form is None:
for i, form in enumerate(self.showforms()):
if i == form_no:
break
assert form, "No form to submit found"
controls = {c.name: c for c in form.inputs}
form_name = form.get('name')
for control_name, control_value in kwd.items():
if control_name not in controls:
continue # these cannot be handled safely - cause the test to barf out
if not isinstance(control_value, list):
control_value = [control_value]
control_value = [str(control_value)]
control = controls[control_name]
control.clear()
if control.is_of_kind("text"):
tc.fv(f.name, control.name, ",".join(control_value))
elif control.is_of_kind("list"):
try:
if control.is_of_kind("multilist"):
if control.type == "checkbox":
def is_checked(value):
# Copied from form_builder.CheckboxField
if value is True:
return True
if isinstance(value, list):
value = value[0]
return isinstance(value, string_types) and value.lower() in ("yes", "true", "on")
try:
checkbox = control.get()
checkbox.selected = is_checked(control_value)
except Exception as e1:
print("Attempting to set checkbox selected value threw exception: ", e1)
# if there's more than one checkbox, probably should use the behaviour for
# ClientForm.ListControl ( see twill code ), but this works for now...
for elem in control_value:
control.get(name=elem).selected = True
else:
for elem in control_value:
try:
# Doubt this case would ever work, but want
# to preserve backward compat.
control.get(name=elem).selected = True
except Exception:
# ... anyway this is really what we want to
# do, probably even want to try the len(
# elem ) > 30 check below.
control.get(label=elem).selected = True
else: # control.is_of_kind( "singlelist" )
for elem in control_value:
try:
tc.fv(f.name, control.name, str(elem))
except Exception:
try:
# Galaxy truncates long file names in the dataset_collector in galaxy/tools/parameters/basic.py
if len(elem) > 30:
elem_name = '%s..%s' % (elem[:17], elem[-11:])
tc.fv(f.name, control.name, str(elem_name))
pass
else:
raise
except Exception:
raise
except Exception:
for formcontrol in formcontrols:
log.debug(formcontrol)
log.exception("Attempting to set control '%s' to value '%s' (also tried '%s') threw exception.", control.name, elem, elem_name)
pass
except Exception as exc:
for formcontrol in formcontrols:
log.debug(formcontrol)
errmsg = "Attempting to set field '%s' to value '%s' in form '%s' threw exception: %s\n" % (control_name, str(control_value), f.name, str(exc))
errmsg += "control: %s\n" % str(control)
errmsg += "If the above control is a DataToolparameter whose data type class does not include a sniff() method,\n"
errmsg += "make sure to include a proper 'ftype' attribute to the tag for the control within the <test> tag set.\n"
raise AssertionError(errmsg)
control_type = getattr(control, "type", None)
if control_type in ("text", "textfield", "submit", "password", "TextareaElement", "checkbox", "radio", None):
for cv in control_value:
tc.fv(form_name, control.name, cv)
else:
# Add conditions for other control types here when necessary.
pass
@@ -402,15 +215,11 @@ class ShedTwillTestCase(FunctionalTestCase):
if params:
url += '?%s' % urlencode(params, doseq=doseq)
new_url = tc.go(url)
return_code = tc.browser.get_code()
return_code = tc.browser.code
assert return_code in allowed_codes, 'Invalid HTTP return code %s, allowed codes: %s' % \
(return_code, ', '.join(str(code) for code in allowed_codes))
return new_url
def wait(self, **kwds):
"""Waits for the tools to finish"""
return self.wait_for(lambda: self.get_running_datasets(), **kwds)
def write_temp_file(self, content, suffix='.html'):
with tempfile.NamedTemporaryFile(suffix=suffix, prefix='twilltestcase-', delete=False) as fh:
fh.write(smart_str(content))
@@ -733,9 +542,6 @@ class ShedTwillTestCase(FunctionalTestCase):
if filename in filenames:
files_to_delete.append(os.path.join(basepath, filename))
self.browse_repository(repository)
# Twill sets hidden form fields to read-only by default. We need to write to this field.
form = tc.browser.get_form('select_files_to_delete')
form.find_control("selected_files_to_delete").readonly = False
tc.fv("2", "selected_files_to_delete", ','.join(files_to_delete))
tc.submit('select_files_to_delete_button')
self.check_for_strings(strings_displayed, strings_not_displayed)
@@ -944,7 +750,6 @@ class ShedTwillTestCase(FunctionalTestCase):
def galaxy_logout(self):
self.visit_galaxy_url("/user/logout", params=dict(session_csrf_token=self.galaxy_token()))
tc.browser.cj.clear()
def generate_complex_dependency_xml(self, filename, filepath, repository_tuples, package, version):
file_path = os.path.join(filepath, filename)
@@ -1024,7 +829,7 @@ class ShedTwillTestCase(FunctionalTestCase):
return os.path.abspath(os.path.join(self.file_dir, filename))
def get_hg_repo(self, path):
return hg.repository(ui.ui(), path)
return hg.repository(ui.ui(), path.encode('utf-8'))
def get_last_reviewed_revision_by_user(self, user, repository):
changelog_tuples = self.get_repository_changelog_tuples(repository)
@@ -1256,21 +1061,34 @@ class ShedTwillTestCase(FunctionalTestCase):
}
self.visit_url('/repository/install_repositories_by_revision', params=params)
self.check_for_strings(strings_displayed, strings_not_displayed)
# This section is tricky, due to the way twill handles form submission. The tool dependency checkbox needs to
# be hacked in through tc.browser, putting the form field in kwd doesn't work.
form = tc.browser.get_form('select_tool_panel_section')
form = tc.browser.form('select_tool_panel_section')
if form is None:
form = tc.browser.get_form('select_shed_tool_panel_config')
form = tc.browser.form('select_shed_tool_panel_config')
assert form is not None, 'Could not find form select_shed_tool_panel_config or select_tool_panel_section.'
kwd = self.set_form_value(form, kwd, 'install_tool_dependencies', install_tool_dependencies)
kwd = self.set_form_value(form, kwd, 'install_repository_dependencies', install_repository_dependencies)
kwd = self.set_form_value(form, kwd, 'install_resolver_dependencies', install_resolver_dependencies)
kwd = self.set_form_value(form, kwd, 'shed_tool_conf', self.shed_tool_conf)
kwds = {
'install_tool_dependencies': install_tool_dependencies,
'install_repository_dependencies': install_repository_dependencies,
'install_resolver_dependencies': install_resolver_dependencies,
'shed_tool_conf': self.shed_tool_conf,
}
if new_tool_panel_section_label is not None:
kwd = self.set_form_value(form, kwd, 'new_tool_panel_section_label', new_tool_panel_section_label)
submit_button_control = form.find_control(type='submit')
assert submit_button_control is not None, 'No submit button found for form %s.' % form.attrs.get('id')
self.submit_form(form.attrs.get('id'), str(submit_button_control.name), **kwd)
kwds['new_tool_panel_section_label'] = new_tool_panel_section_label
submit_button = [inp.name for inp in form.inputs if getattr(inp, 'type', None) == 'submit']
if len(submit_button) == 0:
# TODO: refactor, use regular TS install API
submit_kwargs = {inp.name: inp.value for inp in tc.browser.forms[0].inputs if getattr(inp, 'type', None) == 'submit'}
payload = {_: form.inputs[_].value for _ in form.fields.keys()}
payload.update(kwds)
payload.update(submit_kwargs)
r = tc.browser._session.post(
self.galaxy_url + form.action,
data=payload,
)
tc.browser.result = ResultWrapper(r)
else:
assert len(submit_button) == 1, 'Expected to find a single submit button, found %s (%s)' % (len(submit_button), ','.join(submit_button))
submit_button = submit_button[0]
self.submit_form(form=form, button=submit_button, **kwds)
self.check_for_strings(post_submit_strings_displayed, strings_not_displayed)
repository_ids = self.initiate_installation_process(new_tool_panel_section_label=new_tool_panel_section_label)
log.debug('Waiting for the installation of repository IDs: %s' % str(repository_ids))
@@ -1505,8 +1323,8 @@ class ShedTwillTestCase(FunctionalTestCase):
Set the form field field_name to field_value if it exists, and return the provided dict containing that value. If
the field does not exist in the provided form, return a dict without that index.
'''
form_id = form.attrs.get('id')
controls = [control for control in form.controls if str(control.name) == field_name]
form_id = form.attrib.get('id')
controls = [control for control in form.inputs if str(control.name) == field_name]
if len(controls) > 0:
log.debug('Setting field %s of form %s to %s.' % (field_name, form_id, str(field_value)))
tc.formvalue(form_id, field_name, str(field_value))
@@ -1535,9 +1353,9 @@ class ShedTwillTestCase(FunctionalTestCase):
if changeset_revision is None:
changeset_revision = self.get_repository_tip(repository)
self.display_manage_repository_page(repository, changeset_revision=changeset_revision)
form = tc.browser.get_form('skip_tool_tests')
form = tc.browser.form('skip_tool_tests')
assert form is not None, 'Could not find form skip_tool_tests.'
for control in form.controls:
for control in form.inputs:
control_name = str(control.name)
if control_name == 'skip_tool_tests' and control.type == 'checkbox':
checkbox = control.get()
lib/tool_shed/test/functional/test_1400_review_migration_stages.py
-41
View File
@@ -1,41 +0,0 @@
from ..base.twilltestcase import common, ShedTwillTestCase
class TestToolMigrationStages(ShedTwillTestCase):
'''Verify that the migration stages display correctly.'''
def test_0000_initiate_users(self):
"""Create necessary user accounts and login as an admin user."""
self.login(email=common.admin_email, username=common.admin_username)
admin_user = self.test_db_util.get_user(common.admin_email)
assert admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email
self.test_db_util.get_private_role(admin_user)
self.galaxy_login(email=common.admin_email, username=common.admin_username)
admin_user = self.test_db_util.get_user(common.admin_email)
assert admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email
self.test_db_util.get_private_role(admin_user)
def test_0005_load_migration_stages_page(self):
'''Load the migration page and check for the appropriate migration stages.'''
stages = []
migration_message_strings = ['The&nbsp;Emboss&nbsp;5.0.0&nbsp;tools&nbsp;have&nbsp;been&nbsp;eliminated',
'The&nbsp;freebayes&nbsp;tool&nbsp;has&nbsp;been&nbsp;eliminated',
'The&nbsp;NCBI&nbsp;BLAST+&nbsp;tools',
'The&nbsp;tools&nbsp;&#34;Map&nbsp;with&nbsp;BWA&nbsp;for&nbsp;Illumina&#34;',
'FASTQ&nbsp;to&nbsp;BAM,&nbsp;SAM&nbsp;to&nbsp;FASTQ,&nbsp;BAM&nbsp;',
'Map&nbsp;with&nbsp;Bowtie&nbsp;for&nbsp;Illumina,&nbsp;',
'BAM-to-SAM&nbsp;converts&nbsp;BAM&nbsp;format']
migrated_repository_names = ['emboss_5', 'emboss_datatypes', 'freebayes', 'ncbi_blast_plus',
'blast_datatypes', 'bwa_wrappers', 'picard', 'lastz',
'lastz_paired_reads', 'bowtie_color_wrappers', 'bowtie_wrappers',
'xy_plot', 'bam_to_sam']
migrated_tool_dependencies = ['emboss', '5.0.0', 'freebayes', '0.9.4_a46483351fd0196637614121868fb5c386612b55',
'samtools', '0.1.18', 'blast+', '2.2.26+', 'bwa', '0.5.9', 'picard', '1.56.0',
'lastz', '1.02.00', 'bowtie', '0.12.7', 'FreeBayes requires g++', 'ncurses', 'zlib',
'blast.ncbi.nlm.nih.gov', 'fastx_toolkit', '0.0.13', 'samtools', '0.1.16', 'cufflinks',
'2.1.1', 'R', '2.11.0']
migration_scripts = ['0002_tools.sh', '0003_tools.sh', '0004_tools.sh', '0005_tools.sh', '0006_tools.sh',
'0007_tools.sh', '0008_tools.sh']
stages.extend(migration_scripts + migrated_tool_dependencies + migrated_repository_names)
stages.extend(migration_message_strings)
self.load_galaxy_tool_migrations_page(strings_displayed=stages)
lib/tool_shed/test/functional_tests.py
+12 -6
View File
@@ -95,9 +95,11 @@ class ToolShedTestDriver(driver_util.TestDriver):
use_heartbeat=False)
kwargs.update(toolshed_database_conf)
# Generate the tool_data_table_conf.xml file.
open(default_tool_data_table_config_path, 'w').write(tool_data_table_conf_xml_template)
with open(default_tool_data_table_config_path, 'w') as fh:
fh.write(tool_data_table_conf_xml_template)
# Generate the shed_tool_data_table_conf.xml file.
open(shed_tool_data_table_conf_file, 'w').write(tool_data_table_conf_xml_template)
with open(shed_tool_data_table_conf_file, 'w') as fh:
fh.write(tool_data_table_conf_xml_template)
os.environ['TOOL_SHED_TEST_TOOL_DATA_TABLE_CONF'] = shed_tool_data_table_conf_file
# ---- Build Tool Shed Application --------------------------------------------------
toolshedapp = driver_util.build_shed_app(kwargs)
@@ -121,18 +123,22 @@ class ToolShedTestDriver(driver_util.TestDriver):
# Generate the shed_tool_conf.xml file.
tool_sheds_conf_template_parser = string.Template(tool_sheds_conf_xml_template)
tool_sheds_conf_xml = tool_sheds_conf_template_parser.safe_substitute(shed_url=tool_shed_test_host, shed_port=tool_shed_test_port)
open(galaxy_tool_sheds_conf_file, 'w').write(tool_sheds_conf_xml)
with open(galaxy_tool_sheds_conf_file, 'w') as fh:
fh.write(tool_sheds_conf_xml)
# Generate the tool_sheds_conf.xml file.
shed_tool_conf_template_parser = string.Template(shed_tool_conf_xml_template)
shed_tool_conf_xml = shed_tool_conf_template_parser.safe_substitute(shed_tool_path=galaxy_shed_tool_path)
open(galaxy_shed_tool_conf_file, 'w').write(shed_tool_conf_xml)
with open(galaxy_shed_tool_conf_file, 'w') as fh:
fh.write(shed_tool_conf_xml)
# Generate the migrated_tool_conf.xml file.
migrated_tool_conf_xml = shed_tool_conf_template_parser.safe_substitute(shed_tool_path=galaxy_migrated_tool_path)
open(galaxy_migrated_tool_conf_file, 'w').write(migrated_tool_conf_xml)
with open(galaxy_migrated_tool_conf_file, 'w') as fh:
fh.write(migrated_tool_conf_xml)
os.environ['GALAXY_TEST_SHED_TOOL_CONF'] = galaxy_shed_tool_conf_file
# Generate shed_data_manager_conf.xml
if not os.environ.get('GALAXY_SHED_DATA_MANAGER_CONF'):
open(galaxy_shed_data_manager_conf_file, 'wb').write(shed_data_manager_conf_xml_template)
with open(galaxy_shed_data_manager_conf_file, 'w') as fh:
fh.write(shed_data_manager_conf_xml_template)
kwargs = dict(migrated_tools_config=galaxy_migrated_tool_conf_file,
shed_data_manager_config_file=galaxy_shed_data_manager_conf_file,
shed_tool_path=galaxy_shed_tool_path,
lib/tool_shed/test/test_data/filtering/readme.txt
+2 -2
View File
@@ -1,2 +1,2 @@
These characters should not result in a unicode decoding error: ®Ãâ�âèÃ
....
These characters should not result in a unicode decoding error: cómplǐcḁtëd strĩñg
....
lib/tool_shed/tools/tool_validator.py
+3 -2
View File
@@ -5,6 +5,7 @@ import tempfile
from galaxy.tool_shed.tools.tool_validator import ToolValidator as GalaxyToolValidator
from galaxy.tools import Tool
from galaxy.util import unicodify
from tool_shed.util import (
basic_util,
hg_util,
@@ -79,7 +80,7 @@ class ToolValidator(GalaxyToolValidator):
for changeset in hg_util.reversed_upper_bounded_changelog(repo, changeset_revision):
manifest_ctx = repo[changeset]
for ctx_file in manifest_ctx.files():
ctx_file_name = basic_util.strip_path(ctx_file)
ctx_file_name = basic_util.strip_path(unicodify(ctx_file))
if ctx_file_name == stripped_filename:
try:
fctx = manifest_ctx[ctx_file]
@@ -111,7 +112,7 @@ class ToolValidator(GalaxyToolValidator):
for changeset in hg_util.reversed_upper_bounded_changelog(repo, changeset_revision):
changeset_ctx = repo[changeset]
for ctx_file in changeset_ctx.files():
ctx_file_name = basic_util.strip_path(ctx_file)
ctx_file_name = basic_util.strip_path(unicodify(ctx_file))
# If we decide in the future that files deleted later in the changelog should
# not be used, we can use the following if statement. if ctx_file_name.endswith( '.sample' )
# and ctx_file_name not in sample_files and ctx_file_name not in deleted_sample_files:
lib/tool_shed/util/hg_util.py
+1 -1
View File
@@ -92,7 +92,7 @@ def get_named_tmpfile_from_ctx(ctx, filename, dir):
"""
filename = basic_util.strip_path(filename)
for ctx_file in ctx.files():
ctx_file_name = basic_util.strip_path(ctx_file)
ctx_file_name = basic_util.strip_path(unicodify(ctx_file))
if filename == ctx_file_name:
try:
# If the file was moved, its destination file contents will be returned here.
lib/tool_shed/util/hgweb_config.py
+4 -6
View File
@@ -80,9 +80,8 @@ class HgWebConfigManager(object):
# raise Exception( "Required file %s does not exist - check config setting for hgweb_config_dir." % hgweb_config )
# ...but now we just log the missing file and create a new empty one.
log.debug("Required file %s does not exist, so creating a new, empty file. Check your config setting for hgweb_config_dir." % hgweb_config)
hgweb_config_file = open(hgweb_config, 'wb')
hgweb_config_file.write(new_hgweb_config_template)
hgweb_config_file.close()
with open(hgweb_config, 'w') as hgweb_config_file:
hgweb_config_file.write(new_hgweb_config_template)
return os.path.abspath(hgweb_config)
def make_backup(self):
@@ -101,9 +100,8 @@ class HgWebConfigManager(object):
def write_config(self):
"""Writing the in-memory configuration to the hgweb.config file on disk."""
config_file = open(self.hgweb_config, 'wb')
self.in_memory_config.write(config_file)
config_file.close
with open(self.hgweb_config, 'w') as config_file:
self.in_memory_config.write(config_file)
hgweb_config_manager = HgWebConfigManager()
lib/tool_shed/util/readme_util.py
+15 -9
View File
@@ -1,3 +1,4 @@
import io
import json
import logging
import os
@@ -6,12 +7,18 @@ from mako.template import Template
import tool_shed.util.shed_util_common as suc
from galaxy import web
from galaxy.util import rst_to_html, unicodify, url_get
from tool_shed.util import basic_util
from tool_shed.util import common_util
from tool_shed.util import hg_util
from tool_shed.util import metadata_util
from tool_shed.util import repository_util
from galaxy.util import (
rst_to_html,
unicodify,
url_get,
)
from tool_shed.util import (
basic_util,
common_util,
hg_util,
metadata_util,
repository_util,
)
log = logging.getLogger(__name__)
@@ -39,9 +46,8 @@ def build_readme_files_dict(app, repository, changeset_revision, metadata, tool_
full_path_to_readme_file = os.path.abspath(relative_path_to_readme_file)
text = None
try:
f = open(full_path_to_readme_file, 'r')
text = unicodify(f.read())
f.close()
with io.open(full_path_to_readme_file, 'r', encoding='utf-8') as f:
text = f.read()
except Exception:
log.exception("Error reading README file '%s' from disk", relative_path_to_readme_file)
text = None
lib/tool_shed/util/repository_util.py
+1 -1
View File
@@ -406,7 +406,7 @@ def change_repository_name_in_hgrc_file(hgrc_file, new_name):
config = configparser.ConfigParser()
config.read(hgrc_file)
config.set('web', 'name', new_name)
with open(hgrc_file, 'wb') as fh:
with open(hgrc_file, 'w') as fh:
config.write(fh)
lib/tool_shed/webapp/controllers/hg.py
+1 -1
View File
@@ -19,7 +19,7 @@ class HgController(BaseUIController):
cmd = kwd.get('cmd', None)
def make_web_app():
hgwebapp = hgwebdir(hgweb_config)
hgwebapp = hgwebdir(hgweb_config.encode('utf-8'))
return hgwebapp
wsgi_app = wsgiapplication(make_web_app)
if cmd == 'getbundle':
lib/tool_shed/webapp/controllers/repository.py
+4 -2
View File
@@ -61,7 +61,7 @@ def get_mercurial_default_options_dict(command):
possible = possible[0]
if len(possible) != 1:
raise Exception('unable to find mercurial command "%s"' % command)
return dict((r[1].replace('-', '_'), r[2]) for r in next(iter(possible.values()))[1][1])
return dict((r[1].replace(b'-', b'_'), r[2]) for r in next(iter(possible.values()))[1][1])
class RepositoryController(BaseUIController, ratings_util.ItemRatings):
@@ -2480,9 +2480,10 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
else:
ctx_child = None
diffs = []
options_dict = get_mercurial_default_options_dict('diff')
options_dict = get_mercurial_default_options_dict(b"diff")
# Not quite sure if the following settings make any difference, but with a combination of them and the size check on each
# diff, we don't run out of memory when viewing the changelog of the cisortho2 repository on the test tool shed.
options_dict = {util.unicodify(k): util.unicodify(v) for k, v in options_dict.items()}
options_dict['maxfile'] = basic_util.MAXDIFFSIZE
options_dict['maxtotal'] = basic_util.MAXDIFFSIZE
diffopts = mdiff.diffopts(**options_dict)
@@ -2492,6 +2493,7 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
diffs.append(basic_util.to_html_string(diff))
modified, added, removed, deleted, unknown, ignored, clean = repo.status(node1=ctx_parent.node(), node2=ctx.node())
anchors = modified + added + removed + deleted + unknown + ignored + clean
anchors = util.unicodify(anchors)
metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision(trans.app,
id,
ctx_str,
lib/tool_shed/webapp/model/__init__.py
+8 -3
View File
@@ -217,7 +217,12 @@ class Repository(Dictifiable):
(str(self.name), str(self.user.username)))
def allow_push(self):
return self.hg_repo.ui.config('web', 'allow_push')
hgrc_file = hg_util.get_hgrc_path(self.repo_path())
with open(hgrc_file, 'r') as fh:
for line in fh.read().splitlines():
if line.startswith('allow_push = '):
return line[len('allow_push = '):]
return ''
def can_change_type(self):
# Allow changing the type only if the repository has no contents, has never been installed, or has
@@ -300,9 +305,9 @@ class Repository(Dictifiable):
# repo.ui.setconfig('web', 'allow_push', allow_push)
repo_dir = self.repo_path()
hgrc_file = hg_util.get_hgrc_path(repo_dir)
with open(hgrc_file, 'rb') as fh:
with open(hgrc_file, 'r') as fh:
lines = fh.readlines()
with open(hgrc_file, 'wb') as fh:
with open(hgrc_file, 'w') as fh:
for line in lines:
if line.startswith('allow_push'):
fh.write('allow_push = %s' % allow_push)
lib/tool_shed/webapp/model/migrate/check.py
+9
View File
@@ -5,6 +5,10 @@ import sys
from migrate.versioning import repository, schema
from sqlalchemy import create_engine, MetaData, Table
from sqlalchemy.exc import NoSuchTableError
from sqlalchemy_utils import (
create_database,
database_exists,
)
log = logging.getLogger(__name__)
@@ -26,6 +30,11 @@ def create_or_verify_database(url, engine_options={}):
"""
# Create engine and metadata
if not database_exists(url):
message = "Creating database for URI [%s]" % url
log.info(message)
create_database(url)
engine = create_engine(url, **engine_options)
meta = MetaData(bind=engine)
# Try to load dataset table
lib/tool_shed/webapp/templates/webapps/tool_shed/common/common.mako
+1
View File
@@ -109,6 +109,7 @@
else:
disabled_str = ''
html = ''
rating = rating or 0
for index in range( 1, 6 ):
html += '<input name="%s" type="radio" class="star" value="%s" %s' % ( str( name ), str( index ), disabled_str )
if rating > ( index - 0.5 ) and rating < ( index + 0.5 ):