From 2573f023c8b899bb994bab4cb669423113e1a0f4 Mon Sep 17 00:00:00 2001 From: Dave B Date: Fri, 8 Jan 2016 12:51:41 -0500 Subject: [PATCH 1/5] Some tweaks to shed_util_common. --- .../tool_shed/controllers/repository.py | 8 +++--- ...check_repositories_for_functional_tests.py | 2 +- lib/tool_shed/util/metadata_util.py | 4 +-- lib/tool_shed/util/shed_util_common.py | 28 ++++++++++--------- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py index 2a0ab0a283a..12a4ee7d701 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -2946,13 +2946,13 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): else: tool_shed_status_dict[ 'revision_update' ] = 'False' # Handle revision upgrades. - ordered_metadata_changeset_revisions = suc.get_ordered_metadata_changeset_revisions( repository, repo, downloadable=True ) - num_metadata_revisions = len( ordered_metadata_changeset_revisions ) - for index, metadata_changeset_revision in enumerate( ordered_metadata_changeset_revisions ): + metadata_revisions = suc.get_metadata_revisions( repository, repo, include_numeric=False, downloadable=True ) + num_metadata_revisions = len( metadata_revisions ) + for index, metadata_revision in enumerate( metadata_revisions ): if index == num_metadata_revisions: tool_shed_status_dict[ 'revision_upgrade' ] = 'False' break - if metadata_changeset_revision == changeset_revision: + if metadata_revision == changeset_revision: if num_metadata_revisions - index > 1: tool_shed_status_dict[ 'revision_upgrade' ] = 'True' else: diff --git a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py index 53f968badc2..62b6f9733f4 100644 --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -361,7 +361,7 @@ def should_set_do_not_test_flag( app, repository, changeset_revision, testable_r """ if not testable_revision: repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) - changeset_revisions = suc.get_ordered_metadata_changeset_revisions( repository, repo, downloadable=True ) + changeset_revisions = suc.get_metadata_revisions( repository, repo, include_numeric=False ) if len( changeset_revisions ) > 1: latest_downloadable_revision = changeset_revisions[ -1 ] if changeset_revision != latest_downloadable_revision: diff --git a/lib/tool_shed/util/metadata_util.py b/lib/tool_shed/util/metadata_util.py index a670f4a1c01..0a456700e12 100644 --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -12,7 +12,7 @@ def get_latest_changeset_revision( app, repository, repo ): repository_tip ) if repository_metadata and repository_metadata.downloadable: return repository_tip - changeset_revisions = suc.get_ordered_metadata_changeset_revisions( repository, repo, downloadable=False ) + changeset_revisions = suc.get_metadata_revisions( repository, repo, include_numeric=False, downloadable=False ) if changeset_revisions: return changeset_revisions[ -1 ] return hg_util.INITIAL_CHANGELOG_HASH @@ -38,7 +38,7 @@ def get_previous_metadata_changeset_revision( repository, repo, before_changeset the changeset to which before_changeset_revision refers. If there isn't one, return the hash value of an empty repository changelog, hg_util.INITIAL_CHANGELOG_HASH. """ - changeset_revisions = suc.get_ordered_metadata_changeset_revisions( repository, repo, downloadable=downloadable ) + changeset_revisions = suc.get_metadata_revisions( repository, repo, include_numeric=False, downloadable=downloadable ) if len( changeset_revisions ) == 1: changeset_revision = changeset_revisions[ 0 ] if changeset_revision == before_changeset_revision: diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index 82256715c98..3a6cfaf6926 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -6,6 +6,7 @@ import shutil import socket import string from urllib2 import HTTPError +from operator import itemgetter import sqlalchemy.orm.exc from sqlalchemy import and_, false, or_, true @@ -345,7 +346,7 @@ def get_latest_downloadable_changeset_revision( app, repository, repo ): repository_metadata = get_repository_metadata_by_changeset_revision( app, app.security.encode_id( repository.id ), repository_tip ) if repository_metadata and repository_metadata.downloadable: return repository_tip - changeset_revisions = get_ordered_metadata_changeset_revisions( repository, repo, downloadable=True ) + changeset_revisions = get_metadata_revisions( repository, repo, include_numeric=False, downloadable=True ) if changeset_revisions: return changeset_revisions[ -1 ] return hg_util.INITIAL_CHANGELOG_HASH @@ -369,7 +370,7 @@ def get_next_downloadable_changeset_revision( repository, repo, after_changeset_ Return the installable changeset_revision in the repository changelog after the changeset to which after_changeset_revision refers. If there isn't one, return None. """ - changeset_revisions = get_ordered_metadata_changeset_revisions( repository, repo, downloadable=True ) + changeset_revisions = get_metadata_revisions( repository, repo, include_numeric=False, downloadable=True ) if len( changeset_revisions ) == 1: changeset_revision = changeset_revisions[ 0 ] if changeset_revision == after_changeset_revision: @@ -420,10 +421,9 @@ def get_next_prior_import_or_install_required_dict_entry( prior_required_dict, p return key -def get_ordered_metadata_changeset_revisions( repository, repo, downloadable=True ): +def get_metadata_revisions( repository, repo, sort_revisions=True, reverse=False, include_numeric=True, downloadable=True ): """ - Return an ordered list of changeset_revisions that are associated with metadata - where order is defined by the repository changelog. + Return a list of changesets for the provided repository. """ if downloadable: metadata_revisions = repository.downloadable_revisions @@ -431,16 +431,18 @@ def get_ordered_metadata_changeset_revisions( repository, repo, downloadable=Tru metadata_revisions = repository.metadata_revisions changeset_tups = [] for repository_metadata in metadata_revisions: - changeset_revision = repository_metadata.changeset_revision - ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) + ctx = hg_util.get_changectx_for_changeset( repo, repository_metadata.changeset_revision ) if ctx: - rev = '%04d' % ctx.rev() + rev = ctx.rev() else: - rev = '-1' - changeset_tups.append( ( rev, changeset_revision ) ) - sorted_changeset_tups = sorted( changeset_tups ) - sorted_changeset_revisions = [ str( changeset_tup[ 1 ] ) for changeset_tup in sorted_changeset_tups ] - return sorted_changeset_revisions + rev = -1 + changeset_tups.append( ( rev, repository_metadata.changeset_revision ) ) + if sort_revisions: + changeset_tups = sorted( changeset_tups, key=itemgetter( 0 ), reverse=reverse ) + if not include_numeric: + retval = [ str( changeset[ 1 ] ) for changeset in changeset_tups ] + changeset_tups = retval + return changeset_tups def get_prior_import_or_install_required_dict( app, tsr_ids, repo_info_dicts ): From d38e7a61a1b115099804647d256d67064af99ecb Mon Sep 17 00:00:00 2001 From: Dave B Date: Fri, 8 Jan 2016 13:15:03 -0500 Subject: [PATCH 2/5] Use built-in sort method. --- lib/tool_shed/util/shed_util_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index 3a6cfaf6926..530a7bd61bd 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -438,7 +438,7 @@ def get_metadata_revisions( repository, repo, sort_revisions=True, reverse=False rev = -1 changeset_tups.append( ( rev, repository_metadata.changeset_revision ) ) if sort_revisions: - changeset_tups = sorted( changeset_tups, key=itemgetter( 0 ), reverse=reverse ) + changeset_tups.sort( key=itemgetter( 0 ), reverse=reverse ) if not include_numeric: retval = [ str( changeset[ 1 ] ) for changeset in changeset_tups ] changeset_tups = retval From 72540179262d53e5080385fc3cab3243d984c0c1 Mon Sep 17 00:00:00 2001 From: Dave B Date: Fri, 8 Jan 2016 16:28:55 -0500 Subject: [PATCH 3/5] Correct default behavior. --- .../webapps/tool_shed/controllers/repository.py | 2 +- .../check_repositories_for_functional_tests.py | 2 +- lib/tool_shed/util/metadata_util.py | 4 ++-- lib/tool_shed/util/shed_util_common.py | 12 ++++++------ 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py index 12a4ee7d701..29e02747ec5 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -2946,7 +2946,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): else: tool_shed_status_dict[ 'revision_update' ] = 'False' # Handle revision upgrades. - metadata_revisions = suc.get_metadata_revisions( repository, repo, include_numeric=False, downloadable=True ) + metadata_revisions = suc.get_metadata_revisions( repository, repo, include_hash=False, downloadable=True ) num_metadata_revisions = len( metadata_revisions ) for index, metadata_revision in enumerate( metadata_revisions ): if index == num_metadata_revisions: diff --git a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py index 62b6f9733f4..379c49b2159 100644 --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -361,7 +361,7 @@ def should_set_do_not_test_flag( app, repository, changeset_revision, testable_r """ if not testable_revision: repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) - changeset_revisions = suc.get_metadata_revisions( repository, repo, include_numeric=False ) + changeset_revisions = suc.get_metadata_revisions( repository, repo, include_hash=False ) if len( changeset_revisions ) > 1: latest_downloadable_revision = changeset_revisions[ -1 ] if changeset_revision != latest_downloadable_revision: diff --git a/lib/tool_shed/util/metadata_util.py b/lib/tool_shed/util/metadata_util.py index 0a456700e12..ce5dabc37ab 100644 --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -12,7 +12,7 @@ def get_latest_changeset_revision( app, repository, repo ): repository_tip ) if repository_metadata and repository_metadata.downloadable: return repository_tip - changeset_revisions = suc.get_metadata_revisions( repository, repo, include_numeric=False, downloadable=False ) + changeset_revisions = suc.get_metadata_revisions( repository, repo, include_hash=False, downloadable=False ) if changeset_revisions: return changeset_revisions[ -1 ] return hg_util.INITIAL_CHANGELOG_HASH @@ -38,7 +38,7 @@ def get_previous_metadata_changeset_revision( repository, repo, before_changeset the changeset to which before_changeset_revision refers. If there isn't one, return the hash value of an empty repository changelog, hg_util.INITIAL_CHANGELOG_HASH. """ - changeset_revisions = suc.get_metadata_revisions( repository, repo, include_numeric=False, downloadable=downloadable ) + changeset_revisions = suc.get_metadata_revisions( repository, repo, include_hash=False, downloadable=downloadable ) if len( changeset_revisions ) == 1: changeset_revision = changeset_revisions[ 0 ] if changeset_revision == before_changeset_revision: diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index 530a7bd61bd..ea94379b7af 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -346,7 +346,7 @@ def get_latest_downloadable_changeset_revision( app, repository, repo ): repository_metadata = get_repository_metadata_by_changeset_revision( app, app.security.encode_id( repository.id ), repository_tip ) if repository_metadata and repository_metadata.downloadable: return repository_tip - changeset_revisions = get_metadata_revisions( repository, repo, include_numeric=False, downloadable=True ) + changeset_revisions = get_metadata_revisions( repository, repo, include_hash=False, downloadable=True ) if changeset_revisions: return changeset_revisions[ -1 ] return hg_util.INITIAL_CHANGELOG_HASH @@ -370,7 +370,7 @@ def get_next_downloadable_changeset_revision( repository, repo, after_changeset_ Return the installable changeset_revision in the repository changelog after the changeset to which after_changeset_revision refers. If there isn't one, return None. """ - changeset_revisions = get_metadata_revisions( repository, repo, include_numeric=False, downloadable=True ) + changeset_revisions = get_metadata_revisions( repository, repo, include_hash=False, downloadable=True ) if len( changeset_revisions ) == 1: changeset_revision = changeset_revisions[ 0 ] if changeset_revision == after_changeset_revision: @@ -421,7 +421,7 @@ def get_next_prior_import_or_install_required_dict_entry( prior_required_dict, p return key -def get_metadata_revisions( repository, repo, sort_revisions=True, reverse=False, include_numeric=True, downloadable=True ): +def get_metadata_revisions( repository, repo, sort_revisions=True, reverse=False, include_hash=False, downloadable=True ): """ Return a list of changesets for the provided repository. """ @@ -433,14 +433,14 @@ def get_metadata_revisions( repository, repo, sort_revisions=True, reverse=False for repository_metadata in metadata_revisions: ctx = hg_util.get_changectx_for_changeset( repo, repository_metadata.changeset_revision ) if ctx: - rev = ctx.rev() + rev = '%04d' % str( ctx.rev() ) else: rev = -1 changeset_tups.append( ( rev, repository_metadata.changeset_revision ) ) if sort_revisions: changeset_tups.sort( key=itemgetter( 0 ), reverse=reverse ) - if not include_numeric: - retval = [ str( changeset[ 1 ] ) for changeset in changeset_tups ] + if not include_hash: + retval = [ str( changeset[ 0 ] ) for changeset in changeset_tups ] changeset_tups = retval return changeset_tups From 508c42536d2fcd4ecd5abc2ff828675b7ccdfbdc Mon Sep 17 00:00:00 2001 From: Dave B Date: Mon, 11 Jan 2016 09:22:17 -0500 Subject: [PATCH 4/5] Do not cast to string when formatting expects int. --- lib/tool_shed/util/shed_util_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index ea94379b7af..3af66278278 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -433,7 +433,7 @@ def get_metadata_revisions( repository, repo, sort_revisions=True, reverse=False for repository_metadata in metadata_revisions: ctx = hg_util.get_changectx_for_changeset( repo, repository_metadata.changeset_revision ) if ctx: - rev = '%04d' % str( ctx.rev() ) + rev = '%04d' % ctx.rev() else: rev = -1 changeset_tups.append( ( rev, repository_metadata.changeset_revision ) ) From 0bdc0a48e9d94eb0087490b92d2b53693abe1171 Mon Sep 17 00:00:00 2001 From: Dave B Date: Mon, 11 Jan 2016 12:20:26 -0500 Subject: [PATCH 5/5] Fix failing tests. --- lib/galaxy/webapps/tool_shed/controllers/repository.py | 2 +- .../scripts/check_repositories_for_functional_tests.py | 2 +- lib/tool_shed/util/metadata_util.py | 4 ++-- lib/tool_shed/util/shed_util_common.py | 9 +++------ 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py index 29e02747ec5..0e55d603792 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -2946,7 +2946,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): else: tool_shed_status_dict[ 'revision_update' ] = 'False' # Handle revision upgrades. - metadata_revisions = suc.get_metadata_revisions( repository, repo, include_hash=False, downloadable=True ) + metadata_revisions = [ revision[ 1 ] for revision in suc.get_metadata_revisions( repository, repo ) ] num_metadata_revisions = len( metadata_revisions ) for index, metadata_revision in enumerate( metadata_revisions ): if index == num_metadata_revisions: diff --git a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py index 379c49b2159..5759847d290 100644 --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -361,7 +361,7 @@ def should_set_do_not_test_flag( app, repository, changeset_revision, testable_r """ if not testable_revision: repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) - changeset_revisions = suc.get_metadata_revisions( repository, repo, include_hash=False ) + changeset_revisions = [ revision[ 1 ] for revision in suc.get_metadata_revisions( repository, repo ) ] if len( changeset_revisions ) > 1: latest_downloadable_revision = changeset_revisions[ -1 ] if changeset_revision != latest_downloadable_revision: diff --git a/lib/tool_shed/util/metadata_util.py b/lib/tool_shed/util/metadata_util.py index ce5dabc37ab..422fd4e667a 100644 --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -12,7 +12,7 @@ def get_latest_changeset_revision( app, repository, repo ): repository_tip ) if repository_metadata and repository_metadata.downloadable: return repository_tip - changeset_revisions = suc.get_metadata_revisions( repository, repo, include_hash=False, downloadable=False ) + changeset_revisions = [ revision[ 1 ] for revision in suc.get_metadata_revisions( repository, repo ) ] if changeset_revisions: return changeset_revisions[ -1 ] return hg_util.INITIAL_CHANGELOG_HASH @@ -38,7 +38,7 @@ def get_previous_metadata_changeset_revision( repository, repo, before_changeset the changeset to which before_changeset_revision refers. If there isn't one, return the hash value of an empty repository changelog, hg_util.INITIAL_CHANGELOG_HASH. """ - changeset_revisions = suc.get_metadata_revisions( repository, repo, include_hash=False, downloadable=downloadable ) + changeset_revisions = [ revision[ 1 ] for revision in suc.get_metadata_revisions( repository, repo ) ] if len( changeset_revisions ) == 1: changeset_revision = changeset_revisions[ 0 ] if changeset_revision == before_changeset_revision: diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index 3af66278278..2d93290529a 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -346,7 +346,7 @@ def get_latest_downloadable_changeset_revision( app, repository, repo ): repository_metadata = get_repository_metadata_by_changeset_revision( app, app.security.encode_id( repository.id ), repository_tip ) if repository_metadata and repository_metadata.downloadable: return repository_tip - changeset_revisions = get_metadata_revisions( repository, repo, include_hash=False, downloadable=True ) + changeset_revisions = [ revision[ 1 ] for revision in get_metadata_revisions( repository, repo ) ] if changeset_revisions: return changeset_revisions[ -1 ] return hg_util.INITIAL_CHANGELOG_HASH @@ -370,7 +370,7 @@ def get_next_downloadable_changeset_revision( repository, repo, after_changeset_ Return the installable changeset_revision in the repository changelog after the changeset to which after_changeset_revision refers. If there isn't one, return None. """ - changeset_revisions = get_metadata_revisions( repository, repo, include_hash=False, downloadable=True ) + changeset_revisions = [ revision[ 1 ] for revision in get_metadata_revisions( repository, repo ) ] if len( changeset_revisions ) == 1: changeset_revision = changeset_revisions[ 0 ] if changeset_revision == after_changeset_revision: @@ -421,7 +421,7 @@ def get_next_prior_import_or_install_required_dict_entry( prior_required_dict, p return key -def get_metadata_revisions( repository, repo, sort_revisions=True, reverse=False, include_hash=False, downloadable=True ): +def get_metadata_revisions( repository, repo, sort_revisions=True, reverse=False, downloadable=True ): """ Return a list of changesets for the provided repository. """ @@ -439,9 +439,6 @@ def get_metadata_revisions( repository, repo, sort_revisions=True, reverse=False changeset_tups.append( ( rev, repository_metadata.changeset_revision ) ) if sort_revisions: changeset_tups.sort( key=itemgetter( 0 ), reverse=reverse ) - if not include_hash: - retval = [ str( changeset[ 0 ] ) for changeset in changeset_tups ] - changeset_tups = retval return changeset_tups