From d0f6c6dcf30187c79c31928162eef76dc740cd54 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 25 Aug 2014 12:36:34 -0400 Subject: [PATCH 1/5] Bugfix for 0750859: Don't generate duplicate workflow slugs for private workflows. See full conversation on https://bitbucket.org/galaxy/galaxy-central/commits/07508592cb65845474b9f3e0f781260c. Would like to better and allow slugs of deleted items to be reused - they cannot be for workflows for instance. Because the slugs are not really handled uniformly though - pages do not have this problem and deleted pages can have their slugs reused because this is included in a check in the web controller which directly assigns valid slugs at that point. Slugs for deleted histories cannot be re-used - but they couldn't be prior to this commit - so this isn't making things worse in that respect. --- lib/galaxy/web/base/controller.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/web/base/controller.py b/lib/galaxy/web/base/controller.py index 2c15771f000..eab3875fc99 100644 --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -27,7 +27,7 @@ from galaxy.web import error, url_for from galaxy.web.form_builder import AddressField, CheckboxField, SelectField, TextArea, TextField from galaxy.web.form_builder import build_select_field, HistoryField, PasswordField, WorkflowField, WorkflowMappingField from galaxy.workflow.modules import module_factory -from galaxy.model.orm import eagerload, eagerload_all, desc +from galaxy.model.orm import eagerload, eagerload_all, desc, not_ from galaxy.security.validate_user_input import validate_publicname from galaxy.util.sanitize_html import sanitize_html from galaxy.model.item_attrs import Dictifiable, UsesAnnotations @@ -2694,7 +2694,7 @@ class SharableMixin: item = self.get_item( trans, id ) if item: # Only update slug if slug is not already in use. - if trans.sa_session.query( item.__class__ ).filter_by( user=item.user, slug=new_slug, importable=True ).count() == 0: + if trans.sa_session.query( item.__class__ ).filter_by( user=item.user, slug=new_slug ).count() == 0: item.slug = new_slug trans.sa_session.flush() @@ -2729,7 +2729,9 @@ class SharableMixin: # add integer to end. new_slug = slug_base count = 1 - while sa_session.query( item.__class__ ).filter_by( user=item.user, slug=new_slug, importable=True ).count() != 0: + # Ensure unique across model class and user and don't include this item + # in the check in case it has previously been assigned a valid slug. + while sa_session.query( item.__class__ ).filter( item.__class__.user == item.user, item.__class__.slug == new_slug, item.__class__.id != item.id).count() != 0: # Slug taken; choose a new slug based on count. This approach can # handle numerous items with the same name gracefully. new_slug = '%s-%i' % ( slug_base, count ) From 29d4052667d47d5b94766ab89ea90dbba8135639 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 26 Aug 2014 09:00:14 -0400 Subject: [PATCH 2/5] Update tag latest_2014.08.11 for changeset 9a4b43021fd1 --- .hgtags | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.hgtags b/.hgtags index 34dd1729707..8544160a4a0 100644 --- a/.hgtags +++ b/.hgtags @@ -18,4 +18,4 @@ a4113cc1cb5eaa68091c9a73375f00555b66dd11 release_2013.01.13 81fbe25bd02edcd53065e8e4476dd1dfb5a72cf2 latest_2013.11.04 2a756ca2cb1826db7796018e77d12e2dd7b67603 latest_2014.02.10 ca45b78adb4152fc6e7395514d46eba6b7d0b838 release_2014.08.11 -7e22f35798522100ff03e1fdd4eced962b292360 latest_2014.08.11 +9a4b43021fd1c5808c17329da7b6acbfd9ae27ed latest_2014.08.11 From d26617f1bc96a25ec6c8153a972216f39ad68963 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 27 Aug 2014 14:39:43 -0400 Subject: [PATCH 3/5] Fix using multiple list collections with different identifiers in workflows. Tools will pair off (or triplize, quadraplate, quintukify, etc...) based only on the index of the elements in the collection (so any N element list can be used in the same tool execution as any other N element list). Originally, I had decided to require the element identifiers to be the same in both collections (allows mixing a matching fewer collections in tools - but probably less likely to mismatch in a way the user did not intend). At any rate - I changed that decision and updated tool execution appropriately but not every aspect workflow collection 'pairing' - this fixes that and closes out this card https://trello.com/c/m5PiF1lR. In very partical terms tools would ideally be consuming collection pairs - but for those that are still consuming two separate files (i.e. all of them) one can use two lists of datasets with pairs at matching indices and feed them into these tools - for workflows this doesn't work without this patch unless the identifiers and the indices match on these collections. Thanks for Michael Crusoe and takadonet for the bug reports. --- lib/galaxy/dataset_collections/structure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/dataset_collections/structure.py b/lib/galaxy/dataset_collections/structure.py index e067caaec23..917ec1698ff 100644 --- a/lib/galaxy/dataset_collections/structure.py +++ b/lib/galaxy/dataset_collections/structure.py @@ -35,9 +35,9 @@ class Tree( object ): return self._walk_collections( dict_map( lambda hdca: hdca.collection, hdca_dict ) ) def _walk_collections( self, collection_dict ): - for ( identifier, substructure ) in self.children: + for index, ( identifier, substructure ) in enumerate( self.children ): def element( collection ): - return collection[ identifier ] + return collection[ index ] if substructure.is_leaf: yield dict_map( element, collection_dict ) From bfb4688213d627fd038ab15e4b79983cd500ee67 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Thu, 28 Aug 2014 09:00:13 -0400 Subject: [PATCH 4/5] Update tag latest_2014.08.11 for changeset 109b170188e9 --- .hgtags | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.hgtags b/.hgtags index 8544160a4a0..9362ac875a6 100644 --- a/.hgtags +++ b/.hgtags @@ -18,4 +18,4 @@ a4113cc1cb5eaa68091c9a73375f00555b66dd11 release_2013.01.13 81fbe25bd02edcd53065e8e4476dd1dfb5a72cf2 latest_2013.11.04 2a756ca2cb1826db7796018e77d12e2dd7b67603 latest_2014.02.10 ca45b78adb4152fc6e7395514d46eba6b7d0b838 release_2014.08.11 -9a4b43021fd1c5808c17329da7b6acbfd9ae27ed latest_2014.08.11 +109b170188e97fbfc2c998ec174aff9546dd1bd8 latest_2014.08.11 From 09f1589ea4e77d8cb3662fbba24cd43e056672dd Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 28 Aug 2014 15:09:10 +0200 Subject: [PATCH 5/5] Fix SQlite datatype sniffing for old sqlite3. Remove unused imports. Bug fixed: when uploading a gzipped FASTQ file through upload1 tool on a machine with SQLite 3.3.6, it was detected as 'sqlite' datatype and not decompressed. In fact, the sqlite3 command 'pragma schema_version' on any file returns without error on this SQLite version. --- lib/galaxy/datatypes/binary.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index b6b7dbca433..a0034e3168e 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -11,10 +11,7 @@ import shutil import struct import subprocess import tempfile -import zipfile -import sqlite3 -from urllib import urlencode, quote_plus from galaxy import eggs eggs.require( "bx-python" ) @@ -22,7 +19,6 @@ from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_ from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes import metadata -from galaxy.datatypes.sniff import * import dataproviders log = logging.getLogger(__name__) @@ -552,14 +548,12 @@ Binary.register_sniffable_binary_format("twobit", "twobit", TwoBit) class SQlite ( Binary ): file_ext = "sqlite" - # Connects and runs a query that should work on any real database - # If the file is not sqlite, an exception will be thrown and the sniffer will return false def sniff( self, filename ): + # The first 16 bytes of any SQLite3 database file is 'SQLite format 3\0', and the file is binary. For details + # about the format, see http://www.sqlite.org/fileformat.html try: - conn = sqlite3.connect(filename) - schema_version=conn.cursor().execute("pragma schema_version").fetchone() - conn.close() - if schema_version is not None: + header = open(filename).read(16) + if binascii.b2a_hex(header) == binascii.hexlify('SQLite format 3\0'): return True return False except: @@ -586,5 +580,5 @@ class SQlite ( Binary ): return dataproviders.dataset.SQliteDataProvider( dataset_source, **settings ) -Binary.register_sniffable_binary_format("sqlite","sqlite",SQlite) +Binary.register_sniffable_binary_format("sqlite", "sqlite", SQlite)