From 72f3d25d09a987cdf2a419a39dd32013d53806f6 Mon Sep 17 00:00:00 2001 From: Brad Langhorst Date: Sun, 7 Jul 2019 11:13:11 -0400 Subject: [PATCH 01/17] Switches to ldda update time for list views --- lib/galaxy/webapps/galaxy/api/folder_contents.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/folder_contents.py b/lib/galaxy/webapps/galaxy/api/folder_contents.py index a7ecc98481d..f0e2bdf2e9b 100644 --- a/lib/galaxy/webapps/galaxy/api/folder_contents.py +++ b/lib/galaxy/webapps/galaxy/api/folder_contents.py @@ -81,13 +81,13 @@ class FolderContentsController(BaseAPIController, UsesLibraryMixin, UsesLibraryM for content_item in self._load_folder_contents(trans, folder, deleted): return_item = {} encoded_id = trans.security.encode_id(content_item.id) - update_time = content_item.update_time.strftime("%Y-%m-%d %I:%M %p") create_time = content_item.create_time.strftime("%Y-%m-%d %I:%M %p") if content_item.api_type == 'folder': encoded_id = 'F' + encoded_id can_modify = is_admin or (trans.user and trans.app.security_agent.can_modify_library_item(current_user_roles, folder)) can_manage = is_admin or (trans.user and trans.app.security_agent.can_manage_library_item(current_user_roles, folder)) + update_time = content_item.update_time.strftime("%Y-%m-%d %I:%M %p") return_item.update(dict(can_modify=can_modify, can_manage=can_manage)) if content_item.description: return_item.update(dict(description=content_item.description)) @@ -106,12 +106,14 @@ class FolderContentsController(BaseAPIController, UsesLibraryMixin, UsesLibraryM # Can user manage the permissions on the dataset? can_manage = is_admin or (trans.user and trans.app.security_agent.can_manage_dataset(current_user_roles, content_item.library_dataset_dataset_association.dataset)) raw_size = int(content_item.library_dataset_dataset_association.get_size()) + update_time = content_item.library_dataset_dataset_association.update_time.strftime("%Y-%m-%d %I:%M %p") nice_size = util.nice_size(raw_size) library_dataset_dict = content_item.to_dict() encoded_ldda_id = trans.security.encode_id(content_item.library_dataset_dataset_association.id) return_item.update(dict(file_ext=library_dataset_dict['file_ext'], date_uploaded=library_dataset_dict['date_uploaded'], + update_time=update_time, is_unrestricted=is_unrestricted, is_private=is_private, can_manage=can_manage, From f8d75a7c2c85e109a78df28cea03e5aed693f230 Mon Sep 17 00:00:00 2001 From: Brad Langhorst Date: Sun, 7 Jul 2019 11:15:39 -0400 Subject: [PATCH 02/17] Displays update_time throughout library view --- .../scripts/mvc/library/library-dataset-view.js | 16 ++++++++++++++++ .../mvc/library/library-folderlist-view.js | 2 +- lib/galaxy/managers/library_datasets.py | 2 ++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/client/galaxy/scripts/mvc/library/library-dataset-view.js b/client/galaxy/scripts/mvc/library/library-dataset-view.js index 53c6d30eb6f..8137376b77f 100644 --- a/client/galaxy/scripts/mvc/library/library-dataset-view.js +++ b/client/galaxy/scripts/mvc/library/library-dataset-view.js @@ -703,6 +703,12 @@ var LibraryDatasetView = Backbone.View.extend({ <%= _.escape(item.get("file_size")) %> <% } %> + <% if (item.get("update_time")) { %> + + Date last updated (UTC) + <%= _.escape(item.get("update_time")) %> + + <% } %> <% if (item.get("date_uploaded")) { %> Date uploaded (UTC) @@ -880,6 +886,12 @@ var LibraryDatasetView = Backbone.View.extend({ <%= _.escape(ldda.get("file_size")) %> <% } %> + <% if (ldda.get("update_time")) { %> + + Date last updated (UTC) + <%= _.escape(ldda.get("update_time")) %> + + <% } %> <% if (ldda.get("date_uploaded")) { %> Date uploaded (UTC) @@ -1006,6 +1018,10 @@ var LibraryDatasetView = Backbone.View.extend({ Size <%= _.escape(item.get("file_size")) %> + + Date last updated (UTC) + <%= _.escape(item.get("update_time")) %> + Date uploaded (UTC) <%= _.escape(item.get("date_uploaded")) %> diff --git a/client/galaxy/scripts/mvc/library/library-folderlist-view.js b/client/galaxy/scripts/mvc/library/library-folderlist-view.js index 1b6c271f99c..cedc161f341 100644 --- a/client/galaxy/scripts/mvc/library/library-folderlist-view.js +++ b/client/galaxy/scripts/mvc/library/library-folderlist-view.js @@ -517,7 +517,7 @@ var FolderListView = Backbone.View.extend({ - Time Updated (UTC) + Date Updated (UTC) diff --git a/lib/galaxy/managers/library_datasets.py b/lib/galaxy/managers/library_datasets.py index 6cdf791fd3a..7afd706a7cc 100644 --- a/lib/galaxy/managers/library_datasets.py +++ b/lib/galaxy/managers/library_datasets.py @@ -101,6 +101,7 @@ class LibraryDatasetsManager(datasets.DatasetAssociationManager): ldda.dbkey = new_genome_build changed = True if changed: + ldda.updateParentFolderUpdateTimes() trans.sa_session.add(ldda) trans.sa_session.flush() return changed @@ -223,6 +224,7 @@ class LibraryDatasetsManager(datasets.DatasetAssociationManager): rval['full_path'] = full_path rval['file_size'] = util.nice_size(int(ldda.get_size())) rval['date_uploaded'] = ldda.create_time.strftime("%Y-%m-%d %I:%M %p") + rval['update_time'] = ldda.update_time.strftime("%Y-%m-%d %I:%M %p") rval['can_user_modify'] = trans.user_is_admin or trans.app.security_agent.can_modify_library_item(current_user_roles, ld) rval['is_unrestricted'] = trans.app.security_agent.dataset_is_public(ldda.dataset) rval['tags'] = self.tag_handler.get_tags_str(ldda.tags) From c2f71cf4d880444b08fdea464d735dbd70bed3b5 Mon Sep 17 00:00:00 2001 From: Brad Langhorst Date: Sun, 7 Jul 2019 11:15:48 -0400 Subject: [PATCH 03/17] propagates updates to library items up the folder tree --- lib/galaxy/model/__init__.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 6ad851608f2..78b67b4a1e1 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -31,6 +31,7 @@ from sqlalchemy import ( not_, or_, select, + text, true, type_coerce, types) @@ -3424,6 +3425,7 @@ class LibraryDataset(RepresentById): uploaded_by=ldda.user.email, message=ldda.message, date_uploaded=ldda.create_time.isoformat(), + update_time=ldda.update_time.isoformat(), file_size=int(ldda.get_size()), file_ext=ldda.ext, data_type=ldda.datatype.__class__.__module__ + '.' + ldda.datatype.__class__.__name__, @@ -3592,6 +3594,26 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById): rval['metadata_' + name] = val return rval + def updateParentFolderUpdateTimes(self): + # sets the update_time for all continaing folders up the tree + ldda = self + db_session = object_session(ldda) + + sql = text( + ''' + with recursive + parent_folders_of(folder_id) as ( + select folder_id from library_dataset where id = :library_dataset_id + union all + select library_folder.parent_id from library_folder, parent_folders_of + where library_folder.id = parent_folders_of.folder_id + ) + update library_folder + set update_time = (select update_time from library_dataset_dataset_association where id = :ldda_id) + from parent_folders_of + where library_folder.id = parent_folders_of.folder_id + ''') + db_session.execute(sql, {'library_dataset_id': ldda.library_dataset_id, 'ldda_id': ldda.id}) class ExtendedMetadata(RepresentById): def __init__(self, data): From 1cabf49f0e246eb56cfcec3b8da7ab34cb43f07a Mon Sep 17 00:00:00 2001 From: Brad Langhorst Date: Sun, 7 Jul 2019 11:16:24 -0400 Subject: [PATCH 04/17] Folders should also sort so we can find out which one was most recently updated --- .../scripts/mvc/library/library-util.js | 60 +++++++++---------- 1 file changed, 27 insertions(+), 33 deletions(-) diff --git a/client/galaxy/scripts/mvc/library/library-util.js b/client/galaxy/scripts/mvc/library/library-util.js index c4d546c4b25..7bb864a0407 100644 --- a/client/galaxy/scripts/mvc/library/library-util.js +++ b/client/galaxy/scripts/mvc/library/library-util.js @@ -1,45 +1,39 @@ /** - * Create alphabetical based two-argument comparator - * that takes into account that Folder comes before Dataset. + * Create alphabetical based two-argument comparator to handle library items (including folders) * If sort_key is not present it is set to ''. * @param {str} sort_key key to sort by * @param {str} sort_order order to sort by (asc, desc) * @return {function} two-argument comparator function */ var generateComparator = (sort_key, sort_order) => (itemA, itemB) => { - if (itemA.get("type") === itemB.get("type")) { - if (!itemA.has(sort_key) && !itemB.has(sort_key)) { - return 0; - } else if (!itemA.has(sort_key)) { - return 1; - } else if (!itemB.has(sort_key)) { - return -1; - } - var comparable_itemA_key; - var comparable_itemB_key; - if (typeof itemA.get(sort_key) === "number") { - comparable_itemA_key = itemA.get(sort_key); - comparable_itemB_key = itemB.get(sort_key); - } - else - { - comparable_itemA_key = itemA.get(sort_key).toLowerCase(); - comparable_itemB_key = itemB.get(sort_key).toLowerCase(); - } - if ( comparable_itemA_key > comparable_itemB_key) { - return sort_order === "asc" ? 1 : -1; - } - if (comparable_itemB_key > comparable_itemA_key) { - return sort_order === "asc" ? -1 : 1; - } - - return 0; // equal - } else { - if (itemA.get("type") === "folder") { - return -1; // folder is always before dataset - } + if (!itemA.has(sort_key) && !itemB.has(sort_key)) { + return 0; + } else if (!itemA.has(sort_key)) { return 1; + } else if (!itemB.has(sort_key)) { + return -1; } + var folder_bonus = 0.5; + var comparable_itemA_key; + var comparable_itemB_key; + if (typeof itemA.get(sort_key) === "number") { + comparable_itemA_key = itemA.get(sort_key); + comparable_itemB_key = itemB.get(sort_key); + } + else + { + comparable_itemA_key = itemA.get(sort_key).toLowerCase(); + comparable_itemB_key = itemB.get(sort_key).toLowerCase(); + } + + if ( comparable_itemA_key > comparable_itemB_key) { + return sort_order === "asc" ? 1 : -1; + } + if (comparable_itemB_key > comparable_itemA_key) { + return sort_order === "asc" ? -1 : 1; + } + + return 0; // equal }; export default { generateComparator: generateComparator From e682d100e20a8358be7da1a796aaffb3607f57ab Mon Sep 17 00:00:00 2001 From: Brad Langhorst Date: Sun, 7 Jul 2019 13:17:55 -0400 Subject: [PATCH 05/17] More logging to figure out why updates are not sticking --- lib/galaxy/model/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 78b67b4a1e1..edb7cc88868 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3608,12 +3608,15 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById): select library_folder.parent_id from library_folder, parent_folders_of where library_folder.id = parent_folders_of.folder_id ) - update library_folder - set update_time = (select update_time from library_dataset_dataset_association where id = :ldda_id) + update library_folder set update_time = + (select update_time from library_dataset_dataset_association where id = :ldda_id) from parent_folders_of where library_folder.id = parent_folders_of.folder_id ''') - db_session.execute(sql, {'library_dataset_id': ldda.library_dataset_id, 'ldda_id': ldda.id}) + log.debug('Updating parent folder update_time: {0} {1} {2}'.format(sql,ldda.library_dataset_id,ldda.id)) + ret = db_session.execute(sql, {'library_dataset_id': ldda.library_dataset_id, 'ldda_id': ldda.id}) + log.debug('execute returns: {0}'.format(ret.rowcount)) + db_session.flush() class ExtendedMetadata(RepresentById): def __init__(self, data): From ac1a7c5593b00f4fae8f0a03c77811113f6c0b6c Mon Sep 17 00:00:00 2001 From: Brad Langhorst Date: Sun, 7 Jul 2019 13:18:42 -0400 Subject: [PATCH 06/17] calls time update during import to library too --- lib/galaxy/webapps/base/controller.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/webapps/base/controller.py b/lib/galaxy/webapps/base/controller.py index 41c1d66722e..418ebc842db 100644 --- a/lib/galaxy/webapps/base/controller.py +++ b/lib/galaxy/webapps/base/controller.py @@ -546,6 +546,7 @@ class UsesLibraryMixinItems(SharableItemSecurityMixin): # I don't see a reason why hdas copied into libraries should not be visible. # If there is, refactor `ldda.visible = True` to do this only when adding HDCAs. ldda.visible = True + ldda.updateParentFolderUpdateTimes() trans.sa_session.flush() ldda_dict = ldda.to_dict() rval = trans.security.encode_dict_ids(ldda_dict) From 084cceffe393ee8834188f7716465a4a4d8cc3e2 Mon Sep 17 00:00:00 2001 From: Brad Langhorst Date: Sun, 7 Jul 2019 13:39:47 -0400 Subject: [PATCH 07/17] tried passing in trans to avoid an additional session (Still not sticking) --- lib/galaxy/managers/library_datasets.py | 2 +- lib/galaxy/model/__init__.py | 6 ++---- lib/galaxy/webapps/base/controller.py | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/managers/library_datasets.py b/lib/galaxy/managers/library_datasets.py index 7afd706a7cc..1a78e7d3883 100644 --- a/lib/galaxy/managers/library_datasets.py +++ b/lib/galaxy/managers/library_datasets.py @@ -101,7 +101,7 @@ class LibraryDatasetsManager(datasets.DatasetAssociationManager): ldda.dbkey = new_genome_build changed = True if changed: - ldda.updateParentFolderUpdateTimes() + ldda.updateParentFolderUpdateTimes(trans) trans.sa_session.add(ldda) trans.sa_session.flush() return changed diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index edb7cc88868..fea7700a497 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3594,10 +3594,9 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById): rval['metadata_' + name] = val return rval - def updateParentFolderUpdateTimes(self): + def updateParentFolderUpdateTimes(self,trans): # sets the update_time for all continaing folders up the tree ldda = self - db_session = object_session(ldda) sql = text( ''' @@ -3614,9 +3613,8 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById): where library_folder.id = parent_folders_of.folder_id ''') log.debug('Updating parent folder update_time: {0} {1} {2}'.format(sql,ldda.library_dataset_id,ldda.id)) - ret = db_session.execute(sql, {'library_dataset_id': ldda.library_dataset_id, 'ldda_id': ldda.id}) + ret = trans.sa_session.execute(sql, {'library_dataset_id': ldda.library_dataset_id, 'ldda_id': ldda.id}) log.debug('execute returns: {0}'.format(ret.rowcount)) - db_session.flush() class ExtendedMetadata(RepresentById): def __init__(self, data): diff --git a/lib/galaxy/webapps/base/controller.py b/lib/galaxy/webapps/base/controller.py index 418ebc842db..29174958a1e 100644 --- a/lib/galaxy/webapps/base/controller.py +++ b/lib/galaxy/webapps/base/controller.py @@ -546,7 +546,7 @@ class UsesLibraryMixinItems(SharableItemSecurityMixin): # I don't see a reason why hdas copied into libraries should not be visible. # If there is, refactor `ldda.visible = True` to do this only when adding HDCAs. ldda.visible = True - ldda.updateParentFolderUpdateTimes() + ldda.updateParentFolderUpdateTimes(trans) trans.sa_session.flush() ldda_dict = ldda.to_dict() rval = trans.security.encode_dict_ids(ldda_dict) From 6cde398e6287207a7f3652fee43fcdf6d1c63bd7 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Sun, 7 Jul 2019 20:06:37 +0200 Subject: [PATCH 08/17] add autocommit --- lib/galaxy/model/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index fea7700a497..03fa0429d35 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3611,7 +3611,7 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById): (select update_time from library_dataset_dataset_association where id = :ldda_id) from parent_folders_of where library_folder.id = parent_folders_of.folder_id - ''') + ''').execution_options(autocommit=True) log.debug('Updating parent folder update_time: {0} {1} {2}'.format(sql,ldda.library_dataset_id,ldda.id)) ret = trans.sa_session.execute(sql, {'library_dataset_id': ldda.library_dataset_id, 'ldda_id': ldda.id}) log.debug('execute returns: {0}'.format(ret.rowcount)) From 2f1b120e317aadb7241996431e235f55a86cbf3f Mon Sep 17 00:00:00 2001 From: Brad Langhorst Date: Mon, 8 Jul 2019 04:00:56 -0400 Subject: [PATCH 09/17] =?UTF-8?q?Passing=20the=20trans=20was=20a=20bad=20i?= =?UTF-8?q?dea=E2=80=A6=20commit=20on=20that=20Caused=20user=20edits=20not?= =?UTF-8?q?=20to=20be=20saved?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/galaxy/managers/library_datasets.py | 2 +- lib/galaxy/model/__init__.py | 8 ++++---- lib/galaxy/webapps/base/controller.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/managers/library_datasets.py b/lib/galaxy/managers/library_datasets.py index 1a78e7d3883..7afd706a7cc 100644 --- a/lib/galaxy/managers/library_datasets.py +++ b/lib/galaxy/managers/library_datasets.py @@ -101,7 +101,7 @@ class LibraryDatasetsManager(datasets.DatasetAssociationManager): ldda.dbkey = new_genome_build changed = True if changed: - ldda.updateParentFolderUpdateTimes(trans) + ldda.updateParentFolderUpdateTimes() trans.sa_session.add(ldda) trans.sa_session.flush() return changed diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 03fa0429d35..161fb42d4b8 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3594,7 +3594,7 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById): rval['metadata_' + name] = val return rval - def updateParentFolderUpdateTimes(self,trans): + def updateParentFolderUpdateTimes(self): # sets the update_time for all continaing folders up the tree ldda = self @@ -3612,9 +3612,9 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById): from parent_folders_of where library_folder.id = parent_folders_of.folder_id ''').execution_options(autocommit=True) - log.debug('Updating parent folder update_time: {0} {1} {2}'.format(sql,ldda.library_dataset_id,ldda.id)) - ret = trans.sa_session.execute(sql, {'library_dataset_id': ldda.library_dataset_id, 'ldda_id': ldda.id}) - log.debug('execute returns: {0}'.format(ret.rowcount)) + log.debug('Updating parent folder update_times: {0} {1} {2}'.format(sql,ldda.library_dataset_id,ldda.id)) + ret = object_session(self).execute(sql, {'library_dataset_id': ldda.library_dataset_id, 'ldda_id': ldda.id}) + log.debug('updated parent folders: {0}'.format(ret.rowcount)) class ExtendedMetadata(RepresentById): def __init__(self, data): diff --git a/lib/galaxy/webapps/base/controller.py b/lib/galaxy/webapps/base/controller.py index 29174958a1e..418ebc842db 100644 --- a/lib/galaxy/webapps/base/controller.py +++ b/lib/galaxy/webapps/base/controller.py @@ -546,7 +546,7 @@ class UsesLibraryMixinItems(SharableItemSecurityMixin): # I don't see a reason why hdas copied into libraries should not be visible. # If there is, refactor `ldda.visible = True` to do this only when adding HDCAs. ldda.visible = True - ldda.updateParentFolderUpdateTimes(trans) + ldda.updateParentFolderUpdateTimes() trans.sa_session.flush() ldda_dict = ldda.to_dict() rval = trans.security.encode_dict_ids(ldda_dict) From 3535efe20c369c4044292e1badbcc67d9e0b5b2d Mon Sep 17 00:00:00 2001 From: Brad Langhorst Date: Mon, 8 Jul 2019 04:01:20 -0400 Subject: [PATCH 10/17] minor cleanup --- client/galaxy/scripts/mvc/library/library-util.js | 1 - lib/galaxy/webapps/galaxy/api/folder_contents.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/client/galaxy/scripts/mvc/library/library-util.js b/client/galaxy/scripts/mvc/library/library-util.js index 7bb864a0407..bb174a7cb56 100644 --- a/client/galaxy/scripts/mvc/library/library-util.js +++ b/client/galaxy/scripts/mvc/library/library-util.js @@ -13,7 +13,6 @@ var generateComparator = (sort_key, sort_order) => (itemA, itemB) => { } else if (!itemB.has(sort_key)) { return -1; } - var folder_bonus = 0.5; var comparable_itemA_key; var comparable_itemB_key; if (typeof itemA.get(sort_key) === "number") { diff --git a/lib/galaxy/webapps/galaxy/api/folder_contents.py b/lib/galaxy/webapps/galaxy/api/folder_contents.py index f0e2bdf2e9b..da81e0229d8 100644 --- a/lib/galaxy/webapps/galaxy/api/folder_contents.py +++ b/lib/galaxy/webapps/galaxy/api/folder_contents.py @@ -106,8 +106,8 @@ class FolderContentsController(BaseAPIController, UsesLibraryMixin, UsesLibraryM # Can user manage the permissions on the dataset? can_manage = is_admin or (trans.user and trans.app.security_agent.can_manage_dataset(current_user_roles, content_item.library_dataset_dataset_association.dataset)) raw_size = int(content_item.library_dataset_dataset_association.get_size()) - update_time = content_item.library_dataset_dataset_association.update_time.strftime("%Y-%m-%d %I:%M %p") nice_size = util.nice_size(raw_size) + update_time = content_item.library_dataset_dataset_association.update_time.strftime("%Y-%m-%d %I:%M %p") library_dataset_dict = content_item.to_dict() encoded_ldda_id = trans.security.encode_id(content_item.library_dataset_dataset_association.id) From a8deae2404a8f37a9cd53b38191014a0d0bc46a9 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Mon, 8 Jul 2019 10:23:54 +0200 Subject: [PATCH 11/17] format sql statement for readibility --- lib/galaxy/model/__init__.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 161fb42d4b8..744fee40083 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3600,17 +3600,21 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById): sql = text( ''' - with recursive - parent_folders_of(folder_id) as ( - select folder_id from library_dataset where id = :library_dataset_id - union all - select library_folder.parent_id from library_folder, parent_folders_of - where library_folder.id = parent_folders_of.folder_id - ) - update library_folder set update_time = - (select update_time from library_dataset_dataset_association where id = :ldda_id) - from parent_folders_of - where library_folder.id = parent_folders_of.folder_id + WITH RECURSIVE parent_folders_of(folder_id) AS + (SELECT folder_id + FROM library_dataset + WHERE id = :library_dataset_id + UNION ALL SELECT library_folder.parent_id + FROM library_folder, + parent_folders_of + WHERE library_folder.id = parent_folders_of.folder_id ) + UPDATE library_folder + SET update_time = + (SELECT update_time + FROM library_dataset_dataset_association + WHERE id = :ldda_id) + FROM parent_folders_of + WHERE library_folder.id = parent_folders_of.folder_id ''').execution_options(autocommit=True) log.debug('Updating parent folder update_times: {0} {1} {2}'.format(sql,ldda.library_dataset_id,ldda.id)) ret = object_session(self).execute(sql, {'library_dataset_id': ldda.library_dataset_id, 'ldda_id': ldda.id}) From 705ad4b21c7fd599d57c0a22838da582ca5141f3 Mon Sep 17 00:00:00 2001 From: Brad Langhorst Date: Mon, 8 Jul 2019 04:42:26 -0400 Subject: [PATCH 12/17] Linting fixes --- lib/galaxy/model/__init__.py | 37 ++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 744fee40083..3cc5ef7506a 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3599,27 +3599,28 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById): ldda = self sql = text( - ''' - WITH RECURSIVE parent_folders_of(folder_id) AS - (SELECT folder_id - FROM library_dataset - WHERE id = :library_dataset_id - UNION ALL SELECT library_folder.parent_id - FROM library_folder, - parent_folders_of - WHERE library_folder.id = parent_folders_of.folder_id ) - UPDATE library_folder - SET update_time = - (SELECT update_time - FROM library_dataset_dataset_association - WHERE id = :ldda_id) - FROM parent_folders_of - WHERE library_folder.id = parent_folders_of.folder_id - ''').execution_options(autocommit=True) - log.debug('Updating parent folder update_times: {0} {1} {2}'.format(sql,ldda.library_dataset_id,ldda.id)) + ''' + WITH RECURSIVE parent_folders_of(folder_id) AS + (SELECT folder_id + FROM library_dataset + WHERE id = :library_dataset_id + UNION ALL SELECT library_folder.parent_id + FROM library_folder, + parent_folders_of + WHERE library_folder.id = parent_folders_of.folder_id ) + UPDATE library_folder + SET update_time = + (SELECT update_time + FROM library_dataset_dataset_association + WHERE id = :ldda_id) + FROM parent_folders_of + WHERE library_folder.id = parent_folders_of.folder_id + ''').execution_options(autocommit=True) + log.debug('Updating parent folder update_times: {0} {1} {2}'. format(sql, ldda.library_dataset_id, ldda.id)) ret = object_session(self).execute(sql, {'library_dataset_id': ldda.library_dataset_id, 'ldda_id': ldda.id}) log.debug('updated parent folders: {0}'.format(ret.rowcount)) + class ExtendedMetadata(RepresentById): def __init__(self, data): self.data = data From dc26a388bbbabea917580faeb9f637a02b5812fd Mon Sep 17 00:00:00 2001 From: Brad Langhorst Date: Mon, 8 Jul 2019 04:48:13 -0400 Subject: [PATCH 13/17] removes noisy debug output --- lib/galaxy/model/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 3cc5ef7506a..5084de09e2b 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3616,9 +3616,9 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById): FROM parent_folders_of WHERE library_folder.id = parent_folders_of.folder_id ''').execution_options(autocommit=True) - log.debug('Updating parent folder update_times: {0} {1} {2}'. format(sql, ldda.library_dataset_id, ldda.id)) ret = object_session(self).execute(sql, {'library_dataset_id': ldda.library_dataset_id, 'ldda_id': ldda.id}) - log.debug('updated parent folders: {0}'.format(ret.rowcount)) + if ret.rowcount < 1: + log.warn('Attempt to updated parent folder times failed: {0} records updated.'.format(ret.rowcount)) class ExtendedMetadata(RepresentById): From 4f99db9d3fed3e2f94cee7c9f859c6e036b9339d Mon Sep 17 00:00:00 2001 From: Brad Langhorst Date: Mon, 8 Jul 2019 06:29:57 -0400 Subject: [PATCH 14/17] =?UTF-8?q?starting=20point=20for=20a=20test?= =?UTF-8?q?=E2=80=A6=20Failing=20on=20sqlite?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/api/test_libraries.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/api/test_libraries.py b/test/api/test_libraries.py index a9f8d5f4f05..671753c5eb2 100644 --- a/test/api/test_libraries.py +++ b/test/api/test_libraries.py @@ -232,6 +232,27 @@ class LibrariesApiTestCase(api.ApiTestCase, TestsDatasets): self._assert_status_code_is(create_response, 200) self._assert_has_keys(create_response.json(), "name", "id") + def test_create_dataset_in_subfolder(self): + library = self.library_populator.new_private_library("ForCreateDatasets") + folder_response = self._create_folder(library) + self._assert_status_code_is(folder_response, 200) + folder_id = folder_response.json()[0]['id'] + subfolder_response = self._create_subfolder(folder_id) + self._assert_status_code_is(folder_response, 200) + print(subfolder_response.json()) + subfolder_id = subfolder_response.json()['id'] + history_id = self.dataset_populator.new_history() + hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3 sub")['id'] + payload = {'from_hda_id': hda_id} + create_response = self._post("folders/%s/contents" % subfolder_id, payload) + self._assert_status_code_is(create_response, 200) + self._assert_has_keys(create_response.json(), "name", "id") + dataset_update_time = create_response.json()['update_time'] + container_fetch_response = self.galaxy_interactor.get("folders/%s/contents" % folder_id ) + container_update_time = container_fetch_response.json()['folder_contents'][0]['update_time'] + assert dataset_update_time == container_update_time, container_fetch_response + + def test_update_dataset_in_folder(self): ld = self._create_dataset_in_folder_in_library("ForUpdateDataset") data = {'name': 'updated_name', 'file_ext': 'fastq', 'misc_info': 'updated_info', 'genome_build': 'updated_genome_build'} @@ -301,6 +322,13 @@ class LibrariesApiTestCase(api.ApiTestCase, TestsDatasets): ) return self._post("libraries/%s/contents" % library["id"], data=create_data) + def _create_subfolder(self, containing_folder_id): + create_data = dict( + description="new subfolder desc", + name="New Subfolder", + ) + return self._post("folders/%s" % containing_folder_id, data=create_data) + def _create_dataset_in_folder_in_library(self, library_name): library = self.library_populator.new_private_library(library_name) folder_response = self._create_folder(library) From d35df5eca71318b92b6519c37b7b0eb504d7a3fe Mon Sep 17 00:00:00 2001 From: Brad Langhorst Date: Mon, 8 Jul 2019 13:10:35 -0400 Subject: [PATCH 15/17] Sqlite3 compatible syntax --- lib/galaxy/model/__init__.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 5084de09e2b..0d93ec7ba44 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3601,20 +3601,20 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById): sql = text( ''' WITH RECURSIVE parent_folders_of(folder_id) AS - (SELECT folder_id - FROM library_dataset - WHERE id = :library_dataset_id - UNION ALL SELECT library_folder.parent_id - FROM library_folder, - parent_folders_of - WHERE library_folder.id = parent_folders_of.folder_id ) + (SELECT folder_id + FROM library_dataset + WHERE id = :library_dataset_id + UNION ALL + SELECT library_folder.parent_id + FROM library_folder, parent_folders_of + WHERE library_folder.id = parent_folders_of.folder_id ) UPDATE library_folder SET update_time = - (SELECT update_time - FROM library_dataset_dataset_association - WHERE id = :ldda_id) - FROM parent_folders_of - WHERE library_folder.id = parent_folders_of.folder_id + (SELECT update_time + FROM library_dataset_dataset_association + WHERE id = :ldda_id) + WHERE exists (SELECT 1 FROM parent_folders_of + WHERE library_folder.id = parent_folders_of.folder_id) ''').execution_options(autocommit=True) ret = object_session(self).execute(sql, {'library_dataset_id': ldda.library_dataset_id, 'ldda_id': ldda.id}) if ret.rowcount < 1: From 50011c1e98348f323c2c1e78c4a63efc26bca4a3 Mon Sep 17 00:00:00 2001 From: Brad Langhorst Date: Tue, 9 Jul 2019 04:09:11 -0400 Subject: [PATCH 16/17] linting fixes --- lib/galaxy/model/__init__.py | 4 ++-- test/api/test_libraries.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 0d93ec7ba44..d90101036b5 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3604,7 +3604,7 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById): (SELECT folder_id FROM library_dataset WHERE id = :library_dataset_id - UNION ALL + UNION ALL SELECT library_folder.parent_id FROM library_folder, parent_folders_of WHERE library_folder.id = parent_folders_of.folder_id ) @@ -3613,7 +3613,7 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById): (SELECT update_time FROM library_dataset_dataset_association WHERE id = :ldda_id) - WHERE exists (SELECT 1 FROM parent_folders_of + WHERE exists (SELECT 1 FROM parent_folders_of WHERE library_folder.id = parent_folders_of.folder_id) ''').execution_options(autocommit=True) ret = object_session(self).execute(sql, {'library_dataset_id': ldda.library_dataset_id, 'ldda_id': ldda.id}) diff --git a/test/api/test_libraries.py b/test/api/test_libraries.py index 671753c5eb2..11454c23695 100644 --- a/test/api/test_libraries.py +++ b/test/api/test_libraries.py @@ -248,11 +248,10 @@ class LibrariesApiTestCase(api.ApiTestCase, TestsDatasets): self._assert_status_code_is(create_response, 200) self._assert_has_keys(create_response.json(), "name", "id") dataset_update_time = create_response.json()['update_time'] - container_fetch_response = self.galaxy_interactor.get("folders/%s/contents" % folder_id ) + container_fetch_response = self.galaxy_interactor.get("folders/%s/contents" % folder_id) container_update_time = container_fetch_response.json()['folder_contents'][0]['update_time'] assert dataset_update_time == container_update_time, container_fetch_response - def test_update_dataset_in_folder(self): ld = self._create_dataset_in_folder_in_library("ForUpdateDataset") data = {'name': 'updated_name', 'file_ext': 'fastq', 'misc_info': 'updated_info', 'genome_build': 'updated_genome_build'} From 485e0043a7e25c0002765669a1bfc792ad895ea2 Mon Sep 17 00:00:00 2001 From: Brad Langhorst Date: Wed, 10 Jul 2019 05:22:27 -0400 Subject: [PATCH 17/17] implements nicola's suggestion -> snakecase --- lib/galaxy/managers/library_datasets.py | 2 +- lib/galaxy/model/__init__.py | 2 +- lib/galaxy/webapps/base/controller.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/managers/library_datasets.py b/lib/galaxy/managers/library_datasets.py index 7afd706a7cc..20b76a1bf50 100644 --- a/lib/galaxy/managers/library_datasets.py +++ b/lib/galaxy/managers/library_datasets.py @@ -101,7 +101,7 @@ class LibraryDatasetsManager(datasets.DatasetAssociationManager): ldda.dbkey = new_genome_build changed = True if changed: - ldda.updateParentFolderUpdateTimes() + ldda.update_parent_folder_update_times() trans.sa_session.add(ldda) trans.sa_session.flush() return changed diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index d90101036b5..41a8f8ef5bd 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3594,7 +3594,7 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById): rval['metadata_' + name] = val return rval - def updateParentFolderUpdateTimes(self): + def update_parent_folder_update_times(self): # sets the update_time for all continaing folders up the tree ldda = self diff --git a/lib/galaxy/webapps/base/controller.py b/lib/galaxy/webapps/base/controller.py index 418ebc842db..f08c3b81d0b 100644 --- a/lib/galaxy/webapps/base/controller.py +++ b/lib/galaxy/webapps/base/controller.py @@ -546,7 +546,7 @@ class UsesLibraryMixinItems(SharableItemSecurityMixin): # I don't see a reason why hdas copied into libraries should not be visible. # If there is, refactor `ldda.visible = True` to do this only when adding HDCAs. ldda.visible = True - ldda.updateParentFolderUpdateTimes() + ldda.update_parent_folder_update_times() trans.sa_session.flush() ldda_dict = ldda.to_dict() rval = trans.security.encode_dict_ids(ldda_dict)