From a092aa2f590df8fd483557499b916a30100f3e27 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Tue, 12 May 2015 17:53:18 +0200 Subject: [PATCH 1/5] Start providing usage information for the various cleanup scripts --- scripts/cleanup_datasets/admin_cleanup_datasets.py | 6 +++++- scripts/cleanup_datasets/cleanup_datasets.py | 6 +++++- .../remove_renamed_datasets_from_disk.py | 10 ++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/scripts/cleanup_datasets/admin_cleanup_datasets.py b/scripts/cleanup_datasets/admin_cleanup_datasets.py index 987f064b209..2dedb8f16cb 100644 --- a/scripts/cleanup_datasets/admin_cleanup_datasets.py +++ b/scripts/cleanup_datasets/admin_cleanup_datasets.py @@ -74,7 +74,8 @@ def main(): contains the specified text will be marked as deleted in user's history and the user will be notified by email using the specified template file. """ - parser = OptionParser() + usage = "usage: %prog [options] galaxy.ini" + parser = OptionParser(usage=usage) parser.add_option("-d", "--days", dest="days", action="store", type="int", help="number of days (60)", default=60) parser.add_option("--tool_id", default=None, @@ -100,6 +101,9 @@ def main(): help="From address to use to send email. " "Default: [read from galaxy ini file]") (options, args) = parser.parse_args() + if len(args) != 1 : + parser.print_help() + sys.exit() ini_file = args[0] config_parser = ConfigParser.ConfigParser({'here': os.getcwd()}) diff --git a/scripts/cleanup_datasets/cleanup_datasets.py b/scripts/cleanup_datasets/cleanup_datasets.py index 1c82e8b2203..86818f23bcd 100644 --- a/scripts/cleanup_datasets/cleanup_datasets.py +++ b/scripts/cleanup_datasets/cleanup_datasets.py @@ -66,7 +66,8 @@ def main(): Another way of stating it is: LibraryDatasetDatasetAssociation objects map LibraryDataset objects to Dataset objects, and Dataset objects may be mapped to History objects via HistoryDatasetAssociation objects. """ - parser = OptionParser() + usage = "usage: %prog [options] galaxy.ini" + parser = OptionParser(usage=usage) parser.add_option( "-d", "--days", dest="days", action="store", type="int", help="number of days (60)", default=60 ) parser.add_option( "-r", "--remove_from_disk", action="store_true", dest="remove_from_disk", help="remove datasets from disk when purged", default=False ) parser.add_option( "-i", "--info_only", action="store_true", dest="info_only", help="info about the requested action", default=False ) @@ -79,6 +80,9 @@ def main(): parser.add_option( "-6", "--delete_datasets", action="store_true", dest="delete_datasets", default=False, help="mark deletable datasets as deleted and purge associated dataset instances" ) ( options, args ) = parser.parse_args() + if len(args) != 1 : + parser.print_help() + sys.exit() ini_file = args[0] if not ( options.purge_folders ^ options.delete_userless_histories ^ \ diff --git a/scripts/cleanup_datasets/remove_renamed_datasets_from_disk.py b/scripts/cleanup_datasets/remove_renamed_datasets_from_disk.py index 989572a2e57..fe52e9470b0 100644 --- a/scripts/cleanup_datasets/remove_renamed_datasets_from_disk.py +++ b/scripts/cleanup_datasets/remove_renamed_datasets_from_disk.py @@ -8,7 +8,17 @@ import sys, os assert sys.version_info[:2] >= ( 2, 4 ) +def usage(prog) : + print "usage: %s file" % prog + print "The input file should contain a list of files to be deleted, one per line. The" + print "full path must be specified and must begin with /var/opt/galaxy" + print "\nA log of files deleted is created in a file with the same name as that input" + print "but with .removed.log appended" + def main(): + if len(sys.argv) != 2 or sys.argv == "-h" or sys.argv == "--help" : + usage(sys.argv[0]) + sys.exit() infile = sys.argv[1] outfile = infile + ".removed.log" out = open( outfile, 'w' ) From 4c20494011659da1c595aece91a6f58b4e3aa045 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Tue, 12 May 2015 18:25:32 +0200 Subject: [PATCH 2/5] Make python scripts executable. Tweak remove_renamed_datasets_from_disk.py. --- scripts/cleanup_datasets/admin_cleanup_datasets.py | 0 scripts/cleanup_datasets/cleanup_datasets.py | 0 scripts/cleanup_datasets/populate_uuid.py | 12 ++++++++++++ .../remove_renamed_datasets_from_disk.py | 12 ++++++++---- scripts/cleanup_datasets/rename_purged_datasets.py | 0 scripts/cleanup_datasets/update_dataset_size.py | 0 scripts/cleanup_datasets/update_metadata.py | 0 7 files changed, 20 insertions(+), 4 deletions(-) mode change 100644 => 100755 scripts/cleanup_datasets/admin_cleanup_datasets.py mode change 100644 => 100755 scripts/cleanup_datasets/cleanup_datasets.py mode change 100644 => 100755 scripts/cleanup_datasets/populate_uuid.py mode change 100644 => 100755 scripts/cleanup_datasets/remove_renamed_datasets_from_disk.py mode change 100644 => 100755 scripts/cleanup_datasets/rename_purged_datasets.py mode change 100644 => 100755 scripts/cleanup_datasets/update_dataset_size.py mode change 100644 => 100755 scripts/cleanup_datasets/update_metadata.py diff --git a/scripts/cleanup_datasets/admin_cleanup_datasets.py b/scripts/cleanup_datasets/admin_cleanup_datasets.py old mode 100644 new mode 100755 diff --git a/scripts/cleanup_datasets/cleanup_datasets.py b/scripts/cleanup_datasets/cleanup_datasets.py old mode 100644 new mode 100755 diff --git a/scripts/cleanup_datasets/populate_uuid.py b/scripts/cleanup_datasets/populate_uuid.py old mode 100644 new mode 100755 index f7075b8078b..ebecde92afd --- a/scripts/cleanup_datasets/populate_uuid.py +++ b/scripts/cleanup_datasets/populate_uuid.py @@ -22,7 +22,19 @@ from sqlalchemy import * assert sys.version_info[:2] >= ( 2, 4 ) +def usage(prog) : + print "usage: %s galaxy.ini" % prog + print """ +Populates blank uuid fields in datasets with randomly generated values + +Going forward, these ids will be generated for all new datasets. This +script fixes datasets that were generated before the change. +""" + def main(): + if len(sys.argv) != 2 or sys.argv == "-h" or sys.argv == "--help" : + usage(sys.argv[0]) + sys.exit() ini_file = sys.argv.pop(1) config = get_config(ini_file) diff --git a/scripts/cleanup_datasets/remove_renamed_datasets_from_disk.py b/scripts/cleanup_datasets/remove_renamed_datasets_from_disk.py old mode 100644 new mode 100755 index fe52e9470b0..9c8f879e896 --- a/scripts/cleanup_datasets/remove_renamed_datasets_from_disk.py +++ b/scripts/cleanup_datasets/remove_renamed_datasets_from_disk.py @@ -10,10 +10,14 @@ assert sys.version_info[:2] >= ( 2, 4 ) def usage(prog) : print "usage: %s file" % prog - print "The input file should contain a list of files to be deleted, one per line. The" - print "full path must be specified and must begin with /var/opt/galaxy" - print "\nA log of files deleted is created in a file with the same name as that input" - print "but with .removed.log appended" + print """ +Removes a set of files from disk. The input file should contain a list of files +to be deleted, one per line. The full path must be specified and must begin +with /var/opt/galaxy. + +A log of files deleted is created in a file with the same name as that input but +with .removed.log appended. + """ def main(): if len(sys.argv) != 2 or sys.argv == "-h" or sys.argv == "--help" : diff --git a/scripts/cleanup_datasets/rename_purged_datasets.py b/scripts/cleanup_datasets/rename_purged_datasets.py old mode 100644 new mode 100755 diff --git a/scripts/cleanup_datasets/update_dataset_size.py b/scripts/cleanup_datasets/update_dataset_size.py old mode 100644 new mode 100755 diff --git a/scripts/cleanup_datasets/update_metadata.py b/scripts/cleanup_datasets/update_metadata.py old mode 100644 new mode 100755 From 0763cab37a818a9252a01a7aa387b1fecc07d17e Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Tue, 19 May 2015 21:53:45 +0200 Subject: [PATCH 3/5] Update the remainder of the python cleanup scripts --- .../cleanup_datasets/rename_purged_datasets.py | 15 +++++++++++++++ scripts/cleanup_datasets/update_dataset_size.py | 10 ++++++++++ scripts/cleanup_datasets/update_metadata.py | 11 +++++++++++ 3 files changed, 36 insertions(+) diff --git a/scripts/cleanup_datasets/rename_purged_datasets.py b/scripts/cleanup_datasets/rename_purged_datasets.py index 9b815909272..c1d214f9ce0 100755 --- a/scripts/cleanup_datasets/rename_purged_datasets.py +++ b/scripts/cleanup_datasets/rename_purged_datasets.py @@ -8,7 +8,22 @@ import sys, os assert sys.version_info[:2] >= ( 2, 4 ) +def usage(prog) : + print "usage: %s file" % prog + print """ +Marks a set of files as purged and renames them. The input file should contain a +list of files to be purged, one per line. The full path must be specified and +must begin with /var/opt/galaxy. +A log of files marked as purged is created in a file with the same name as that +input but with _purged appended. The resulting files can finally be removed from +disk with remove_renamed_datasets_from_disk.py, by supplying it with a list of +them. + """ + def main(): + if len(sys.argv) != 2 or sys.argv == "-h" or sys.argv == "--help" : + usage(sys.argv[0]) + sys.exit() infile = sys.argv[1] outfile = infile + ".renamed.log" out = open( outfile, 'w' ) diff --git a/scripts/cleanup_datasets/update_dataset_size.py b/scripts/cleanup_datasets/update_dataset_size.py index 260cc4694c7..f71f537b157 100755 --- a/scripts/cleanup_datasets/update_dataset_size.py +++ b/scripts/cleanup_datasets/update_dataset_size.py @@ -9,7 +9,17 @@ import galaxy.app assert sys.version_info[:2] >= ( 2, 4 ) +def usage(prog) : + print "usage: %s galaxy.ini" % prog + print """ +Updates the dataset.size column. Users are advised to backup the database before +running. + """ + def main(): + if len(sys.argv) != 1 or sys.argv[1] == "-h" or sys.argv[1] == "--help" : + usage(sys.argv[0]) + sys.exit() ini_file = sys.argv.pop(1) conf_parser = ConfigParser.ConfigParser( {'here':os.getcwd()} ) conf_parser.read( ini_file ) diff --git a/scripts/cleanup_datasets/update_metadata.py b/scripts/cleanup_datasets/update_metadata.py index 44b84b29a85..3b1f6ab2c34 100755 --- a/scripts/cleanup_datasets/update_metadata.py +++ b/scripts/cleanup_datasets/update_metadata.py @@ -13,7 +13,18 @@ import galaxy.datatypes.tabular assert sys.version_info[:2] >= ( 2, 4 ) +def usage(prog) : + print "usage: %s galaxy.ini" % prog + print """ +Updates the metadata in the database to match rev 1981. + +Remember to backup your database before running. + """ + def main(): + if len(sys.argv) != 1 or sys.argv[1] == "-h" or sys.argv[1] == "--help" : + usage(sys.argv[0]) + sys.exit() ini_file = sys.argv.pop(1) conf_parser = ConfigParser.ConfigParser({'here':os.getcwd()}) conf_parser.read(ini_file) From 0ff5a050d5f463ceb1bd11b6e35500add7f8d9c6 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Tue, 19 May 2015 21:55:36 +0200 Subject: [PATCH 4/5] Indentation fix --- scripts/cleanup_datasets/populate_uuid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cleanup_datasets/populate_uuid.py b/scripts/cleanup_datasets/populate_uuid.py index ebecde92afd..8a46cc8bbe6 100755 --- a/scripts/cleanup_datasets/populate_uuid.py +++ b/scripts/cleanup_datasets/populate_uuid.py @@ -29,7 +29,7 @@ Populates blank uuid fields in datasets with randomly generated values Going forward, these ids will be generated for all new datasets. This script fixes datasets that were generated before the change. -""" + """ def main(): if len(sys.argv) != 2 or sys.argv == "-h" or sys.argv == "--help" : From 00eb96923f7edb6947eb3e5b1ec3dd4783887e49 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Wed, 20 May 2015 13:01:40 +0200 Subject: [PATCH 5/5] Add a missing period in populate_uuid.py --- scripts/cleanup_datasets/populate_uuid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cleanup_datasets/populate_uuid.py b/scripts/cleanup_datasets/populate_uuid.py index 8a46cc8bbe6..81548d4e17d 100755 --- a/scripts/cleanup_datasets/populate_uuid.py +++ b/scripts/cleanup_datasets/populate_uuid.py @@ -25,7 +25,7 @@ assert sys.version_info[:2] >= ( 2, 4 ) def usage(prog) : print "usage: %s galaxy.ini" % prog print """ -Populates blank uuid fields in datasets with randomly generated values +Populates blank uuid fields in datasets with randomly generated values. Going forward, these ids will be generated for all new datasets. This script fixes datasets that were generated before the change.