From d2ac5a001c1f2d17034622c5ad9298899b370ed3 Mon Sep 17 00:00:00 2001 From: zipho Date: Fri, 19 Aug 2016 15:18:03 +0200 Subject: [PATCH 1/7] adding admin feature to allow user to turn on or off line ending conversion --- .../galaxy/controllers/library_common.py | 11 +++++++++ .../webapps/galaxy/library/common/common.mako | 23 +++++++++++++++++++ tools/data_source/upload.py | 17 ++++++++++++-- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/library_common.py b/lib/galaxy/webapps/galaxy/controllers/library_common.py index 9fe8f5b605c..bcfa4e8f149 100644 --- a/lib/galaxy/webapps/galaxy/controllers/library_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/library_common.py @@ -799,6 +799,10 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet replace_id = kwd.get( 'replace_id', None ) replace_dataset = None upload_option = kwd.get( 'upload_option', 'upload_file' ) + if kwd.get( 'files_0|uni_to_posix', False ): + to_posix_lines = kwd.get( 'files_0|to_posix_lines', '' ) + else: + to_posix_lines = kwd.get( 'to_posix_lines', '' ) if kwd.get( 'files_0|space_to_tab', False ): space_to_tab = kwd.get( 'files_0|space_to_tab', '' ) else: @@ -1027,6 +1031,7 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet history=history, widgets=widgets, template_id=template_id, + to_posix_lines=to_posix_lines, space_to_tab=space_to_tab, link_data_only=link_data_only, show_deleted=show_deleted, @@ -1143,6 +1148,7 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet uploaded_dataset.ext = None uploaded_dataset.file_type = file_type uploaded_dataset.dbkey = params.get( 'dbkey', None ) + uploaded_dataset.to_posix_lines = params.get('to_posix_lines', None) uploaded_dataset.space_to_tab = params.get( 'space_to_tab', None ) if in_folder: uploaded_dataset.in_folder = in_folder @@ -1269,6 +1275,10 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet replace_id = kwd.get( 'replace_id', None ) replace_dataset = None upload_option = kwd.get( 'upload_option', 'import_from_history' ) + if kwd.get( 'files_0|to_posix_lines', False ): + to_posix_lines = kwd.get( 'files_0|to_posix_lines', '' ) + else: + to_posix_lines = kwd.get( 'to_posix_lines', '' ) if kwd.get( 'files_0|space_to_tab', False ): space_to_tab = kwd.get( 'files_0|space_to_tab', '' ) else: @@ -1442,6 +1452,7 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet history=history, widgets=[], template_id=template_id, + to_posix_lines=to_posix_lines, space_to_tab=space_to_tab, link_data_only=link_data_only, show_deleted=show_deleted, diff --git a/templates/webapps/galaxy/library/common/common.mako b/templates/webapps/galaxy/library/common/common.mako index de475208667..d507f5fa297 100644 --- a/templates/webapps/galaxy/library/common/common.mako +++ b/templates/webapps/galaxy/library/common/common.mako @@ -267,6 +267,29 @@
+ +
+ +
+ <% + if to_posix_lines == 'true': + checked = ' checked' + else: + checked = '' + if upload_option == 'upload_file': + name = 'files_0|to_posix_lines' + else: + name = 'to_posix_lines' + uni2posix = 'No' % ( name, checked ) + %> + ${uni2posix} +
+
+ Use this option if you don't need your datasets to be converted from uni to posix line endings. +
+
diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index a123aa327f2..a95cba872f7 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -12,6 +12,7 @@ import shutil import sys import tempfile import zipfile +import cProfile from json import dumps, loads from six.moves.urllib.request import urlopen @@ -79,7 +80,19 @@ def parse_outputs( args ): rval[int( id )] = ( path, files_path ) return rval +def do_cprofile(func): + def profiled_func(*args, **kwargs): + profile = cProfile.Profile() + try: + profile.enable() + result = func(*args, **kwargs) + profile.disable() + return result + finally: + profile.print_stats(sort='time') + return profiled_func +@do_cprofile def add_file( dataset, registry, json_file, output_path ): data_type = None line_count = None @@ -297,9 +310,9 @@ def add_file( dataset, registry, json_file, output_path ): if dataset.to_posix_lines: tmpdir = output_adjacent_tmpdir( output_path ) tmp_prefix = 'data_id_%s_convert_' % dataset.dataset_id - if dataset.space_to_tab: + if dataset.space_to_tab and not dataset.to_posix_lines: line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place, tmp_dir=tmpdir, tmp_prefix=tmp_prefix ) - else: + elif(not dataset.to_posix_lines): line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place, tmp_dir=tmpdir, tmp_prefix=tmp_prefix ) if dataset.file_type == 'auto': ext = sniff.guess_ext( dataset.path, registry.sniff_order ) From bede2907b99951163b8cee69f79405abf4e44e1a Mon Sep 17 00:00:00 2001 From: zipho Date: Fri, 19 Aug 2016 16:01:32 +0200 Subject: [PATCH 2/7] added some changes to fix --- templates/webapps/galaxy/library/common/common.mako | 4 ++-- tools/data_source/upload.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/webapps/galaxy/library/common/common.mako b/templates/webapps/galaxy/library/common/common.mako index d507f5fa297..28b9664de1a 100644 --- a/templates/webapps/galaxy/library/common/common.mako +++ b/templates/webapps/galaxy/library/common/common.mako @@ -282,12 +282,12 @@ name = 'files_0|to_posix_lines' else: name = 'to_posix_lines' - uni2posix = 'No' % ( name, checked ) + uni2posix = 'Yes' % ( name, checked ) %> ${uni2posix}
- Use this option if you don't need your datasets to be converted from uni to posix line endings. + Use this option if you need your datasets to be converted from uni to posix line endings.
diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index a95cba872f7..838aa069a8f 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -310,9 +310,9 @@ def add_file( dataset, registry, json_file, output_path ): if dataset.to_posix_lines: tmpdir = output_adjacent_tmpdir( output_path ) tmp_prefix = 'data_id_%s_convert_' % dataset.dataset_id - if dataset.space_to_tab and not dataset.to_posix_lines: + if dataset.space_to_tab: line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place, tmp_dir=tmpdir, tmp_prefix=tmp_prefix ) - elif(not dataset.to_posix_lines): + else: line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place, tmp_dir=tmpdir, tmp_prefix=tmp_prefix ) if dataset.file_type == 'auto': ext = sniff.guess_ext( dataset.path, registry.sniff_order ) From 8d8ea2158b5a7ba763985f369719f18c7b4230cc Mon Sep 17 00:00:00 2001 From: zipho Date: Wed, 24 Aug 2016 23:14:54 +0200 Subject: [PATCH 3/7] remove cProfile import --- tools/data_source/upload.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index 838aa069a8f..0802f6a3dd0 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -12,7 +12,6 @@ import shutil import sys import tempfile import zipfile -import cProfile from json import dumps, loads from six.moves.urllib.request import urlopen From c53585e8b5227e9d9f3536fa1434ae74acc0ef68 Mon Sep 17 00:00:00 2001 From: zipho Date: Fri, 26 Aug 2016 17:44:35 +0200 Subject: [PATCH 4/7] added cProfile --- tools/data_source/upload.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index 0802f6a3dd0..838aa069a8f 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -12,6 +12,7 @@ import shutil import sys import tempfile import zipfile +import cProfile from json import dumps, loads from six.moves.urllib.request import urlopen From 66e3f8540f4dff918a90c2d02eca4a3f46f7fbdd Mon Sep 17 00:00:00 2001 From: zipho Date: Fri, 26 Aug 2016 17:52:43 +0200 Subject: [PATCH 5/7] removed traces of cProfile and use --- tools/data_source/upload.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index 838aa069a8f..4a43c21f21a 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -12,7 +12,6 @@ import shutil import sys import tempfile import zipfile -import cProfile from json import dumps, loads from six.moves.urllib.request import urlopen @@ -80,19 +79,6 @@ def parse_outputs( args ): rval[int( id )] = ( path, files_path ) return rval -def do_cprofile(func): - def profiled_func(*args, **kwargs): - profile = cProfile.Profile() - try: - profile.enable() - result = func(*args, **kwargs) - profile.disable() - return result - finally: - profile.print_stats(sort='time') - return profiled_func - -@do_cprofile def add_file( dataset, registry, json_file, output_path ): data_type = None line_count = None From bc1be68bb6f0eef840465dd2b578136d0094ae20 Mon Sep 17 00:00:00 2001 From: zipho Date: Wed, 31 Aug 2016 15:07:48 +0200 Subject: [PATCH 6/7] added the extra blank line --- tools/data_source/upload.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index 4a43c21f21a..a123aa327f2 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -79,6 +79,7 @@ def parse_outputs( args ): rval[int( id )] = ( path, files_path ) return rval + def add_file( dataset, registry, json_file, output_path ): data_type = None line_count = None From e85cd284d099af16329d6592b46dc3d6ce37dba4 Mon Sep 17 00:00:00 2001 From: zipho Date: Wed, 31 Aug 2016 15:32:02 +0200 Subject: [PATCH 7/7] fixed the line tabs --- lib/galaxy/webapps/galaxy/controllers/library_common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/library_common.py b/lib/galaxy/webapps/galaxy/controllers/library_common.py index bcfa4e8f149..5d680fbe702 100644 --- a/lib/galaxy/webapps/galaxy/controllers/library_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/library_common.py @@ -800,9 +800,9 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet replace_dataset = None upload_option = kwd.get( 'upload_option', 'upload_file' ) if kwd.get( 'files_0|uni_to_posix', False ): - to_posix_lines = kwd.get( 'files_0|to_posix_lines', '' ) + to_posix_lines = kwd.get( 'files_0|to_posix_lines', '' ) else: - to_posix_lines = kwd.get( 'to_posix_lines', '' ) + to_posix_lines = kwd.get( 'to_posix_lines', '' ) if kwd.get( 'files_0|space_to_tab', False ): space_to_tab = kwd.get( 'files_0|space_to_tab', '' ) else: