From 3a6e5f3eddb7bcca6399a637169b11ac124615b0 Mon Sep 17 00:00:00 2001 From: Alanamosse Date: Thu, 28 Mar 2019 12:50:46 +0100 Subject: [PATCH 01/18] add a gis datatype class and a new composite datatype shp --- config/datatypes_conf.xml.sample | 1 + lib/galaxy/datatypes/gis.py | 127 +++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 lib/galaxy/datatypes/gis.py diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 8ab6211f09a..9302b64135b 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -718,6 +718,7 @@ + + + + From 89996dcd935ecc141e406ff19795eccf654cc5f2 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Fri, 26 Apr 2019 22:34:20 +0200 Subject: [PATCH 09/18] remove redundant datatype def --- config/datatypes_conf.xml.sample | 1 - lib/galaxy/datatypes/gis.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index b1a305b90d9..f423cdd8167 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -727,7 +727,6 @@ - diff --git a/lib/galaxy/datatypes/gis.py b/lib/galaxy/datatypes/gis.py index 92b1dfe91cd..47f4349522e 100644 --- a/lib/galaxy/datatypes/gis.py +++ b/lib/galaxy/datatypes/gis.py @@ -19,6 +19,7 @@ class Shapefile(Binary): allow_datatype_change = False def __init__(self, **kwd): + Binary.__init__(self, **kwd) self.add_composite_file('shapefile.shp', description='Geometry File (shp)', is_binary=True, optional=False) self.add_composite_file('shapefile.shx', description='Geometry index File (shx)', is_binary=True, optional=False) self.add_composite_file('shapefile.dbf', description='Columnar attributes for each shape (dbf)', is_binary=True, optional=False) From 75c53409a0091c86ce05432d9900f205d0bdd8b5 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Mon, 29 Apr 2019 09:50:00 +0200 Subject: [PATCH 10/18] add datatype and tool to the tests --- test/functional/tools/sample_datatypes_conf.xml | 1 + test/functional/tools/samples_tool_conf.xml | 1 + 2 files changed, 2 insertions(+) diff --git a/test/functional/tools/sample_datatypes_conf.xml b/test/functional/tools/sample_datatypes_conf.xml index 6b9c6d5326c..ba5dbdcf9cb 100644 --- a/test/functional/tools/sample_datatypes_conf.xml +++ b/test/functional/tools/sample_datatypes_conf.xml @@ -42,6 +42,7 @@ + diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index 28798011dcb..c40a545f144 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -52,6 +52,7 @@ + + From 1d2126a84e9a672fabc3a9ab2de3972bb7ab620b Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Mon, 29 Apr 2019 19:09:18 +0200 Subject: [PATCH 13/18] enable input composite data --- test/functional/tools/composite_shapefile.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/functional/tools/composite_shapefile.xml b/test/functional/tools/composite_shapefile.xml index 6822519b540..ded8fcde7b1 100644 --- a/test/functional/tools/composite_shapefile.xml +++ b/test/functional/tools/composite_shapefile.xml @@ -17,7 +17,7 @@ - + @@ -28,12 +28,12 @@ - + From bec0d87f4cfa67104896e56a54adbe7d7247cf3e Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Tue, 30 Apr 2019 00:12:28 +0200 Subject: [PATCH 14/18] ordering matters --- test/functional/tools/composite_shapefile.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/functional/tools/composite_shapefile.xml b/test/functional/tools/composite_shapefile.xml index ded8fcde7b1..1e2c41b5e46 100644 --- a/test/functional/tools/composite_shapefile.xml +++ b/test/functional/tools/composite_shapefile.xml @@ -4,7 +4,7 @@ '$output' && + python -c "import fiona; shape = fiona.open('$input.extra_files_path/shapefile.shp'); print(shape.schema)" > '$output' && cp '$input.extra_files_path'/* '$output_shapefile.extra_files_path' ]]> @@ -18,21 +18,21 @@ - - + + - - - + + + From 7303ee677782da825f21968bccd0cc11c29e9a54 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 30 Apr 2019 00:26:59 +0200 Subject: [PATCH 15/18] Open tempfile in writing mode for python3 compatibility --- lib/galaxy/jobs/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index e3460c38fc8..8f90c94c2d1 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1337,7 +1337,7 @@ class JobWrapper(HasResourceParameters): # Handle composite datatypes of auto_primary_file type if dataset.datatype.composite_type == 'auto_primary_file' and not dataset.has_data(): try: - with NamedTemporaryFile() as temp_fh: + with NamedTemporaryFile(mode='w') as temp_fh: temp_fh.write(dataset.datatype.generate_primary_file(dataset)) temp_fh.flush() self.object_store.update_from_file(dataset.dataset, file_name=temp_fh.name, create=True) From 102b619c436e489460fb7b8ddae639dde418619c Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Tue, 30 Apr 2019 09:47:06 +0200 Subject: [PATCH 16/18] make tests stricter --- lib/galaxy/datatypes/gis.py | 2 +- test/functional/tools/composite_shapefile.xml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/datatypes/gis.py b/lib/galaxy/datatypes/gis.py index 47f4349522e..501f77907ac 100644 --- a/lib/galaxy/datatypes/gis.py +++ b/lib/galaxy/datatypes/gis.py @@ -48,7 +48,7 @@ class Shapefile(Binary): rval.append('
  • %s (%s)%s
    • ' % (fn, fn, composite_file.get('description'), opt_text)) else: rval.append('
  • %s%s
    • ' % (fn, fn, opt_text)) - rval.append('') + rval.append('\n') return "\n".join(rval) def set_peek(self, dataset, is_multi_byte=False): diff --git a/test/functional/tools/composite_shapefile.xml b/test/functional/tools/composite_shapefile.xml index 1e2c41b5e46..34caff62a46 100644 --- a/test/functional/tools/composite_shapefile.xml +++ b/test/functional/tools/composite_shapefile.xml @@ -29,10 +29,10 @@ - - - - + + + + From 5000d290edafa2889b7856caa15961c42a35d9ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Tue, 30 Apr 2019 13:41:47 +0200 Subject: [PATCH 17/18] remove logging module --- lib/galaxy/datatypes/gis.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/galaxy/datatypes/gis.py b/lib/galaxy/datatypes/gis.py index 501f77907ac..c3362a7bc87 100644 --- a/lib/galaxy/datatypes/gis.py +++ b/lib/galaxy/datatypes/gis.py @@ -6,8 +6,6 @@ import logging from galaxy.datatypes.binary import Binary -log = logging.getLogger(__name__) - class Shapefile(Binary): """ The Shapefile data format: From cf6f10431315eec21df4dd0235a221fde2c350f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Tue, 30 Apr 2019 13:50:26 +0200 Subject: [PATCH 18/18] add remove the logging, thanks Marius --- lib/galaxy/datatypes/gis.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/galaxy/datatypes/gis.py b/lib/galaxy/datatypes/gis.py index c3362a7bc87..276de5cb97c 100644 --- a/lib/galaxy/datatypes/gis.py +++ b/lib/galaxy/datatypes/gis.py @@ -2,8 +2,6 @@ GIS classes """ -import logging - from galaxy.datatypes.binary import Binary