[19.05] Add shapefile datatype (#7819)

[19.05] Add shapefile datatype
2026-09-24 16:30:27 +08:00 · 2019-05-02 10:18:40 +02:00
parent 6230404623 cf6f104313
commit c545aaf4d4
6 changed files with 107 additions and 1 deletions
@@ -322,6 +322,7 @@
    </datatype>
    <datatype extension="scf" type="galaxy.datatypes.binary:Scf" mimetype="application/octet-stream" display_in_upload="true" description="A binary sequence file in 'scf' format with a '.scf' file extension.  You must manually select this 'File Format' when uploading the file." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Scf"/>
    <datatype extension="sequences" type="galaxy.datatypes.assembly:Sequences" display_in_upload="false"/>
+    <datatype extension="shp" type="galaxy.datatypes.gis:Shapefile" mimetype="application/octet-stream" display_in_upload="true"/>
    <datatype extension="snpeffdb" type="galaxy.datatypes.text:SnpEffDb" display_in_upload="true"/>
    <datatype extension="snpsiftdbnsfp" type="galaxy.datatypes.text:SnpSiftDbNSFP" display_in_upload="true"/>
    <datatype extension="dbnsfp.tabular" type="galaxy.datatypes.tabular:Tabular" subclass="true" display_in_upload="true">
@@ -0,0 +1,64 @@
+"""
+GIS classes
+"""
+
+from galaxy.datatypes.binary import Binary
+
+
+class Shapefile(Binary):
+    """ The Shapefile data format:
+            For more information please see http://en.wikipedia.org/wiki/Shapefile
+    """
+
+    composite_type = 'auto_primary_file'
+    file_ext = "shp"
+    allow_datatype_change = False
+
+    def __init__(self, **kwd):
+        Binary.__init__(self, **kwd)
+        self.add_composite_file('shapefile.shp', description='Geometry File (shp)', is_binary=True, optional=False)
+        self.add_composite_file('shapefile.shx', description='Geometry index File (shx)', is_binary=True, optional=False)
+        self.add_composite_file('shapefile.dbf', description='Columnar attributes for each shape (dbf)', is_binary=True, optional=False)
+        # optional
+        self.add_composite_file('shapefile.prj', description='Projection description (prj)', is_binary=False, optional=True)
+        self.add_composite_file('shapefile.sbn', description='Spatial index of the features (sbn)', is_binary=True, optional=True)
+        self.add_composite_file('shapefile.sbx', description='Spatial index of the features (sbx)', is_binary=True, optional=True)
+        self.add_composite_file('shapefile.fbn', description='Read only spatial index of the features (fbn)', is_binary=True, optional=True)
+        self.add_composite_file('shapefile.fbx', description='Read only spatial index of the features (fbx)', is_binary=True, optional=True)
+        self.add_composite_file('shapefile.ain', description='Attribute index of the active fields in a table (ain)', is_binary=True, optional=True)
+        self.add_composite_file('shapefile.aih', description='Attribute index of the active fields in a table (aih)', is_binary=True, optional=True)
+        self.add_composite_file('shapefile.atx', description='Attribute index for the dbf file (atx)', is_binary=True, optional=True)
+        self.add_composite_file('shapefile.ixs', description='Geocoding index (ixs)', is_binary=True, optional=True)
+        self.add_composite_file('shapefile.mxs', description='Geocoding index in ODB format (mxs)', is_binary=True, optional=True)
+        self.add_composite_file('shapefile.shp.xml', description='Geospatial metadata in XML format (xml)', is_binary=False, optional=True)
+
+    def generate_primary_file(self, dataset=None):
+        rval = ['<html><head><title>Shapefile Galaxy Composite Dataset</title></head><p/>']
+        rval.append('<div>This composite dataset is composed of the following files:<p/><ul>')
+        for composite_name, composite_file in self.get_composite_files(dataset=dataset).items():
+            fn = composite_name
+            opt_text = ''
+            if composite_file.optional:
+                opt_text = ' (optional)'
+            if composite_file.get('description'):
+                rval.append('<li><a href="%s" type="application/binary">%s (%s)</a>%s</li>' % (fn, fn, composite_file.get('description'), opt_text))
+            else:
+                rval.append('<li><a href="%s" type="application/binary">%s</a>%s</li>' % (fn, fn, opt_text))
+        rval.append('</ul></div></html>\n')
+        return "\n".join(rval)
+
+    def set_peek(self, dataset, is_multi_byte=False):
+        """Set the peek and blurb text."""
+        if not dataset.dataset.purged:
+            dataset.peek = "Shapefile data"
+            dataset.blurb = "Shapefile data"
+        else:
+            dataset.peek = "file does not exist"
+            dataset.blurb = "file purged from disk"
+
+    def display_peek(self, dataset):
+        """Create HTML content, used for displaying peek."""
+        try:
+            return dataset.peek
+        except Exception:
+            return "Shapefile data"
@@ -1348,7 +1348,7 @@ class JobWrapper(HasResourceParameters):
                # Handle composite datatypes of auto_primary_file type
                if dataset.datatype.composite_type == 'auto_primary_file' and not dataset.has_data():
                    try:
-                        with NamedTemporaryFile() as temp_fh:
+                        with NamedTemporaryFile(mode='w') as temp_fh:
                            temp_fh.write(dataset.datatype.generate_primary_file(dataset))
                            temp_fh.flush()
                            self.object_store.update_from_file(dataset.dataset, file_name=temp_fh.name, create=True)
@@ -0,0 +1,39 @@
+<tool id="shapefile_composite" name="composite shapefile tool" version="1.0.0">
+    <requirements>
+        <requirement type="package" version="1.8.6">fiona</requirement>
+    </requirements>
+    <command><![CDATA[
+        mkdir '$output_shapefile.extra_files_path' &&
+        python -c "import fiona; shape = fiona.open('$input.extra_files_path/shapefile.shp'); print(shape.schema)" > '$output' &&
+        cp '$input.extra_files_path'/* '$output_shapefile.extra_files_path'
+    ]]>
+    </command>
+    <inputs>
+        <param name="input" type="data" format="shp" label="Shapefile" help="Used in GIS applications"/>
+    </inputs>
+    <outputs>
+        <data name="output" format="txt" label="${tool.name} on ${on_string}: Shapefile test" />
+        <data name="output_shapefile" format="shp" label="${tool.name} on ${on_string}: Shapefile test - shp" />
+    </outputs>
+    <tests>
+        <test>
+            <param name="input" value="" ftype="shp" >
+                <composite_data value="shapefile/shapefile.shp"/>
+                <composite_data value="shapefile/shapefile.shx"/>
+                <composite_data value="shapefile/shapefile.dbf"/>
+                <composite_data value="shapefile/shapefile.prj"/>
+            </param>
+            <output name="output">
+                <assert_contents>
+                    <has_text text="{'properties': OrderedDict([('FIPS', 'str:2'), ('ISO2', 'str:2'), ('ISO3', 'str:3'), ('UN', 'int:3')," />
+                </assert_contents>
+            </output>
+            <output name="output_shapefile" file="shapefile/output.html">
+                <extra_files type="file" name="shapefile.shp" value="shapefile/shapefile.shp" md5="ae4648a8a6be706e910e46607f261317" />
+                <extra_files type="file" name="shapefile.shx" value="shapefile/shapefile.shx" md5="6321ef787b7826afe06e9cf3eb20e656" />
+                <extra_files type="file" name="shapefile.dbf" value="shapefile/shapefile.dbf" md5="d65d87e1570510af8a81d7d174ea361a" />
+                <extra_files type="file" name="shapefile.prj" value="shapefile/shapefile.prj" lines_diff="2" />
+            </output>
+        </test>
+    </tests>
+</tool>
@@ -42,6 +42,7 @@
    <datatype extension="data_manager_json" type="galaxy.datatypes.text:Json" mimetype="application/json" subclass="true" display_in_upload="false"/>
    <datatype extension="data" type="galaxy.datatypes.data:Data" mimetype="application/octet-stream" max_optional_metadata_filesize="1048576" />
    <datatype extension="binary" type="galaxy.datatypes.binary:Binary" mimetype="application/octet-stream" max_optional_metadata_filesize="1048576" />
+    <datatype extension="shp" type="galaxy.datatypes.gis:Shapefile" mimetype="application/octet-stream" display_in_upload="true"/>
    <datatype extension="zip" type="galaxy.datatypes.binary:CompressedZipArchive" display_in_upload="true">
    </datatype>
    <datatype extension="tar" type="galaxy.datatypes.binary:CompressedArchive" subclass="true" display_in_upload="true">
@@ -52,6 +52,7 @@
  <tool file="detect_errors_aggressive.xml" />
  <tool file="md5sum.xml" />
  <tool file="checksum.xml" />
+  <tool file="composite_shapefile.xml" />
  <tool file="is_valid_xml.xml" />
  <!--
  TODO: Figure out why this transiently fails on Jenkins.