From 13995757f26a8819638332b3541fa68811c86cfe Mon Sep 17 00:00:00 2001 From: Zipho Mashologu Date: Sat, 10 Jun 2017 22:50:31 +0200 Subject: [PATCH 1/9] added the zip format (datatype) for neostore. --- config/datatypes_conf.xml.sample | 4 + .../neostorezip_to_neostore_converter.xml | 11 ++ lib/galaxy/datatypes/neo4j.py | 107 ++++++++++++++---- 3 files changed, 98 insertions(+), 24 deletions(-) create mode 100644 lib/galaxy/datatypes/converters/neostorezip_to_neostore_converter.xml diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index fb19bebea86..e6ab3e05d71 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -585,6 +585,9 @@ + + + @@ -676,6 +679,7 @@ + diff --git a/lib/galaxy/datatypes/converters/neostorezip_to_neostore_converter.xml b/lib/galaxy/datatypes/converters/neostorezip_to_neostore_converter.xml new file mode 100644 index 00000000000..e2eb598bca9 --- /dev/null +++ b/lib/galaxy/datatypes/converters/neostorezip_to_neostore_converter.xml @@ -0,0 +1,11 @@ + diff --git a/lib/galaxy/datatypes/neo4j.py b/lib/galaxy/datatypes/neo4j.py index f3b97b9e85a..fe28702826d 100644 --- a/lib/galaxy/datatypes/neo4j.py +++ b/lib/galaxy/datatypes/neo4j.py @@ -7,8 +7,15 @@ import shutil import os from galaxy.datatypes.data import Data +from galaxy.datatypes.binary import Binary from galaxy.datatypes.images import Html from galaxy.util import FILENAME_VALID_CHARS +from galaxy.util.checkers import ( + is_bz2, + is_gzip +) + +from galaxy.datatypes.metadata import MetadataElement, MetadataParameter gal_Log = logging.getLogger(__name__) verbose = True @@ -20,11 +27,13 @@ class Neo4j(Html): derived from html - composite datatype elements stored in extra files path """ + def generate_primary_file(self, dataset=None): """ This is called only at upload to write the html file cannot rename the datasets here - they come with the default unfortunately """ + # self.regenerate_primary_file(dataset) rval = [ 'Files for Composite Dataset (%s)

\ This composite dataset is composed of the following files:

    ' % ( @@ -33,7 +42,8 @@ class Neo4j(Html): opt_text = '' if composite_file.optional: opt_text = ' (optional)' - rval.append('
  • %s%s' % (composite_name, composite_name, opt_text)) + rval.append('
  • %s%s' % + (composite_name, composite_name, opt_text)) rval.append('
') return "\n".join(rval) @@ -41,6 +51,13 @@ class Neo4j(Html): """Returns the mime type of the datatype""" return 'text/html' + def sniff(self, filename): + compressed = is_gzip(filename) or is_bz2(filename) + if compressed and not isinstance(self, Binary): + return False + else: + return True + def set_peek(self, dataset, is_multi_byte=False): """Set the peek and blurb text""" if not dataset.dataset.purged: @@ -67,22 +84,27 @@ class Neo4j(Html): trans.log_event("Display dataset id: %s" % str(data.id)) # the target directory name - neo4j_dir_name = '/dataset_{}_files/neo4jdb'.format(data.dataset.id) - dir_name = str(os.path.dirname(trans.app.object_store.get_filename(data.dataset))) + neo4j_dir_name + neo4j_dir_name = '/dataset_{}_files/neo4jdb'.format( + data.dataset.id) + dir_name = str(os.path.dirname( + trans.app.object_store.get_filename(data.dataset))) + neo4j_dir_name # generate unique filename for this dataset - fname = ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in data.name)[0:150] + fname = ''.join( + c in FILENAME_VALID_CHARS and c or '_' for c in data.name)[0:150] # zip the target directory (dir_name) using the fname shutil.make_archive(fname, 'zip', dir_name) download_zip = fname + '.zip' # setup headers for the download - trans.response.headers['Content-Length'] = int(os.stat(download_zip).st_size) + trans.response.headers['Content-Length'] = int( + os.stat(download_zip).st_size) # force octet-stream so Safari doesn't append mime extensions to filename trans.response.set_content_type("application/octet-stream") trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % \ - (data.hid, download_zip, "zip") + (data.hid, + download_zip, "zip") return open(download_zip) @@ -99,36 +121,73 @@ class Neo4jDB(Neo4j, Data): self.add_composite_file('neostore.counts.db.a', is_binary=True) self.add_composite_file('neostore.counts.db.b', is_binary=True) self.add_composite_file('neostore.labeltokenstore.db', is_binary=True) - self.add_composite_file('neostore.labeltokenstore.db.id', is_binary=True) - self.add_composite_file('neostore.labeltokenstore.db.names', is_binary=True) - self.add_composite_file('neostore.labeltokenstore.db.names.id', is_binary=True) + self.add_composite_file( + 'neostore.labeltokenstore.db.id', is_binary=True) + self.add_composite_file( + 'neostore.labeltokenstore.db.names', is_binary=True) + self.add_composite_file( + 'neostore.labeltokenstore.db.names.id', is_binary=True) self.add_composite_file('neostore.nodestore.db', is_binary=True) self.add_composite_file('neostore.nodestore.db.id', is_binary=True) self.add_composite_file('neostore.nodestore.db.labels', is_binary=True) - self.add_composite_file('neostore.nodestore.db.labels.id', is_binary=True) + self.add_composite_file( + 'neostore.nodestore.db.labels.id', is_binary=True) self.add_composite_file('neostore.propertystore.db', is_binary=True) self.add_composite_file('neostore.propertystore.db.id', is_binary=True) - self.add_composite_file('neostore.propertystore.db.arrays', is_binary=True) - self.add_composite_file('neostore.propertystore.db.arrays.id', is_binary=True) - self.add_composite_file('neostore.propertystore.db.index', is_binary=True) - self.add_composite_file('neostore.propertystore.db.index.id', is_binary=True) - self.add_composite_file('neostore.propertystore.db.index.keys', is_binary=True) - self.add_composite_file('neostore.propertystore.db.index.keys.id', is_binary=True) - self.add_composite_file('neostore.propertystore.db.strings', is_binary=True) - self.add_composite_file('neostore.propertystore.db.strings.id', is_binary=True) + self.add_composite_file( + 'neostore.propertystore.db.arrays', is_binary=True) + self.add_composite_file( + 'neostore.propertystore.db.arrays.id', is_binary=True) + self.add_composite_file( + 'neostore.propertystore.db.index', is_binary=True) + self.add_composite_file( + 'neostore.propertystore.db.index.id', is_binary=True) + self.add_composite_file( + 'neostore.propertystore.db.index.keys', is_binary=True) + self.add_composite_file( + 'neostore.propertystore.db.index.keys.id', is_binary=True) + self.add_composite_file( + 'neostore.propertystore.db.strings', is_binary=True) + self.add_composite_file( + 'neostore.propertystore.db.strings.id', is_binary=True) - self.add_composite_file('neostore.relationshipgroupstore.db', is_binary=True) - self.add_composite_file('neostore.relationshipgroupstore.db.id', is_binary=True) - self.add_composite_file('neostore.relationshipstore.db', is_binary=True) - self.add_composite_file('neostore.relationshipstore.db.id', is_binary=True) - self.add_composite_file('neostore.relationshiptypestore.db.names', is_binary=True) - self.add_composite_file('neostore.relationshiptypestore.db.names.id', is_binary=True) + self.add_composite_file( + 'neostore.relationshipgroupstore.db', is_binary=True) + self.add_composite_file( + 'neostore.relationshipgroupstore.db.id', is_binary=True) + self.add_composite_file( + 'neostore.relationshipstore.db', is_binary=True) + self.add_composite_file( + 'neostore.relationshipstore.db.id', is_binary=True) + self.add_composite_file( + 'neostore.relationshiptypestore.db.names', is_binary=True) + self.add_composite_file( + 'neostore.relationshiptypestore.db.names.id', is_binary=True) self.add_composite_file('neostore.schemastore.db', is_binary=True) self.add_composite_file('neostore.schemastore.db.id', is_binary=True) self.add_composite_file('neostore.transaction.db.0', is_binary=True) +class Neo4jDBzip(Neo4j, Data): + """Class for neo4jDB database files.""" + MetadataElement(name='reference_name', default='neostore', desc='Reference Name', + readonly=True, visible=True, set_in_upload=True, no_value='neostore') + MetadataElement(name="neostore_zip", default=None, desc="Neostore zip", + readonly=True, visible=True, set_in_upload=True, no_value=None, optional=True) + + file_ext = "noestore.zip" + allow_datatype_change = False + + def __init__(self, **kwd): + Data.__init__(self, **kwd) + self.add_composite_file('%s.zip', description='neostore zip', substitute_name_with_metadata='reference_name', + is_binary=True) + + +Binary.register_sniffable_binary_format( + "neostore.zip", "neostore.zip", Neo4jDBzip) + if __name__ == '__main__': import doctest doctest.testmod(sys.modules[__name__]) From 6e9a66556f0e172eb0cace2cf7266525880e587c Mon Sep 17 00:00:00 2001 From: Zipho Mashologu Date: Sun, 11 Jun 2017 00:11:30 +0200 Subject: [PATCH 2/9] fixing unit test failure --- lib/galaxy/datatypes/neo4j.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/datatypes/neo4j.py b/lib/galaxy/datatypes/neo4j.py index fe28702826d..282579a35a0 100644 --- a/lib/galaxy/datatypes/neo4j.py +++ b/lib/galaxy/datatypes/neo4j.py @@ -15,7 +15,7 @@ from galaxy.util.checkers import ( is_gzip ) -from galaxy.datatypes.metadata import MetadataElement, MetadataParameter +from galaxy.datatypes.metadata import MetadataElement gal_Log = logging.getLogger(__name__) verbose = True @@ -51,13 +51,6 @@ class Neo4j(Html): """Returns the mime type of the datatype""" return 'text/html' - def sniff(self, filename): - compressed = is_gzip(filename) or is_bz2(filename) - if compressed and not isinstance(self, Binary): - return False - else: - return True - def set_peek(self, dataset, is_multi_byte=False): """Set the peek and blurb text""" if not dataset.dataset.purged: @@ -174,7 +167,7 @@ class Neo4jDBzip(Neo4j, Data): MetadataElement(name='reference_name', default='neostore', desc='Reference Name', readonly=True, visible=True, set_in_upload=True, no_value='neostore') MetadataElement(name="neostore_zip", default=None, desc="Neostore zip", - readonly=True, visible=True, set_in_upload=True, no_value=None, optional=True) + readonly=True, visible=True, set_in_upload=True, no_value=None, optional=True) file_ext = "noestore.zip" allow_datatype_change = False From 013b03ab76e7b45d6cd5779fa196e64c2f086a82 Mon Sep 17 00:00:00 2001 From: Zipho Mashologu Date: Sun, 11 Jun 2017 08:05:29 +0200 Subject: [PATCH 3/9] fixing travis build failures --- config/datatypes_conf.xml.sample | 2 +- lib/galaxy/datatypes/neo4j.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 3c11bc8779f..0a860cb2588 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -697,7 +697,6 @@ - @@ -757,5 +756,6 @@ + diff --git a/lib/galaxy/datatypes/neo4j.py b/lib/galaxy/datatypes/neo4j.py index 282579a35a0..b2f19f15cc8 100644 --- a/lib/galaxy/datatypes/neo4j.py +++ b/lib/galaxy/datatypes/neo4j.py @@ -10,10 +10,6 @@ from galaxy.datatypes.data import Data from galaxy.datatypes.binary import Binary from galaxy.datatypes.images import Html from galaxy.util import FILENAME_VALID_CHARS -from galaxy.util.checkers import ( - is_bz2, - is_gzip -) from galaxy.datatypes.metadata import MetadataElement From 8731ec764635afa9fdb831d5829b1825424eec76 Mon Sep 17 00:00:00 2001 From: Zipho Mashologu Date: Sun, 11 Jun 2017 22:27:28 +0200 Subject: [PATCH 4/9] making neostore.counts.db.a and neostore.counts.db.b optional --- lib/galaxy/datatypes/neo4j.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/datatypes/neo4j.py b/lib/galaxy/datatypes/neo4j.py index b2f19f15cc8..35440ba835a 100644 --- a/lib/galaxy/datatypes/neo4j.py +++ b/lib/galaxy/datatypes/neo4j.py @@ -107,8 +107,8 @@ class Neo4jDB(Neo4j, Data): Data.__init__(self, **kwd) self.add_composite_file('neostore', is_binary=True) self.add_composite_file('neostore.id', is_binary=True) - self.add_composite_file('neostore.counts.db.a', is_binary=True) - self.add_composite_file('neostore.counts.db.b', is_binary=True) + self.add_composite_file('neostore.counts.db.a', optional=True, is_binary=True) + self.add_composite_file('neostore.counts.db.b', optional=True, is_binary=True) self.add_composite_file('neostore.labeltokenstore.db', is_binary=True) self.add_composite_file( 'neostore.labeltokenstore.db.id', is_binary=True) From 40edf26ceba2e6484e3b856cfd0487b9b728a8c4 Mon Sep 17 00:00:00 2001 From: Zipho Mashologu Date: Tue, 13 Jun 2017 14:51:23 +0200 Subject: [PATCH 5/9] changes to the version --- .../neostorezip_to_neostore_converter.xml | 5 +- lib/galaxy/datatypes/neo4j.py | 9 +-- tools/data_source/upload.py | 68 ++++++++++--------- 3 files changed, 44 insertions(+), 38 deletions(-) diff --git a/lib/galaxy/datatypes/converters/neostorezip_to_neostore_converter.xml b/lib/galaxy/datatypes/converters/neostorezip_to_neostore_converter.xml index e2eb598bca9..74486fe30d4 100644 --- a/lib/galaxy/datatypes/converters/neostorezip_to_neostore_converter.xml +++ b/lib/galaxy/datatypes/converters/neostorezip_to_neostore_converter.xml @@ -1,5 +1,8 @@