opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5062 from nsoranzo/remove_wchartype_dep

Browse Source 
Remove wchartype dependency and is_multi_byte [1/2]
This commit is contained in:
John Chilton
2017-11-28 15:54:00 -05:00
committed by GitHub
parent 68795e415c 59f44cf2be
commit 8d7f344140
31 changed files with 760 additions and 281 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/datatypes/annotation.py
+1 -1
View File
@@ -14,7 +14,7 @@ class SnapHmm(Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "SNAP HMM model"
else:
dataset.peek = 'file does not exist'
lib/galaxy/datatypes/binary.py
+1 -1
View File
@@ -1088,7 +1088,7 @@ class TwoBit(Binary):
dataset.peek = "Binary TwoBit format nucleotide file"
dataset.blurb = nice_size(dataset.get_size())
else:
return super(TwoBit, self).set_peek(dataset, is_multi_byte)
return super(TwoBit, self).set_peek(dataset)
def display_peek(self, dataset):
try:
lib/galaxy/datatypes/blast.py
+1 -1
View File
@@ -53,7 +53,7 @@ class BlastXml(GenericXml):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = 'NCBI Blast XML data'
else:
dataset.peek = 'file does not exist'
lib/galaxy/datatypes/constructive_solid_geometry.py
+2 -2
View File
@@ -102,7 +102,7 @@ class Ply(object):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "Faces: %s, Vertices: %s" % (str(dataset.metadata.face), str(dataset.metadata.vertex))
else:
dataset.peek = 'File does not exist'
@@ -429,7 +429,7 @@ class Vtk(object):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = self.get_blurb(dataset)
else:
dataset.peek = 'File does not exist'
lib/galaxy/datatypes/data.py
+16 -23
View File
@@ -195,7 +195,12 @@ class Data(object):
max_optional_metadata_filesize = property(get_max_optional_metadata_filesize, set_max_optional_metadata_filesize)
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
"""
Set the peek and blurb text
:param is_multi_byte: deprecated
:type is_multi_byte: bool
"""
if not dataset.dataset.purged:
dataset.peek = ''
dataset.blurb = 'data'
@@ -838,7 +843,7 @@ class Text(Data):
"""
if not dataset.dataset.purged:
# The file must exist on disk for the get_file_peek() method
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte, WIDTH=WIDTH, skipchars=skipchars, line_wrap=line_wrap)
dataset.peek = get_file_peek(dataset.file_name, WIDTH=WIDTH, skipchars=skipchars, line_wrap=line_wrap)
if line_count is None:
# See if line_count is stored in the metadata
if dataset.metadata.data_lines:
@@ -1046,7 +1051,10 @@ def get_test_fname(fname):
def get_file_peek(file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skipchars=None, line_wrap=True):
"""
Returns the first LINE_COUNT lines wrapped to WIDTH
Returns the first LINE_COUNT lines wrapped to WIDTH.
:param is_multi_byte: deprecated
:type is_multi_byte: bool
>>> fname = get_test_fname('4.bed')
>>> get_file_peek(fname, LINE_COUNT=1)
@@ -1061,20 +1069,12 @@ def get_file_peek(file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skipc
skipchars = []
lines = []
count = 0
file_type = None
data_checked = False
with compression_utils.get_fileobj(file_name, "U") as temp:
while count < LINE_COUNT:
line = temp.readline(WIDTH)
if line and not is_multi_byte and not data_checked:
# See if we have a compressed or binary file
for char in line:
if ord(char) > 128:
file_type = 'binary'
break
data_checked = True
if file_type == 'binary':
break
try:
line = temp.readline(WIDTH)
except UnicodeDecodeError:
return "binary file"
if not line_wrap:
if line.endswith('\n'):
line = line[:-1]
@@ -1091,11 +1091,4 @@ def get_file_peek(file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skipc
if not skip_line:
lines.append(line)
count += 1
if file_type == 'binary':
text = "%s file" % file_type
else:
try:
text = util.unicodify('\n'.join(lines))
except UnicodeDecodeError:
text = "binary/unknown file"
return text
return '\n'.join(lines)
lib/galaxy/datatypes/graph.py
+2 -2
View File
@@ -27,7 +27,7 @@ class Xgmml(xml.GenericXml):
Set the peek and blurb text
"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'XGMML data'
else:
dataset.peek = 'file does not exist'
@@ -73,7 +73,7 @@ class Sif(tabular.Tabular):
Set the peek and blurb text
"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'SIF data'
else:
dataset.peek = 'file does not exist'
lib/galaxy/datatypes/images.py
+2 -9
View File
@@ -7,7 +7,6 @@ import zipfile
from six.moves.urllib.parse import quote_plus
from galaxy.datatypes.binary import Binary
from galaxy.datatypes.sniff import get_headers
from galaxy.datatypes.text import Html as HtmlFromText
from galaxy.util import nice_size
from galaxy.util.image_util import check_image_type
@@ -160,14 +159,8 @@ class Pdf(Image):
def sniff(self, filename):
"""Determine if the file is in pdf format."""
headers = get_headers(filename, None, 1)
try:
if headers[0][0].startswith("%PDF"):
return True
else:
return False
except IndexError:
return False
with open(filename, 'rb') as fh:
return fh.read(4) == b"%PDF"
Binary.register_sniffable_binary_format("pdf", "pdf", Pdf)
lib/galaxy/datatypes/molecules.py
+8 -12
View File
@@ -61,12 +61,11 @@ class GenericMolFile(data.Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
if (dataset.metadata.number_of_molecules == 1):
dataset.blurb = "1 molecule"
else:
dataset.blurb = "%s molecules" % dataset.metadata.number_of_molecules
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
@@ -471,7 +470,7 @@ class PHAR(GenericMolFile):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "pharmacophore"
else:
dataset.peek = 'file does not exist'
@@ -524,7 +523,7 @@ class PDB(GenericMolFile):
if not dataset.dataset.purged:
atom_numbers = count_special_lines("^ATOM", dataset.file_name)
hetatm_numbers = count_special_lines("^HETATM", dataset.file_name)
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "%s atoms and %s HET-atoms" % (atom_numbers, hetatm_numbers)
else:
dataset.peek = 'file does not exist'
@@ -575,7 +574,7 @@ class PDBQT(GenericMolFile):
if not dataset.dataset.purged:
root_numbers = count_special_lines("^ROOT", dataset.file_name)
branch_numbers = count_special_lines("^BRANCH", dataset.file_name)
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "%s roots and %s branches" % (root_numbers, branch_numbers)
else:
dataset.peek = 'file does not exist'
@@ -587,7 +586,7 @@ class grd(data.Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "grids for docking"
else:
dataset.peek = 'file does not exist'
@@ -621,12 +620,11 @@ class InChI(Tabular):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
if (dataset.metadata.number_of_molecules == 1):
dataset.blurb = "1 molecule"
else:
dataset.blurb = "%s molecules" % dataset.metadata.number_of_molecules
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
@@ -666,12 +664,11 @@ class SMILES(Tabular):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
if dataset.metadata.number_of_molecules == 1:
dataset.blurb = "1 molecule"
else:
dataset.blurb = "%s molecules" % dataset.metadata.number_of_molecules
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
@@ -727,12 +724,11 @@ class CML(GenericXml):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
if (dataset.metadata.number_of_molecules == 1):
dataset.blurb = "1 molecule"
else:
dataset.blurb = "%s molecules" % dataset.metadata.number_of_molecules
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
lib/galaxy/datatypes/msa.py
+3 -5
View File
@@ -17,7 +17,7 @@ class Hmmer(Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "HMMER Database"
else:
dataset.peek = 'file does not exist'
@@ -104,12 +104,11 @@ class Stockholm_1_0(Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
if (dataset.metadata.number_of_models == 1):
dataset.blurb = "1 alignment"
else:
dataset.blurb = "%s alignments" % dataset.metadata.number_of_models
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disc'
@@ -187,12 +186,11 @@ class MauveXmfa(Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
if (dataset.metadata.number_of_models == 1):
dataset.blurb = "1 alignment"
else:
dataset.blurb = "%s alignments" % dataset.metadata.number_of_models
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disc'
lib/galaxy/datatypes/plant_tribes.py
+13 -13
View File
@@ -23,7 +23,7 @@ class Smat(Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "ESTScan scores matrices"
else:
dataset.peek = 'file does not exist'
@@ -125,7 +125,7 @@ class PlantTribesKsComponents(Tabular):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
if (dataset.metadata.number_comp == 1):
dataset.blurb = "1 significant component"
else:
@@ -159,7 +159,7 @@ class PlantTribesOrtho(PlantTribes):
file_ext = "ptortho"
def set_peek(self, dataset, is_multi_byte=False):
super(PlantTribesOrtho, self).set_peek(dataset, is_multi_byte=is_multi_byte)
super(PlantTribesOrtho, self).set_peek(dataset)
dataset.blurb = "Proteins orthogroup fasta files: %d items" % dataset.metadata.num_files
@@ -171,7 +171,7 @@ class PlantTribesOrthoCodingSequence(PlantTribes):
file_ext = "ptorthocs"
def set_peek(self, dataset, is_multi_byte=False):
super(PlantTribesOrthoCodingSequence, self).set_peek(dataset, is_multi_byte=is_multi_byte)
super(PlantTribesOrthoCodingSequence, self).set_peek(dataset)
dataset.blurb = "Protein and coding sequences orthogroup fasta files: %d items" % dataset.metadata.num_files
@@ -182,7 +182,7 @@ class PlantTribesTargetedGeneFamilies(PlantTribes):
file_ext = "pttgf"
def set_peek(self, dataset, is_multi_byte=False):
super(PlantTribesTargetedGeneFamilies, self).set_peek(dataset, is_multi_byte=is_multi_byte)
super(PlantTribesTargetedGeneFamilies, self).set_peek(dataset)
dataset.blurb = "Targeted gene families"
@@ -194,7 +194,7 @@ class PlantTribesPhylogeneticTree(PlantTribes):
file_ext = "pttree"
def set_peek(self, dataset, is_multi_byte=False):
super(PlantTribesPhylogeneticTree, self).set_peek(dataset, is_multi_byte=is_multi_byte)
super(PlantTribesPhylogeneticTree, self).set_peek(dataset)
dataset.blurb = "Phylogenetic trees: %d items" % dataset.metadata.num_files
@@ -205,7 +205,7 @@ class PlantTribesPhylip(PlantTribes):
file_ext = "ptphylip"
def set_peek(self, dataset, is_multi_byte=False):
super(PlantTribesPhylip, self).set_peek(dataset, is_multi_byte=is_multi_byte)
super(PlantTribesPhylip, self).set_peek(dataset)
dataset.blurb = "Orthogroup phylip multiple sequence alignments: %d items" % dataset.metadata.num_files
@@ -216,7 +216,7 @@ class PlantTribesMultipleSequenceAlignment(PlantTribes):
file_ext = "ptalign"
def set_peek(self, dataset, is_multi_byte=False):
super(PlantTribesMultipleSequenceAlignment, self).set_peek(dataset, is_multi_byte=is_multi_byte)
super(PlantTribesMultipleSequenceAlignment, self).set_peek(dataset)
dataset.blurb = "Proteins orthogroup alignments: %d items" % dataset.metadata.num_files
@@ -227,7 +227,7 @@ class PlantTribesMultipleSequenceAlignmentCodonAlignment(PlantTribes):
file_ext = "ptalignca"
def set_peek(self, dataset, is_multi_byte=False):
super(PlantTribesMultipleSequenceAlignmentCodonAlignment, self).set_peek(dataset, is_multi_byte=is_multi_byte)
super(PlantTribesMultipleSequenceAlignmentCodonAlignment, self).set_peek(dataset)
dataset.blurb = "Protein and coding sequences orthogroup alignments: %d items" % dataset.metadata.num_files
@@ -238,7 +238,7 @@ class PlantTribesMultipleSequenceAlignmentTrimmed(PlantTribes):
file_ext = "ptaligntrimmed"
def set_peek(self, dataset, is_multi_byte=False):
super(PlantTribesMultipleSequenceAlignmentTrimmed, self).set_peek(dataset, is_multi_byte=is_multi_byte)
super(PlantTribesMultipleSequenceAlignmentTrimmed, self).set_peek(dataset)
dataset.blurb = "Trimmed proteins orthogroup alignments: %d items" % dataset.metadata.num_files
@@ -249,7 +249,7 @@ class PlantTribesMultipleSequenceAlignmentTrimmedCodonAlignment(PlantTribes):
file_ext = "ptaligntrimmedca"
def set_peek(self, dataset, is_multi_byte=False):
super(PlantTribesMultipleSequenceAlignmentTrimmedCodonAlignment, self).set_peek(dataset, is_multi_byte=is_multi_byte)
super(PlantTribesMultipleSequenceAlignmentTrimmedCodonAlignment, self).set_peek(dataset)
dataset.blurb = "Trimmed protein and coding sequences orthogroup alignments: %d items" % dataset.metadata.num_files
@@ -260,7 +260,7 @@ class PlantTribesMultipleSequenceAlignmentFiltered(PlantTribes):
file_ext = "ptalignfiltered"
def set_peek(self, dataset, is_multi_byte=False):
super(PlantTribesMultipleSequenceAlignmentFiltered, self).set_peek(dataset, is_multi_byte=is_multi_byte)
super(PlantTribesMultipleSequenceAlignmentFiltered, self).set_peek(dataset)
dataset.blurb = "Filtered proteins orthogroup alignments: %d items" % dataset.metadata.num_files
@@ -271,5 +271,5 @@ class PlantTribesMultipleSequenceAlignmentFilteredCodonAlignment(PlantTribes):
file_ext = "ptalignfilteredca"
def set_peek(self, dataset, is_multi_byte=False):
super(PlantTribesMultipleSequenceAlignmentFilteredCodonAlignment, self).set_peek(dataset, is_multi_byte=is_multi_byte)
super(PlantTribesMultipleSequenceAlignmentFilteredCodonAlignment, self).set_peek(dataset)
dataset.blurb = "Filtered protein and coding sequences orthogroup alignments: %d items" % dataset.metadata.num_files
lib/galaxy/datatypes/proteomics.py
+5 -5
View File
@@ -114,7 +114,7 @@ class ProteomicsXml(GenericXml):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = self.blurb
else:
dataset.peek = 'file does not exist'
@@ -221,7 +221,7 @@ class Mgf(Text):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'mgf Mascot Generic Format'
else:
dataset.peek = 'file does not exist'
@@ -249,7 +249,7 @@ class MascotDat(Text):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'mascotdat Mascot Search Results'
else:
dataset.peek = 'file does not exist'
@@ -334,7 +334,7 @@ class SPLibNoIndex(Text):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'Spectral Library without index files'
else:
dataset.peek = 'file does not exist'
@@ -374,7 +374,7 @@ class SPLib(Msp):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'splib Spectral Library Format'
else:
dataset.peek = 'file does not exist'
lib/galaxy/datatypes/registry.py
+8 -7
View File
@@ -314,13 +314,14 @@ class Registry(object):
def append_to_sniff_order():
# Just in case any supported data types are not included in the config's sniff_order section.
for ext, datatype in self.datatypes_by_extension.items():
included = False
for atype in self.sniff_order:
if isinstance(atype, datatype.__class__):
included = True
break
if not included:
self.sniff_order.append(datatype)
if hasattr(datatype, 'sniff'):
included = False
for atype in self.sniff_order:
if isinstance(atype, datatype.__class__):
included = True
break
if not included:
self.sniff_order.append(datatype)
append_to_sniff_order()
def _load_build_sites(self, root):
lib/galaxy/datatypes/sequence.py
+3 -3
View File
@@ -61,7 +61,7 @@ class SequenceSplitLocations(data.Text):
try:
parsed_data = json.load(open(dataset.file_name))
# dataset.peek = json.dumps(data, sort_keys=True, indent=4)
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = '%d sections' % len(parsed_data['sections'])
except Exception:
dataset.peek = 'Not FQTOC file'
@@ -112,7 +112,7 @@ class Sequence(data.Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
if dataset.metadata.sequences:
dataset.blurb = "%s sequences" % util.commaify(str(dataset.metadata.sequences))
else:
@@ -861,7 +861,7 @@ class Maf(Alignment):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
# The file must exist on disk for the get_file_peek() method
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
if dataset.metadata.blocks:
dataset.blurb = "%s blocks" % util.commaify(str(dataset.metadata.blocks))
else:
lib/galaxy/datatypes/sniff.py
+23 -36
View File
@@ -20,7 +20,6 @@ from galaxy.datatypes.binary import Binary
from galaxy.util import (
compression_utils,
multi_byte,
unicodify
)
from galaxy.util.checkers import (
check_binary,
@@ -204,17 +203,11 @@ def convert_newlines_sep2tabs(fname, in_place=True, patt="\\s+", tmp_dir=None, t
return (i + 1, temp_name)
def iter_headers(fname, sep, count=60, is_multi_byte=False, comment_designator=None):
def iter_headers(fname, sep, count=60, comment_designator=None):
with compression_utils.get_fileobj(fname) as in_file:
idx = 0
for line in in_file:
line = line.rstrip('\n\r')
if is_multi_byte:
# TODO: fix this - sep is never found in line
line = unicodify(line, 'utf-8')
sep = sep.encode('utf-8')
if comment_designator is not None and comment_designator != '':
comment_designator = comment_designator.encode('utf-8')
if comment_designator is not None and comment_designator != '' and line.startswith(comment_designator):
continue
yield line.split(sep)
@@ -223,22 +216,22 @@ def iter_headers(fname, sep, count=60, is_multi_byte=False, comment_designator=N
break
def get_headers(fname, sep, count=60, is_multi_byte=False, comment_designator=None):
def get_headers(fname, sep, count=60, comment_designator=None):
"""
Returns a list with the first 'count' lines split by 'sep', ignoring lines
starting with 'comment_designator'
>>> fname = get_test_fname('complete.bed')
>>> get_headers(fname,'\\t')
[['chr7', '127475281', '127491632', 'NM_000230', '0', '+', '127486022', '127488767', '0', '3', '29,172,3225,', '0,10713,13126,'], ['chr7', '127486011', '127488900', 'D49487', '0', '+', '127486022', '127488767', '0', '2', '155,490,', '0,2399']]
>>> get_headers(fname,'\\t') == [['chr7', '127475281', '127491632', 'NM_000230', '0', '+', '127486022', '127488767', '0', '3', '29,172,3225,', '0,10713,13126,'], ['chr7', '127486011', '127488900', 'D49487', '0', '+', '127486022', '127488767', '0', '2', '155,490,', '0,2399']]
True
>>> fname = get_test_fname('test.gff')
>>> get_headers(fname, '\\t', count=5, comment_designator='#')
[[''], ['chr7', 'bed2gff', 'AR', '26731313', '26731437', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731491', '26731536', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731541', '26731649', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731659', '26731841', '.', '+', '.', 'score']]
>>> get_headers(fname, '\\t', count=5, comment_designator='#') == [[''], ['chr7', 'bed2gff', 'AR', '26731313', '26731437', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731491', '26731536', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731541', '26731649', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731659', '26731841', '.', '+', '.', 'score']]
True
"""
return list(iter_headers(fname=fname, sep=sep, count=count, is_multi_byte=is_multi_byte, comment_designator=comment_designator))
return list(iter_headers(fname=fname, sep=sep, count=count, comment_designator=comment_designator))
def is_column_based(fname, sep='\t', skip=0, is_multi_byte=False):
def is_column_based(fname, sep='\t', skip=0):
"""
Checks whether the file is column based with respect to a separator
(defaults to tab separator).
@@ -266,7 +259,10 @@ def is_column_based(fname, sep='\t', skip=0, is_multi_byte=False):
>>> is_column_based(fname)
True
"""
headers = get_headers(fname, sep, is_multi_byte=is_multi_byte)
try:
headers = get_headers(fname, sep)
except UnicodeDecodeError:
return False
count = 0
if not headers:
return False
@@ -284,7 +280,7 @@ def is_column_based(fname, sep='\t', skip=0, is_multi_byte=False):
return True
def guess_ext(fname, sniff_order, is_multi_byte=False):
def guess_ext(fname, sniff_order):
"""
Returns an extension that can be used in the datatype factory to
generate a data for the 'fname' file
@@ -393,6 +389,9 @@ def guess_ext(fname, sniff_order, is_multi_byte=False):
>>> fname = get_test_fname('biom2_sparse_otu_table_hdf5.biom')
>>> guess_ext(fname, sniff_order)
'biom2'
>>> fname = get_test_fname('454Score.pdf')
>>> guess_ext(fname, sniff_order)
'pdf'
"""
file_ext = None
for datatype in sniff_order:
@@ -414,28 +413,16 @@ def guess_ext(fname, sniff_order, is_multi_byte=False):
# to tsv but it doesn't have a sniffer - is TSV was sniffed just check
# if it is an okay tabular and use that instead.
if file_ext == 'tsv':
if is_column_based(fname, '\t', 1, is_multi_byte=is_multi_byte):
if is_column_based(fname, '\t', 1):
file_ext = 'tabular'
if file_ext is not None:
return file_ext
headers = get_headers(fname, None)
is_binary = False
if is_multi_byte:
is_binary = False
else:
for hdr in headers:
for char in hdr:
# old behavior had 'char' possibly having length > 1,
# need to determine when/if this occurs
is_binary = util.is_binary(char)
if is_binary:
break
if is_binary:
break
if is_binary:
try:
get_headers(fname, None)
except UnicodeDecodeError:
return 'data' # default binary data type file extension
if is_column_based(fname, '\t', 1, is_multi_byte=is_multi_byte):
if is_column_based(fname, '\t', 1):
return 'tabular' # default tabular data type file extension
return 'txt' # default text data type file extension
@@ -489,14 +476,14 @@ def handle_compressed_file(filename, datatypes_registry, ext='auto'):
return is_valid, ext
def handle_uploaded_dataset_file(filename, datatypes_registry, ext='auto', is_multi_byte=False):
def handle_uploaded_dataset_file(filename, datatypes_registry, ext='auto'):
is_valid, ext = handle_compressed_file(filename, datatypes_registry, ext=ext)
if not is_valid:
raise InappropriateDatasetContentError('The compressed uploaded file contains inappropriate content.')
if ext in AUTO_DETECT_EXTENSIONS:
ext = guess_ext(filename, sniff_order=datatypes_registry.sniff_order, is_multi_byte=is_multi_byte)
ext = guess_ext(filename, sniff_order=datatypes_registry.sniff_order)
if check_binary(filename):
if not Binary.is_ext_unsniffable(ext) and not datatypes_registry.get_datatype_by_extension(ext).sniff(filename):
lib/galaxy/datatypes/tabular.py
+2 -2
View File
@@ -51,7 +51,7 @@ class TabularData(data.Text):
raise NotImplementedError
def set_peek(self, dataset, line_count=None, is_multi_byte=False, WIDTH=256, skipchars=None):
super(TabularData, self).set_peek(dataset, line_count=line_count, is_multi_byte=is_multi_byte, WIDTH=WIDTH, skipchars=skipchars, line_wrap=False)
super(TabularData, self).set_peek(dataset, line_count=line_count, WIDTH=WIDTH, skipchars=skipchars, line_wrap=False)
if dataset.metadata.comment_lines:
dataset.blurb = "%s, %s comments" % (dataset.blurb, util.commaify(str(dataset.metadata.comment_lines)))
@@ -825,7 +825,7 @@ class Eland(Tabular):
- LANE, TILEm X, Y, INDEX, READ_NO, SEQ, QUAL, POSITION, *STRAND, FILT must be correct
- We will only check that up to the first 5 alignments are correctly formatted.
"""
with compression_utils.get_fileobj(filename, gzip_only=True) as fh:
with compression_utils.get_fileobj(filename, compressed_formats=['gzip']) as fh:
count = 0
while True:
line = fh.readline()
lib/galaxy/datatypes/test/454Score.pdf
+545
View File
@@ -0,0 +1,545 @@
%PDF-1.1
%�â�ã�Ï�Ó\r
1 0 obj
<<
/CreationDate (D:20080403110358)
/ModDate (D:20080403110358)
/Title (R Graphics Output)
/Producer (R 2.6.2)
/Creator (R)
>>
endobj
2 0 obj
<<
/Type /Catalog
/Pages 3 0 R
>>
endobj
5 0 obj
<<
/Type /Font
/Subtype /Type1
/Name /F1
/BaseFont /ZapfDingbats
>>
endobj
6 0 obj
<<
/Type /Page
/Parent 3 0 R
/Contents 7 0 R
/Resources 4 0 R
>>
endobj
7 0 obj
<<
/Length 8 0 R
>>
stream
q
Q q 59.04 73.44 342.72 299.52 re W n
0.000 0.000 0.000 RG
2.25 w
[] 0 d
1 J
1 j
10.00 M
73.40 149.79 m 86.76 149.79 l S
0.75 w
[ 3.00 5.00] 0 d
80.08 100.85 m 80.08 149.79 l S
80.08 296.61 m 80.08 263.98 l S
0.75 w
[] 0 d
76.74 100.85 m 83.42 100.85 l S
76.74 296.61 m 83.42 296.61 l S
73.40 149.79 m
86.76 149.79 l
86.76 263.98 l
73.40 263.98 l
73.40 149.79 l
S
2.25 w
[] 0 d
90.11 280.30 m 103.47 280.30 l S
0.75 w
[ 3.00 5.00] 0 d
96.79 263.98 m 96.79 280.30 l S
96.79 296.61 m 96.79 296.61 l S
0.75 w
[] 0 d
93.45 263.98 m 100.13 263.98 l S
93.45 296.61 m 100.13 296.61 l S
90.11 280.30 m
103.47 280.30 l
103.47 296.61 l
90.11 296.61 l
90.11 280.30 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 93.82 342.96 Tm (l) Tj 0 Tr
2.25 w
[] 0 d
ET
106.81 280.30 m 120.17 280.30 l S
0.75 w
[ 3.00 5.00] 0 d
113.49 263.98 m 113.49 263.98 l S
113.49 280.30 m 113.49 280.30 l S
0.75 w
[] 0 d
110.15 263.98 m 116.83 263.98 l S
110.15 280.30 m 116.83 280.30 l S
106.81 263.98 m
120.17 263.98 l
120.17 280.30 l
106.81 280.30 l
106.81 263.98 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 110.53 179.82 Tm (l) Tj 0 Tr
2.25 w
[] 0 d
ET
123.51 247.67 m 136.87 247.67 l S
0.75 w
[ 3.00 5.00] 0 d
130.19 247.67 m 130.19 247.67 l S
130.19 247.67 m 130.19 247.67 l S
0.75 w
[] 0 d
126.85 247.67 m 133.53 247.67 l S
126.85 247.67 m 133.53 247.67 l S
123.51 247.67 m
136.87 247.67 l
136.87 247.67 l
123.51 247.67 l
123.51 247.67 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 127.23 261.39 Tm (l) Tj 0 Tr
2.25 w
[] 0 d
ET
140.21 280.30 m 153.57 280.30 l S
0.75 w
[ 3.00 5.00] 0 d
146.89 231.36 m 146.89 247.67 l S
146.89 361.87 m 146.89 296.61 l S
0.75 w
[] 0 d
143.55 231.36 m 150.23 231.36 l S
143.55 361.87 m 150.23 361.87 l S
140.21 247.67 m
153.57 247.67 l
153.57 296.61 l
140.21 296.61 l
140.21 247.67 l
S
2.25 w
[] 0 d
156.91 280.30 m 170.27 280.30 l S
0.75 w
[ 3.00 5.00] 0 d
163.59 198.73 m 163.59 231.36 l S
163.59 280.30 m 163.59 280.30 l S
0.75 w
[] 0 d
160.25 198.73 m 166.93 198.73 l S
160.25 280.30 m 166.93 280.30 l S
156.91 231.36 m
170.27 231.36 l
170.27 280.30 l
156.91 280.30 l
156.91 231.36 l
S
2.25 w
[] 0 d
173.61 280.30 m 186.98 280.30 l S
0.75 w
[ 3.00 5.00] 0 d
180.29 247.67 m 180.29 263.98 l S
180.29 280.30 m 180.29 280.30 l S
0.75 w
[] 0 d
176.95 247.67 m 183.64 247.67 l S
176.95 280.30 m 183.64 280.30 l S
173.61 263.98 m
186.98 263.98 l
186.98 280.30 l
173.61 280.30 l
173.61 263.98 l
S
2.25 w
[] 0 d
190.32 247.67 m 203.68 247.67 l S
0.75 w
[ 3.00 5.00] 0 d
197.00 247.67 m 197.00 247.67 l S
197.00 263.98 m 197.00 263.98 l S
0.75 w
[] 0 d
193.66 247.67 m 200.34 247.67 l S
193.66 263.98 m 200.34 263.98 l S
190.32 247.67 m
203.68 247.67 l
203.68 263.98 l
190.32 263.98 l
190.32 247.67 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 194.03 294.02 Tm (l) Tj 0 Tr
2.25 w
[] 0 d
ET
207.02 263.98 m 220.38 263.98 l S
0.75 w
[ 3.00 5.00] 0 d
213.70 247.67 m 213.70 247.67 l S
213.70 263.98 m 213.70 263.98 l S
0.75 w
[] 0 d
210.36 247.67 m 217.04 247.67 l S
210.36 263.98 m 217.04 263.98 l S
207.02 247.67 m
220.38 247.67 l
220.38 263.98 l
207.02 263.98 l
207.02 247.67 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 210.74 342.96 Tm (l) Tj 0 Tr
2.25 w
[] 0 d
ET
223.72 263.98 m 237.08 263.98 l S
0.75 w
[ 3.00 5.00] 0 d
230.40 263.98 m 230.40 263.98 l S
230.40 280.30 m 230.40 280.30 l S
0.75 w
[] 0 d
227.06 263.98 m 233.74 263.98 l S
227.06 280.30 m 233.74 280.30 l S
223.72 263.98 m
237.08 263.98 l
237.08 280.30 l
223.72 280.30 l
223.72 263.98 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 227.44 163.51 Tm (l) Tj 0 Tr
2.25 w
[] 0 d
ET
240.42 280.30 m 253.78 280.30 l S
0.75 w
[ 3.00 5.00] 0 d
247.10 263.98 m 247.10 263.98 l S
247.10 280.30 m 247.10 280.30 l S
0.75 w
[] 0 d
243.76 263.98 m 250.44 263.98 l S
243.76 280.30 m 250.44 280.30 l S
240.42 263.98 m
253.78 263.98 l
253.78 280.30 l
240.42 280.30 l
240.42 263.98 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 244.14 179.82 Tm (l) Tj 0 Tr
2.25 w
[] 0 d
ET
257.12 263.98 m 270.48 263.98 l S
0.75 w
[ 3.00 5.00] 0 d
263.80 247.67 m 263.80 247.67 l S
263.80 280.30 m 263.80 280.30 l S
0.75 w
[] 0 d
260.46 247.67 m 267.14 247.67 l S
260.46 280.30 m 267.14 280.30 l S
257.12 247.67 m
270.48 247.67 l
270.48 280.30 l
257.12 280.30 l
257.12 247.67 l
S
2.25 w
[] 0 d
273.82 247.67 m 287.19 247.67 l S
0.75 w
[ 3.00 5.00] 0 d
280.51 84.53 m 280.51 182.42 l S
280.51 296.61 m 280.51 296.61 l S
0.75 w
[] 0 d
277.16 84.53 m 283.85 84.53 l S
277.16 296.61 m 283.85 296.61 l S
273.82 182.42 m
287.19 182.42 l
287.19 296.61 l
273.82 296.61 l
273.82 182.42 l
S
2.25 w
[] 0 d
290.53 280.30 m 303.89 280.30 l S
0.75 w
[ 3.00 5.00] 0 d
297.21 280.30 m 297.21 280.30 l S
297.21 280.30 m 297.21 280.30 l S
0.75 w
[] 0 d
293.87 280.30 m 300.55 280.30 l S
293.87 280.30 m 300.55 280.30 l S
290.53 280.30 m
303.89 280.30 l
303.89 280.30 l
290.53 280.30 l
290.53 280.30 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 294.25 294.02 Tm (l) Tj 0 Tr
/F1 1 Tf 1 Tr 7.48 0 0 7.48 294.25 228.76 Tm (l) Tj 0 Tr
2.25 w
[] 0 d
ET
307.23 263.98 m 320.59 263.98 l S
0.75 w
[ 3.00 5.00] 0 d
313.91 247.67 m 313.91 247.67 l S
313.91 280.30 m 313.91 280.30 l S
0.75 w
[] 0 d
310.57 247.67 m 317.25 247.67 l S
310.57 280.30 m 317.25 280.30 l S
307.23 247.67 m
320.59 247.67 l
320.59 280.30 l
307.23 280.30 l
307.23 247.67 l
S
2.25 w
[] 0 d
323.93 231.36 m 337.29 231.36 l S
0.75 w
[ 3.00 5.00] 0 d
330.61 198.73 m 330.61 215.04 l S
330.61 231.36 m 330.61 231.36 l S
0.75 w
[] 0 d
327.27 198.73 m 333.95 198.73 l S
327.27 231.36 m 333.95 231.36 l S
323.93 215.04 m
337.29 215.04 l
337.29 231.36 l
323.93 231.36 l
323.93 215.04 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 327.65 261.39 Tm (l) Tj 0 Tr
2.25 w
[] 0 d
ET
340.63 263.98 m 353.99 263.98 l S
0.75 w
[ 3.00 5.00] 0 d
347.31 263.98 m 347.31 263.98 l S
347.31 263.98 m 347.31 263.98 l S
0.75 w
[] 0 d
343.97 263.98 m 350.65 263.98 l S
343.97 263.98 m 350.65 263.98 l S
340.63 263.98 m
353.99 263.98 l
353.99 263.98 l
340.63 263.98 l
340.63 263.98 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 344.35 179.82 Tm (l) Tj 0 Tr
/F1 1 Tf 1 Tr 7.48 0 0 7.48 344.35 277.70 Tm (l) Tj 0 Tr
2.25 w
[] 0 d
ET
357.33 263.98 m 370.69 263.98 l S
0.75 w
[ 3.00 5.00] 0 d
364.01 247.67 m 364.01 247.67 l S
364.01 280.30 m 364.01 280.30 l S
0.75 w
[] 0 d
360.67 247.67 m 367.35 247.67 l S
360.67 280.30 m 367.35 280.30 l S
357.33 247.67 m
370.69 247.67 l
370.69 280.30 l
357.33 280.30 l
357.33 247.67 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 361.05 81.94 Tm (l) Tj 0 Tr
2.25 w
[] 0 d
ET
374.04 263.98 m 387.40 263.98 l S
0.75 w
[ 3.00 5.00] 0 d
380.72 263.98 m 380.72 263.98 l S
380.72 296.61 m 380.72 296.61 l S
0.75 w
[] 0 d
377.38 263.98 m 384.06 263.98 l S
377.38 296.61 m 384.06 296.61 l S
374.04 263.98 m
387.40 263.98 l
387.40 296.61 l
374.04 296.61 l
374.04 263.98 l
S
BT
/F1 1 Tf 1 Tr 7.48 0 0 7.48 377.75 163.51 Tm (l) Tj 0 Tr
ET
Q q
0.000 0.000 0.000 RG
0.75 w
[] 0 d
1 J
1 j
10.00 M
59.04 133.47 m 59.04 296.61 l S
59.04 133.47 m 51.84 133.47 l S
59.04 215.04 m 51.84 215.04 l S
59.04 296.61 m 51.84 296.61 l S
BT
0.000 0.000 0.000 rg
/F2 1 Tf 0.00 12.00 -12.00 0.00 41.76 126.80 Tm (20) Tj
/F2 1 Tf 0.00 12.00 -12.00 0.00 41.76 208.37 Tm (25) Tj
/F2 1 Tf 0.00 12.00 -12.00 0.00 41.76 289.94 Tm (30) Tj
ET
Q q
BT
0.000 0.000 0.000 rg
/F3 1 Tf 14.00 0.00 -0.00 14.00 147.76 397.45 Tm (boxplot of quality scores) Tj
/F2 1 Tf 12.00 0.00 -0.00 12.00 130.36 18.72 Tm (position within read \(% of total length\)) Tj
ET
Q q
0.000 0.000 0.000 RG
0.75 w
[] 0 d
1 J
1 j
10.00 M
59.04 73.44 m
401.76 73.44 l
401.76 372.96 l
59.04 372.96 l
59.04 73.44 l
S
63.38 73.44 m 380.72 73.44 l S
63.38 73.44 m 63.38 66.24 l S
80.08 73.44 m 80.08 66.24 l S
96.79 73.44 m 96.79 66.24 l S
113.49 73.44 m 113.49 66.24 l S
130.19 73.44 m 130.19 66.24 l S
146.89 73.44 m 146.89 66.24 l S
163.59 73.44 m 163.59 66.24 l S
180.29 73.44 m 180.29 66.24 l S
197.00 73.44 m 197.00 66.24 l S
213.70 73.44 m 213.70 66.24 l S
230.40 73.44 m 230.40 66.24 l S
247.10 73.44 m 247.10 66.24 l S
263.80 73.44 m 263.80 66.24 l S
280.51 73.44 m 280.51 66.24 l S
297.21 73.44 m 297.21 66.24 l S
313.91 73.44 m 313.91 66.24 l S
330.61 73.44 m 330.61 66.24 l S
347.31 73.44 m 347.31 66.24 l S
364.01 73.44 m 364.01 66.24 l S
380.72 73.44 m 380.72 66.24 l S
BT
0.000 0.000 0.000 rg
/F2 1 Tf 12.00 0.00 -0.00 12.00 60.05 47.52 Tm (0) Tj
/F2 1 Tf 12.00 0.00 -0.00 12.00 76.75 47.52 Tm (5) Tj
/F2 1 Tf 12.00 0.00 -0.00 12.00 106.82 47.52 Tm (15) Tj
/F2 1 Tf 12.00 0.00 -0.00 12.00 140.22 47.52 Tm (25) Tj
/F2 1 Tf 12.00 0.00 -0.00 12.00 173.62 47.52 Tm (35) Tj
/F2 1 Tf 12.00 0.00 -0.00 12.00 207.03 47.52 Tm (45) Tj
/F2 1 Tf 12.00 0.00 -0.00 12.00 240.43 47.52 Tm (55) Tj
/F2 1 Tf 12.00 0.00 -0.00 12.00 273.83 47.52 Tm (65) Tj
/F2 1 Tf 12.00 0.00 -0.00 12.00 307.24 47.52 Tm (75) Tj
/F2 1 Tf 12.00 0.00 -0.00 12.00 340.64 47.52 Tm (85) Tj
/F2 1 Tf 12.00 0.00 -0.00 12.00 374.04 47.52 Tm (95) Tj
ET
Q
endstream
endobj
8 0 obj
8714
endobj
3 0 obj
<<
/Type /Pages
/Kids [
6 0 R
]
/Count 1
/MediaBox [0 0 432 432]
>>
endobj
4 0 obj
<<
/ProcSet [/PDF /Text]
/Font << /F1 5 0 R /F2 10 0 R /F3 11 0 R >>
/ExtGState << >>
>>
endobj
9 0 obj
<<
/Type /Encoding
/BaseEncoding /WinAnsiEncoding
/Differences [ 45/minus 96/quoteleft
144/dotlessi /grave /acute /circumflex /tilde /macron /breve /dotaccent
/dieresis /.notdef /ring /cedilla /.notdef /hungarumlaut /ogonek /caron /space]
>>
endobj
10 0 obj <<
/Type /Font
/Subtype /Type1
/Name /F2
/BaseFont /Helvetica
/Encoding 9 0 R
>> endobj
11 0 obj <<
/Type /Font
/Subtype /Type1
/Name /F3
/BaseFont /Helvetica-Bold
/Encoding 9 0 R
>> endobj
xref
0 12
0000000000 65535 f
0000000021 00000 n
0000000163 00000 n
0000009162 00000 n
0000009245 00000 n
0000000212 00000 n
0000000295 00000 n
0000000375 00000 n
0000009142 00000 n
0000009349 00000 n
0000009606 00000 n
0000009703 00000 n
trailer
<<
/Size 12
/Info 1 0 R
/Root 2 0 R
>>
startxref
9805
%%EOF
lib/galaxy/datatypes/text.py
+16 -20
View File
@@ -50,13 +50,10 @@ class Html(Text):
True
"""
headers = iter_headers(filename, None)
try:
for i, hdr in enumerate(headers):
if hdr and hdr[0].lower().find('<html>') >= 0:
return True
return False
except Exception:
return True
for i, hdr in enumerate(headers):
if hdr and hdr[0].lower().find('<html>') >= 0:
return True
return False
class Json(Text):
@@ -65,7 +62,7 @@ class Json(Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "JavaScript Object Notation (JSON)"
else:
dataset.peek = 'file does not exist'
@@ -113,7 +110,7 @@ class Ipynb(Json):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "Jupyter Notebook"
else:
dataset.peek = 'file does not exist'
@@ -186,7 +183,7 @@ class Biom1(Json):
MetadataElement(name="table_columns", default=[], desc="table_columns", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=[])
def set_peek(self, dataset, is_multi_byte=False):
super(Biom1, self).set_peek(dataset, is_multi_byte)
super(Biom1, self).set_peek(dataset)
if not dataset.dataset.purged:
dataset.blurb = "Biological Observation Matrix v1"
@@ -270,7 +267,7 @@ class Obo(Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "Open Biomedical Ontology (OBO)"
else:
dataset.peek = 'file does not exist'
@@ -309,7 +306,7 @@ class Arff(Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "Attribute-Relation File Format (ARFF)"
dataset.blurb += ", %s comments, %s attributes" % (dataset.metadata.comment_lines, dataset.metadata.columns)
else:
@@ -503,20 +500,19 @@ class SnpSiftDbNSFP(Text):
This is called only at upload to write the html file
cannot rename the datasets here - they come with the default unfortunately
"""
self.regenerate_primary_file(dataset)
return '<html><head><title>SnpSiftDbNSFP Composite Dataset</title></head></html>'
def regenerate_primary_file(self, dataset):
"""
cannot do this until we are setting metadata
"""
annotations = "dbNSFP Annotations: %s\n" % ','.join(dataset.metadata.annotation)
f = open(dataset.file_name, 'a')
if dataset.metadata.bgzip:
bn = dataset.metadata.bgzip
f.write(bn)
f.write('\n')
f.write(annotations)
f.close()
with open(dataset.file_name, 'a') as f:
if dataset.metadata.bgzip:
bn = dataset.metadata.bgzip
f.write(bn)
f.write('\n')
f.write(annotations)
def set_meta(self, dataset, overwrite=True, **kwd):
try:
lib/galaxy/datatypes/triples.py
+7 -7
View File
@@ -31,7 +31,7 @@ class Triples(data.Data):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'Triple data'
else:
dataset.peek = 'file does not exist'
@@ -55,7 +55,7 @@ class NTriples(data.Text, Triples):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'N-Triples triple data'
else:
dataset.peek = 'file does not exist'
@@ -78,7 +78,7 @@ class N3(data.Text, Triples):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'Notation-3 Triple data'
else:
dataset.peek = 'file does not exist'
@@ -105,7 +105,7 @@ class Turtle(data.Text, Triples):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'Turtle triple data'
else:
dataset.peek = 'file does not exist'
@@ -132,7 +132,7 @@ class Rdf(xml.GenericXml, Triples):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'RDF/XML triple data'
else:
dataset.peek = 'file does not exist'
@@ -158,7 +158,7 @@ class Jsonld(text.Json, Triples):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'JSON-LD triple data'
else:
dataset.peek = 'file does not exist'
@@ -181,7 +181,7 @@ class HDT(binary.Binary, Triples):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'HDT triple data'
else:
dataset.peek = 'file does not exist'
lib/galaxy/datatypes/xml.py
+5 -5
View File
@@ -21,7 +21,7 @@ class GenericXml(data.Text):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'XML data'
else:
dataset.peek = 'file does not exist'
@@ -68,7 +68,7 @@ class MEMEXml(GenericXml):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'MEME XML data'
else:
dataset.peek = 'file does not exist'
@@ -85,7 +85,7 @@ class CisML(GenericXml):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'CisML data'
else:
dataset.peek = 'file does not exist'
@@ -104,7 +104,7 @@ class Phyloxml(GenericXml):
def set_peek(self, dataset, is_multi_byte=False):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = 'Phyloxml data'
else:
dataset.peek = 'file does not exist'
@@ -139,7 +139,7 @@ class Owl(GenericXml):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.peek = data.get_file_peek(dataset.file_name)
dataset.blurb = "Web Ontology Language OWL"
else:
dataset.peek = 'file does not exist'
lib/galaxy/jobs/__init__.py
+2 -8
View File
@@ -1287,15 +1287,9 @@ class JobWrapper(object, HasResourceParameters):
dataset.metadata.from_JSON_dict(output_filename, path_rewriter=path_rewriter)
try:
assert context.get('line_count', None) is not None
if (not dataset.datatype.composite_type and dataset.dataset.is_multi_byte()) or self.tool.is_multi_byte:
dataset.set_peek(line_count=context['line_count'], is_multi_byte=True)
else:
dataset.set_peek(line_count=context['line_count'])
dataset.set_peek(line_count=context['line_count'])
except Exception:
if (not dataset.datatype.composite_type and dataset.dataset.is_multi_byte()) or self.tool.is_multi_byte:
dataset.set_peek(is_multi_byte=True)
else:
dataset.set_peek()
dataset.set_peek()
else:
# Handle an empty dataset.
dataset.blurb = "empty"
lib/galaxy/model/__init__.py
+2 -2
View File
@@ -2071,8 +2071,8 @@ class DatasetInstance(object):
"""Data consists of multi-byte characters"""
return self.dataset.is_multi_byte()
def set_peek(self, is_multi_byte=False):
return self.datatype.set_peek(self, is_multi_byte=is_multi_byte)
def set_peek(self):
return self.datatype.set_peek(self)
def init_meta(self, copy_from=None):
return self.datatype.init_meta(self, copy_from=copy_from)
lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py
+2 -2
View File
@@ -241,8 +241,8 @@ class DatasetInstance(object):
"""Saves the data on the disc"""
self.datatype.set_raw_data(self, data)
def set_peek(self, is_multi_byte=False):
return self.datatype.set_peek(self, is_multi_byte=is_multi_byte)
def set_peek(self):
return self.datatype.set_peek(self)
def init_meta(self, copy_from=None):
return self.datatype.init_meta(self, copy_from=copy_from)
lib/galaxy/tools/verify/__init__.py
+59 -62
View File
@@ -168,77 +168,74 @@ def files_diff(file1, file2, attributes=None):
if (line.startswith('+') and not line.startswith('+++')) or (line.startswith('-') and not line.startswith('---')):
count += 1
return count
if not filecmp.cmp(file1, file2):
files_differ = False
if attributes is None:
attributes = {}
decompress = attributes.get("decompress", None)
if not decompress:
local_file = open(file1, 'U').readlines()
history_data = open(file2, 'U').readlines()
if decompress:
# None means all compressed formats are allowed
compressed_formats = None
else:
local_file = get_fileobj(file1, 'U').readlines()
history_data = get_fileobj(file2, 'U').readlines()
compressed_formats = []
is_pdf = False
try:
local_file = get_fileobj(file1, 'U', compressed_formats=compressed_formats).readlines()
history_data = get_fileobj(file2, 'U', compressed_formats=compressed_formats).readlines()
except UnicodeDecodeError:
if file1.endswith('.pdf') or file2.endswith('.pdf'):
is_pdf = True
local_file = open(file1, 'rb').readlines()
history_data = open(file2, 'rb').readlines()
else:
raise AssertionError("Binary data detected, not displaying diff")
if attributes.get('sort', False):
history_data.sort()
# Why even bother with the check loop below, why not just use the diff output? This seems wasteful.
if len(local_file) == len(history_data):
for i in range(len(history_data)):
if local_file[i].rstrip('\r\n') != history_data[i].rstrip('\r\n'):
files_differ = True
break
else:
files_differ = True
if files_differ:
allowed_diff_count = int(attributes.get('lines_diff', 0))
diff = list(difflib.unified_diff(local_file, history_data, "local_file", "history_data"))
diff_lines = get_lines_diff(diff)
if diff_lines > allowed_diff_count:
if 'GALAXY_TEST_RAW_DIFF' in os.environ:
diff_slice = diff
allowed_diff_count = int(attributes.get('lines_diff', 0))
diff = list(difflib.unified_diff(local_file, history_data, "local_file", "history_data"))
diff_lines = get_lines_diff(diff)
if diff_lines > allowed_diff_count:
if 'GALAXY_TEST_RAW_DIFF' in os.environ:
diff_slice = diff
else:
if len(diff) < 60:
diff_slice = diff[0:40]
else:
if len(diff) < 60:
diff_slice = diff[0:40]
else:
diff_slice = diff[:25] + ["********\n", "*SNIP *\n", "********\n"] + diff[-25:]
# FIXME: This pdf stuff is rather special cased and has not been updated to consider lines_diff
# due to unknown desired behavior when used in conjunction with a non-zero lines_diff
# PDF forgiveness can probably be handled better by not special casing by __extension__ here
# and instead using lines_diff or a regular expression matching
# or by creating and using a specialized pdf comparison function
if file1.endswith('.pdf') or file2.endswith('.pdf'):
# PDF files contain creation dates, modification dates, ids and descriptions that change with each
# new file, so we need to handle these differences. As long as the rest of the PDF file does
# not differ we're ok.
valid_diff_strs = ['description', 'createdate', 'creationdate', 'moddate', 'id', 'producer', 'creator']
valid_diff = False
invalid_diff_lines = 0
for line in diff_slice:
# Make sure to lower case strings before checking.
line = line.lower()
# Diff lines will always start with a + or - character, but handle special cases: '--- local_file \n', '+++ history_data \n'
if (line.startswith('+') or line.startswith('-')) and line.find('local_file') < 0 and line.find('history_data') < 0:
for vdf in valid_diff_strs:
if line.find(vdf) < 0:
valid_diff = False
else:
valid_diff = True
# Stop checking as soon as we know we have a valid difference
break
if not valid_diff:
invalid_diff_lines += 1
log.info('## files diff on %s and %s lines_diff=%d, found diff = %d, found pdf invalid diff = %d' % (file1, file2, allowed_diff_count, diff_lines, invalid_diff_lines))
if invalid_diff_lines > allowed_diff_count:
# Print out diff_slice so we can see what failed
log.info("###### diff_slice ######")
raise AssertionError("".join(diff_slice))
else:
log.info('## files diff on %s and %s lines_diff=%d, found diff = %d' % (file1, file2, allowed_diff_count, diff_lines))
for line in diff_slice:
for char in line:
if ord(char) > 128:
raise AssertionError("Binary data detected, not displaying diff")
diff_slice = diff[:25] + ["********\n", "*SNIP *\n", "********\n"] + diff[-25:]
# FIXME: This pdf stuff is rather special cased and has not been updated to consider lines_diff
# due to unknown desired behavior when used in conjunction with a non-zero lines_diff
# PDF forgiveness can probably be handled better by not special casing by __extension__ here
# and instead using lines_diff or a regular expression matching
# or by creating and using a specialized pdf comparison function
if is_pdf:
# PDF files contain creation dates, modification dates, ids and descriptions that change with each
# new file, so we need to handle these differences. As long as the rest of the PDF file does
# not differ we're ok.
valid_diff_strs = ['description', 'createdate', 'creationdate', 'moddate', 'id', 'producer', 'creator']
valid_diff = False
invalid_diff_lines = 0
for line in diff_slice:
# Make sure to lower case strings before checking.
line = line.lower()
# Diff lines will always start with a + or - character, but handle special cases: '--- local_file \n', '+++ history_data \n'
if (line.startswith('+') or line.startswith('-')) and line.find('local_file') < 0 and line.find('history_data') < 0:
for vdf in valid_diff_strs:
if line.find(vdf) < 0:
valid_diff = False
else:
valid_diff = True
# Stop checking as soon as we know we have a valid difference
break
if not valid_diff:
invalid_diff_lines += 1
log.info('## files diff on %s and %s lines_diff=%d, found diff = %d, found pdf invalid diff = %d' % (file1, file2, allowed_diff_count, diff_lines, invalid_diff_lines))
if invalid_diff_lines > allowed_diff_count:
# Print out diff_slice so we can see what failed
log.info("###### diff_slice ######")
raise AssertionError("".join(diff_slice))
else:
log.info('## files diff on %s and %s lines_diff=%d, found diff = %d' % (file1, file2, allowed_diff_count, diff_lines))
raise AssertionError("".join(diff_slice))
def files_re_match(file1, file2, attributes=None):
lib/galaxy/util/compression_utils.py
+25 -12
View File
@@ -1,4 +1,5 @@
import gzip
import io
import sys
import zipfile
@@ -9,32 +10,44 @@ from .checkers import (
if sys.version_info < (3, 3):
import bz2file as bz2
gzip.GzipFile.read1 = gzip.GzipFile.read # workaround for https://bugs.python.org/issue12591
else:
import bz2
def get_fileobj(filename, mode="r", gzip_only=False, bz2_only=False, zip_only=False):
def get_fileobj(filename, mode="r", compressed_formats=None):
"""
Returns a fileobj. If the file is compressed, return appropriate file reader.
Returns a fileobj. If the file is compressed, return an appropriate file
reader. In text mode, always use 'utf-8' encoding.
:param filename: path to file that should be opened
:param mode: mode to pass to opener
:param gzip_only: only open file if file is gzip compressed or not compressed
:param bz2_only: only open file if file is bz2 compressed or not compressed
:param zip_only: only open file if file is zip compressed or not compressed
:param compressed_formats: list of allowed compressed file formats among
'bz2', 'gzip' and 'zip'. If left to None, all 3 formats are allowed
"""
if compressed_formats is None:
compressed_formats = ['bz2', 'gzip', 'zip']
# Remove 't' from mode, which may cause an error for compressed files
mode = mode.replace('t', '')
# the various compression readers don't support 'U' mode,
# so we open in 'r'.
if mode == 'U':
cmode = 'r'
else:
cmode = mode
if not bz2_only and not zip_only and is_gzip(filename):
return gzip.GzipFile(filename, cmode)
if not gzip_only and not zip_only and is_bz2(filename):
return bz2.BZ2File(filename, cmode)
if not bz2_only and not gzip_only and zipfile.is_zipfile(filename):
if 'gzip' in compressed_formats and is_gzip(filename):
fh = gzip.GzipFile(filename, cmode)
elif 'bz2' in compressed_formats and is_bz2(filename):
fh = bz2.BZ2File(filename, cmode)
elif 'zip' in compressed_formats and zipfile.is_zipfile(filename):
# Return fileobj for the first file in a zip file.
with zipfile.ZipFile(filename, cmode) as zh:
return zh.open(zh.namelist()[0], cmode)
return open(filename, mode)
fh = zh.open(zh.namelist()[0], cmode)
elif 'b' in mode:
return open(filename, mode)
else:
return io.open(filename, mode, encoding='utf-8')
if 'b' not in mode:
return io.TextIOWrapper(fh, encoding='utf-8')
else:
return fh
lib/tool_shed/util/tool_util.py
+1 -35
View File
@@ -4,8 +4,8 @@ import shutil
import galaxy.tools
from galaxy import util
from galaxy.datatypes.sniff import is_column_based
from galaxy.util import checkers
from galaxy.util import unicodify
from galaxy.util.expressions import ExpressionContext
from galaxy.web.form_builder import SelectField
from tool_shed.util import basic_util
@@ -125,20 +125,6 @@ def generate_message_for_invalid_tools(app, invalid_file_tups, repository, metad
return message
def get_headers(fname, sep, count=60, is_multi_byte=False):
"""Returns a list with the first 'count' lines split by 'sep'."""
headers = []
for idx, line in enumerate(open(fname)):
line = line.rstrip('\n\r')
if is_multi_byte:
line = unicodify(line, 'utf-8')
sep = sep.encode('utf-8')
headers.append(line.split(sep))
if idx == count:
break
return headers
def get_tool_path_install_dir(partial_install_dir, shed_tool_conf_dict, tool_dict, config_elems):
for elem in config_elems:
if elem.tag == 'tool':
@@ -184,26 +170,6 @@ def handle_missing_index_file(app, tool_path, sample_files, repository_tools_tup
return repository_tools_tups, sample_files_copied
def is_column_based(fname, sep='\t', skip=0, is_multi_byte=False):
"""See if the file is column based with respect to a separator."""
headers = get_headers(fname, sep, is_multi_byte=is_multi_byte)
count = 0
if not headers:
return False
for hdr in headers[skip:]:
if hdr and hdr[0] and not hdr[0].startswith('#'):
if len(hdr) > 1:
count = len(hdr)
break
if count < 2:
return False
for hdr in headers[skip:]:
if hdr and hdr[0] and not hdr[0].startswith('#'):
if len(hdr) != count:
return False
return True
def is_data_index_sample_file(file_path):
"""
Attempt to determine if a .sample file is appropriate for copying to ~/tool-data when
test-data/1.fastqsanger
+1 -1
View File
@@ -5,4 +5,4 @@ AATACTTTCGGCGCCCTAAACCAGCTCACTGGGG
@1831_573_1050/1
TTTATGGGTATGGCCGCTCACAGGCCAGCGGCCT
+
;@@17?@=>7??@A8?==@4A?A4)&+.'&+'1,
;@@17?@=>7??@A8?==@4A?A4)&+.'&+'1,
test-data/1.fastqsolexa
+1 -1
View File
@@ -5,4 +5,4 @@ GAATTGATCAGGACATAGGACAACTGTAGGCACCAT
@HANNIBAL_1_FC302VTAAXX:2:1:156:340
GAGTTCTCGTCGCCTGTAGGCACCATCAATCGTATG
+HANNIBAL_1_FC302VTAAXX:2:1:156:340
40 15 40 17 6 36 40 40 40 25 40 9 35 33 40 14 14 18 15 17 19 28 31 4 24 18 27 14 15 18 2 8 12 8 11 9
40 15 40 17 6 36 40 40 40 25 40 9 35 33 40 14 14 18 15 17 19 28 31 4 24 18 27 14 15 18 2 8 12 8 11 9
test-data/simple_line_alternative.txt
+1 -1
View File
@@ -1 +1 @@
This is a different line of text.
This is a different line of text.
test-data/simple_lines_interleaved.txt
+1 -1
View File
@@ -1,4 +1,4 @@
This is a line of text.
This is a different line of text.
This is a line of text.
This is a different line of text.
This is a different line of text.
tools/data_source/data_source.py
+1 -1
View File
@@ -105,7 +105,7 @@ def __main__():
# here import checks that upload tool performs
if enhanced_handling:
try:
ext = sniff.handle_uploaded_dataset_file(filename, datatypes_registry, ext=data_dict['ext'], is_multi_byte=is_multi_byte)
ext = sniff.handle_uploaded_dataset_file(filename, datatypes_registry, ext=data_dict['ext'])
except Exception as e:
stop_err(str(e))
info = dict(type='dataset',
tools/data_source/upload.py
+1 -1
View File
@@ -124,7 +124,7 @@ def add_file(dataset, registry, json_file, output_path):
# Is dataset content multi-byte?
elif dataset.is_multi_byte:
data_type = 'multi-byte char'
ext = sniff.guess_ext(dataset.path, registry.sniff_order, is_multi_byte=True)
ext = sniff.guess_ext(dataset.path, registry.sniff_order)
# Is dataset content supported sniffable binary?
else:
# FIXME: This ignores the declared sniff order in datatype_conf.xml