From 953ee3259999f774887e97d5e72bccb1102540f5 Mon Sep 17 00:00:00 2001 From: Ignacio Date: Sun, 9 Oct 2016 15:42:05 +0200 Subject: [PATCH 1/5] change parameters to samtools sort command (new samtools vers) --- lib/galaxy/datatypes/converters/sam_to_bam.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/converters/sam_to_bam.py b/lib/galaxy/datatypes/converters/sam_to_bam.py index 97b97c6efbb..45a8fbaafd5 100644 --- a/lib/galaxy/datatypes/converters/sam_to_bam.py +++ b/lib/galaxy/datatypes/converters/sam_to_bam.py @@ -11,6 +11,7 @@ import optparse import tempfile import subprocess import shutil +from distutils.version import LooseVersion CHUNK_SIZE = 2 ** 20 # 1mb @@ -19,6 +20,18 @@ def cleanup_before_exit( tmp_dir ): if tmp_dir and os.path.exists( tmp_dir ): shutil.rmtree( tmp_dir ) +def getSamtoolsVersion(): + cmd = 'samtools --version' + proc = subprocess.Popen( args=cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True ) + stdout, stderr = proc.communicate() + if proc.returncode: + sys.stderr.write(stderr) + return None + else: + return stdout.split("\n")[0].split()[-1] + + + def __main__(): # Parse Command Line @@ -52,7 +65,12 @@ def __main__(): # sort sam, so indexing will not fail sorted_stderr_filename = os.path.join( tmp_dir, 'sorted.stderr' ) sorting_prefix = os.path.join( tmp_dir, 'sorted_bam' ) - cmd = 'samtools sort -o "%s" "%s" > "%s"' % ( unsorted_bam_filename, sorting_prefix, output_filename ) + # samtools changed sort command arguments (starting from version 1.3) + samtools_version = LooseVersion(getSamtoolsVersion()) + if samtools_version < LooseVersion('1.3'): + cmd = 'samtools sort -o "%s" "%s" > "%s"' % ( unsorted_bam_filename, sorting_prefix, output_filename ) + else: + cmd = 'samtools sort -T "%s" "%s" > "%s"' % ( sorting_prefix, unsorted_bam_filename, output_filename ) proc = subprocess.Popen( args=cmd, stderr=open( sorted_stderr_filename, 'wb' ), shell=True, cwd=tmp_dir ) return_code = proc.wait() From 5bb86a3b687732913dee746732060f973b622cff Mon Sep 17 00:00:00 2001 From: Ignacio Date: Sun, 9 Oct 2016 15:57:09 +0200 Subject: [PATCH 2/5] correct indentation --- lib/galaxy/datatypes/converters/sam_to_bam.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/datatypes/converters/sam_to_bam.py b/lib/galaxy/datatypes/converters/sam_to_bam.py index 45a8fbaafd5..2faef6d3e22 100644 --- a/lib/galaxy/datatypes/converters/sam_to_bam.py +++ b/lib/galaxy/datatypes/converters/sam_to_bam.py @@ -20,17 +20,16 @@ def cleanup_before_exit( tmp_dir ): if tmp_dir and os.path.exists( tmp_dir ): shutil.rmtree( tmp_dir ) + def getSamtoolsVersion(): cmd = 'samtools --version' proc = subprocess.Popen( args=cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True ) stdout, stderr = proc.communicate() if proc.returncode: - sys.stderr.write(stderr) - return None + sys.stderr.write(stderr) + return None else: - return stdout.split("\n")[0].split()[-1] - - + return stdout.split("\n")[0].split()[-1] def __main__(): @@ -66,11 +65,11 @@ def __main__(): sorted_stderr_filename = os.path.join( tmp_dir, 'sorted.stderr' ) sorting_prefix = os.path.join( tmp_dir, 'sorted_bam' ) # samtools changed sort command arguments (starting from version 1.3) - samtools_version = LooseVersion(getSamtoolsVersion()) - if samtools_version < LooseVersion('1.3'): - cmd = 'samtools sort -o "%s" "%s" > "%s"' % ( unsorted_bam_filename, sorting_prefix, output_filename ) + samtools_version = LooseVersion(getSamtoolsVersion()) + if samtools_version < LooseVersion('1.3'): + cmd = 'samtools sort -o "%s" "%s" > "%s"' % ( unsorted_bam_filename, sorting_prefix, output_filename ) else: - cmd = 'samtools sort -T "%s" "%s" > "%s"' % ( sorting_prefix, unsorted_bam_filename, output_filename ) + cmd = 'samtools sort -T "%s" "%s" > "%s"' % ( sorting_prefix, unsorted_bam_filename, output_filename ) proc = subprocess.Popen( args=cmd, stderr=open( sorted_stderr_filename, 'wb' ), shell=True, cwd=tmp_dir ) return_code = proc.wait() From efde00d80a916afc20869608a7a1922249581a62 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 16 Nov 2016 21:10:17 +0100 Subject: [PATCH 3/5] Handle samtools 0.x --- lib/galaxy/datatypes/converters/sam_to_bam.py | 45 ++++++++++++++----- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/datatypes/converters/sam_to_bam.py b/lib/galaxy/datatypes/converters/sam_to_bam.py index 2faef6d3e22..4659d58e359 100644 --- a/lib/galaxy/datatypes/converters/sam_to_bam.py +++ b/lib/galaxy/datatypes/converters/sam_to_bam.py @@ -21,15 +21,38 @@ def cleanup_before_exit( tmp_dir ): shutil.rmtree( tmp_dir ) -def getSamtoolsVersion(): - cmd = 'samtools --version' - proc = subprocess.Popen( args=cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True ) - stdout, stderr = proc.communicate() - if proc.returncode: - sys.stderr.write(stderr) - return None - else: - return stdout.split("\n")[0].split()[-1] +def cmd_exists(cmd): + # http://stackoverflow.com/questions/5226958/which-equivalent-function-in-python + for path in os.environ["PATH"].split(":"): + if os.path.exists(os.path.join(path, cmd)): + return True + return False + + +def _get_samtools_version(): + version = '0.0.0' + if not cmd_exists('samtools'): + raise Exception('This tool needs samtools, but it is not on PATH.') + # Get the version of samtools via --version-only, if available + p = subprocess.Popen( ['samtools', '--version-only'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output, error = p.communicate() + + # --version-only is available + # Format is +htslib- + if p.returncode == 0: + version = output.split('+')[0] + return version + + output = subprocess.Popen( [ 'samtools' ], stderr=subprocess.PIPE, stdout=subprocess.PIPE ).communicate()[1] + lines = output.split( '\n' ) + for line in lines: + if line.lower().startswith( 'version' ): + # Assuming line looks something like: version: 0.1.12a (r862) + version = line.split()[1] + break + return version def __main__(): @@ -65,8 +88,8 @@ def __main__(): sorted_stderr_filename = os.path.join( tmp_dir, 'sorted.stderr' ) sorting_prefix = os.path.join( tmp_dir, 'sorted_bam' ) # samtools changed sort command arguments (starting from version 1.3) - samtools_version = LooseVersion(getSamtoolsVersion()) - if samtools_version < LooseVersion('1.3'): + samtools_version = LooseVersion(_get_samtools_version()) + if samtools_version < LooseVersion('1.0'): cmd = 'samtools sort -o "%s" "%s" > "%s"' % ( unsorted_bam_filename, sorting_prefix, output_filename ) else: cmd = 'samtools sort -T "%s" "%s" > "%s"' % ( sorting_prefix, unsorted_bam_filename, output_filename ) From 592f15c7b5da3f962dda9f98cb978e5c5c6b4a1d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 16 Nov 2016 22:01:47 +0100 Subject: [PATCH 4/5] Add sam-to-bam conversion test tool --- test-data/bam_from_sam.bam | Bin 0 -> 502 bytes test-data/sam_with_header.sam | 14 ++++++++++++++ test/functional/tools/sam_to_bam.xml | 19 +++++++++++++++++++ 3 files changed, 33 insertions(+) create mode 100644 test-data/bam_from_sam.bam create mode 100644 test-data/sam_with_header.sam create mode 100644 test/functional/tools/sam_to_bam.xml diff --git a/test-data/bam_from_sam.bam b/test-data/bam_from_sam.bam new file mode 100644 index 0000000000000000000000000000000000000000..a67e8b3de4d0cfe49dfe4da87c986714bdf5d3bc GIT binary patch literal 502 zcmb2|=3rp}f&Xj_PR>jWsSL$MUsAUuCL|PmD0s;8d9%?K<7YhIwtJmBukUs8jJJ;G z(;^POCp_OaYkB!T{_=6Nk)f-Q0aLd`fg$79%?ySC9ggB2o1JdB@klwaF*x+?U7QFs zTprDqV#d-*zTSr&1lpFb73}VsyW^Eskd(lgjrZdw2dtFmXkGM0qhnsTdVi$9(6^O%ja+C%bj$6-&floP0r_HC+K;;znjM&^sCcT@T7XF z=gbKjE^7|GSTE7-YIJ{*(wnP}2ZZ8g>^kh{yVvd3mBY7I`X6Ord^xk^jO*@1%$Q*=%>WdH0su-y*5?2K literal 0 HcmV?d00001 diff --git a/test-data/sam_with_header.sam b/test-data/sam_with_header.sam new file mode 100644 index 00000000000..33449b176bc --- /dev/null +++ b/test-data/sam_with_header.sam @@ -0,0 +1,14 @@ +@SQ SN:ref LN:45 +@SQ SN:ref2 LN:40 +r001 163 ref 7 30 8M4I4M1D3M = 37 39 TTAGATAAAGAGGATACTG * XX:B:S,12561,2,20,112 +r002 0 ref 9 30 1S2I6M1P1I1P1I4M2I * 0 0 AAAAGATAAGGGATAAA * +r003 0 ref 9 30 5H6M * 0 0 AGCTAA * +r004 0 ref 16 30 6M14N1I5M * 0 0 ATAGCTCTCAGC * +r003 16 ref 29 30 6H5M * 0 0 TAGGC * +r001 83 ref 37 30 9M = 7 -39 CAGCGCCAT * +x1 0 ref2 1 30 20M * 0 0 aggttttataaaacaaataa ???????????????????? +x2 0 ref2 2 30 21M * 0 0 ggttttataaaacaaataatt ????????????????????? +x3 0 ref2 6 30 9M4I13M * 0 0 ttataaaacAAATaattaagtctaca ?????????????????????????? +x4 0 ref2 10 30 25M * 0 0 CaaaTaattaagtctacagagcaac ????????????????????????? +x5 0 ref2 12 30 24M * 0 0 aaTaattaagtctacagagcaact ???????????????????????? +x6 0 ref2 14 30 23M * 0 0 Taattaagtctacagagcaacta ??????????????????????? diff --git a/test/functional/tools/sam_to_bam.xml b/test/functional/tools/sam_to_bam.xml new file mode 100644 index 00000000000..46b92f2a208 --- /dev/null +++ b/test/functional/tools/sam_to_bam.xml @@ -0,0 +1,19 @@ + + + cat '$input1' > '$out_file1' + + + + + + + + + + + + + + + + From 8ace021c0af79e5d325d6a590f525bce881715f8 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 17 Nov 2016 10:06:07 +0100 Subject: [PATCH 5/5] Single quote variables in sam-to-bam conversion --- lib/galaxy/datatypes/converters/sam_to_bam.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/datatypes/converters/sam_to_bam.py b/lib/galaxy/datatypes/converters/sam_to_bam.py index 4659d58e359..6fe14fed055 100644 --- a/lib/galaxy/datatypes/converters/sam_to_bam.py +++ b/lib/galaxy/datatypes/converters/sam_to_bam.py @@ -68,7 +68,7 @@ def __main__(): # convert to SAM unsorted_bam_filename = os.path.join( tmp_dir, 'unsorted.bam' ) unsorted_stderr_filename = os.path.join( tmp_dir, 'unsorted.stderr' ) - cmd = 'samtools view -bS "%s" > "%s"' % ( input_filename, unsorted_bam_filename ) + cmd = "samtools view -bS '%s' > '%s'" % ( input_filename, unsorted_bam_filename ) proc = subprocess.Popen( args=cmd, stderr=open( unsorted_stderr_filename, 'wb' ), shell=True, cwd=tmp_dir ) return_code = proc.wait() if return_code: @@ -90,9 +90,9 @@ def __main__(): # samtools changed sort command arguments (starting from version 1.3) samtools_version = LooseVersion(_get_samtools_version()) if samtools_version < LooseVersion('1.0'): - cmd = 'samtools sort -o "%s" "%s" > "%s"' % ( unsorted_bam_filename, sorting_prefix, output_filename ) + cmd = "samtools sort -o '%s' '%s' > '%s'" % ( unsorted_bam_filename, sorting_prefix, output_filename ) else: - cmd = 'samtools sort -T "%s" "%s" > "%s"' % ( sorting_prefix, unsorted_bam_filename, output_filename ) + cmd = "samtools sort -T '%s' '%s' > '%s'" % ( sorting_prefix, unsorted_bam_filename, output_filename ) proc = subprocess.Popen( args=cmd, stderr=open( sorted_stderr_filename, 'wb' ), shell=True, cwd=tmp_dir ) return_code = proc.wait()