mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
Merge pull request #11859 from bernt-matthias/topic/grep-header
grep, tail, head: add option to keep header
This commit is contained in:
Marius van den Beek
@@ -202,6 +202,9 @@ WORKFLOW_SAFE_TOOL_VERSION_UPDATES = {
|
||||
'__BUILD_LIST__': safe_update(packaging.version.parse("1.0.0"), packaging.version.parse("1.0.1")),
|
||||
'__APPLY_RULES__': safe_update(packaging.version.parse("1.0.0"), packaging.version.parse("1.1.0")),
|
||||
'__EXTRACT_DATASET__': safe_update(packaging.version.parse("1.0.0"), packaging.version.parse("1.0.1")),
|
||||
'Grep1': safe_update(packaging.version.parse("1.0.1"), packaging.version.parse("1.0.2")),
|
||||
'Show beginning1': safe_update(packaging.version.parse("1.0.0"), packaging.version.parse("1.0.1")),
|
||||
'Show tail1': safe_update(packaging.version.parse("1.0.0"), packaging.version.parse("1.0.1")),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,137 +0,0 @@
|
||||
# Filename: grep.py
|
||||
# Author: Ian N. Schenck
|
||||
# Version: 8/23/2005
|
||||
#
|
||||
# This script accepts regular expressions, as well as an "invert"
|
||||
# option, and applies the regular expression using grep. This wrapper
|
||||
# provides security and pipeline.
|
||||
#
|
||||
# Grep is launched based on these inputs:
|
||||
# -i Input file
|
||||
# -o Output file
|
||||
# -pattern RegEx pattern
|
||||
# -v true or false (output NON-matching lines)
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from subprocess import PIPE, Popen
|
||||
from tempfile import NamedTemporaryFile
|
||||
|
||||
|
||||
# This function is exceedingly useful, perhaps package for reuse?
|
||||
def getopts(argv):
|
||||
opts = {}
|
||||
while argv:
|
||||
if argv[0][0] == '-':
|
||||
opts[argv[0]] = argv[1]
|
||||
argv = argv[2:]
|
||||
else:
|
||||
argv = argv[1:]
|
||||
return opts
|
||||
|
||||
|
||||
def main():
|
||||
args = sys.argv[1:]
|
||||
|
||||
try:
|
||||
opts = getopts(args)
|
||||
except IndexError:
|
||||
print("Usage:")
|
||||
print(" -i Input file")
|
||||
print(" -o Output file")
|
||||
print(" -pattern RegEx pattern")
|
||||
print(" -v true or false (Invert match)")
|
||||
return 0
|
||||
|
||||
outputfile = opts.get("-o")
|
||||
if outputfile is None:
|
||||
print("No output file specified.")
|
||||
return -1
|
||||
|
||||
inputfile = opts.get("-i")
|
||||
if inputfile is None:
|
||||
print("No input file specified.")
|
||||
return -2
|
||||
|
||||
invert = opts.get("-v")
|
||||
if invert is None:
|
||||
print("Match style (Invert or normal) not specified.")
|
||||
return -3
|
||||
|
||||
pattern = opts.get("-pattern")
|
||||
if pattern is None:
|
||||
print("RegEx pattern not specified.")
|
||||
return -4
|
||||
|
||||
# All inputs have been specified at this point, now validate.
|
||||
|
||||
# replace if input has been escaped, remove sq
|
||||
# characters that are allowed but need to be escaped
|
||||
mapped_chars = {'>': '__gt__',
|
||||
'<': '__lt__',
|
||||
'\'': '__sq__',
|
||||
'"': '__dq__',
|
||||
'[': '__ob__',
|
||||
']': '__cb__',
|
||||
'{': '__oc__',
|
||||
'}': '__cc__'}
|
||||
|
||||
# with new sanitizing we only need to replace for single quote,
|
||||
# but this needs to remain for backwards compatibility
|
||||
for key, value in mapped_chars.items():
|
||||
pattern = pattern.replace(value, key)
|
||||
|
||||
# match filename and invert flag
|
||||
fileRegEx = re.compile(r"^[A-Za-z0-9./\-_]+$")
|
||||
invertRegEx = re.compile(r"(true)|(false)")
|
||||
|
||||
# verify that filename and inversion flag are in the correct format
|
||||
if not fileRegEx.match(outputfile):
|
||||
print("Illegal output filename.")
|
||||
return -5
|
||||
if not fileRegEx.match(inputfile):
|
||||
print("Illegal input filename.")
|
||||
return -6
|
||||
if not invertRegEx.match(invert):
|
||||
print("Illegal invert option.")
|
||||
return -7
|
||||
|
||||
# invert grep search?
|
||||
if invert == "true":
|
||||
invertflag = "-v"
|
||||
print("Not matching pattern: %s" % pattern)
|
||||
else:
|
||||
invertflag = ""
|
||||
print("Matching pattern: %s" % pattern)
|
||||
|
||||
# set version flag
|
||||
versionflag = "-P"
|
||||
|
||||
# MacOS 10.8.2 does not support -P option for perl-regex anymore
|
||||
versionmatch = Popen("grep -V | grep 'BSD'", shell=True, stdout=PIPE).communicate()[0]
|
||||
if versionmatch:
|
||||
versionflag = "-E"
|
||||
|
||||
# create temp file holding pattern
|
||||
# by using a file to hold the pattern, we don't have worry about sanitizing grep commandline and can include single quotes in pattern
|
||||
pattern_file_name = NamedTemporaryFile().name
|
||||
open(pattern_file_name, 'w').write(pattern)
|
||||
|
||||
# generate grep command
|
||||
commandline = "grep %s %s -f %s %s > %s" % (versionflag, invertflag, pattern_file_name, inputfile, outputfile)
|
||||
|
||||
# run grep
|
||||
errorcode = subprocess.call(commandline, shell=True)
|
||||
|
||||
# remove temp pattern file
|
||||
os.unlink(pattern_file_name)
|
||||
|
||||
# return error code
|
||||
return errorcode
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+47
-10
@@ -1,36 +1,73 @@
|
||||
<tool id="Grep1" name="Select" version="1.0.1">
|
||||
<tool id="Grep1" name="Select" version="1.0.2" profile="20.05">
|
||||
<requirements>
|
||||
<requirement type="package" version="3.4">grep</requirement>
|
||||
</requirements>
|
||||
<description>lines that match an expression</description>
|
||||
<edam_operations>
|
||||
<edam_operation>operation_3695</edam_operation>
|
||||
</edam_operations>
|
||||
<command interpreter="python">grep.py -i $input -o $out_file1 -pattern '$pattern' -v $invert</command>
|
||||
<stdio>
|
||||
<exit_code range="2:" level="warning" description="Select failed" />
|
||||
</stdio>
|
||||
<command><![CDATA[
|
||||
set -eo pipefail;
|
||||
#if $keep_header
|
||||
head -n 1 '$input' > '$out_file1' &&
|
||||
tail -n +2 '$input'
|
||||
#else
|
||||
cat '$input'
|
||||
#end if
|
||||
| grep -f '$pattern_file' $invert > '$out_file1'
|
||||
]]></command>
|
||||
<configfiles>
|
||||
<configfile name="pattern_file">$pattern</configfile>
|
||||
</configfiles>
|
||||
<inputs>
|
||||
<param format="txt" name="input" type="data" label="Select lines from"/>
|
||||
<param name="invert" type="select" label="that">
|
||||
<option value="false">Matching</option>
|
||||
<option value="true">NOT Matching</option>
|
||||
<option value="">Matching</option>
|
||||
<option value="-v">NOT Matching</option>
|
||||
</param>
|
||||
<param name="pattern" type="text" value="^chr([0-9A-Za-z])+" label="the pattern" help="here you can enter text or regular expression (for syntax check lower part of this frame)">
|
||||
<sanitizer>
|
||||
<valid initial="string.printable">
|
||||
<remove value="'"/>
|
||||
</valid>
|
||||
<mapping initial="none">
|
||||
<add source="'" target="__sq__"/>
|
||||
</mapping>
|
||||
</sanitizer>
|
||||
</param>
|
||||
<param name="keep_header" type="boolean" truevalue="true" falsevalue="false" label="Keep header line" help="i.e. the first line is kept independent of the regular expression"/>
|
||||
</inputs>
|
||||
<outputs>
|
||||
<data format="input" name="out_file1" metadata_source="input"/>
|
||||
<data format_source="input" name="out_file1" metadata_source="input"/>
|
||||
</outputs>
|
||||
<tests>
|
||||
<test>
|
||||
<param name="input" value="1.bed"/>
|
||||
<param name="invert" value="false"/>
|
||||
<param name="invert" value=""/>
|
||||
<param name="pattern" value="^chr[0-9]*"/>
|
||||
<output name="out_file1" file="fs-grep.dat"/>
|
||||
</test>
|
||||
<test>
|
||||
<param name="input" value="1.bed"/>
|
||||
<param name="invert" value=""/>
|
||||
<param name="pattern" value="+"/>
|
||||
<param name="keep_header" value="true"/>
|
||||
<output name="out_file1">
|
||||
<assert_contents>
|
||||
<not_has_text text="CCDS989.1_cds_0_0_chr1_147962193_r"/>
|
||||
</assert_contents>
|
||||
</output>
|
||||
</test>
|
||||
<test>
|
||||
<param name="input" value="1.bed"/>
|
||||
<param name="invert" value=""/>
|
||||
<param name="pattern" value="<>\"[A]{1}"/>
|
||||
<param name="keep_header" value="true"/>
|
||||
<output name="out_file1">
|
||||
<assert_contents>
|
||||
<not_has_text text="CCDS989.1_cds_0_0_chr1_147962193_r"/>
|
||||
</assert_contents>
|
||||
</output>
|
||||
</test>
|
||||
</tests>
|
||||
<help>
|
||||
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
#! /usr/bin/env perl
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
# a wrapper for head for use in galaxy
|
||||
# headWrapper.pl [filename] [# lines to show] [output]
|
||||
|
||||
die "Check arguments" unless @ARGV == 3;
|
||||
die "Line number must be an integer\n" unless $ARGV[1]=~ m/^\d+$/;
|
||||
|
||||
open (OUT, ">$ARGV[2]") or die "Cannot create $ARGV[2]:$!\n";
|
||||
open (HEAD, "head -n $ARGV[1] $ARGV[0]|") or die "Cannot run head:$!\n";
|
||||
while (<HEAD>) {
|
||||
print OUT;
|
||||
}
|
||||
close OUT;
|
||||
close HEAD;
|
||||
@@ -1,23 +1,51 @@
|
||||
<tool id="Show beginning1" name="Select first" version="1.0.0">
|
||||
<tool id="Show beginning1" name="Select first" version="1.0.1" profile="20.05">
|
||||
<description>lines from a dataset</description>
|
||||
<edam_operations>
|
||||
<edam_operation>operation_3695</edam_operation>
|
||||
</edam_operations>
|
||||
<command>
|
||||
perl '$__tool_directory__/headWrapper.pl' '$input' $lineNum '$out_file1'
|
||||
</command>
|
||||
<requirements>
|
||||
<requirement type="package" version="8.31">coreutils</requirement>
|
||||
</requirements>
|
||||
<command detect_errors="exit_code"><![CDATA[
|
||||
set -eo pipefail;
|
||||
#if $header
|
||||
head -n 1 '$input' > '$out_file1' &&
|
||||
tail -n +2 '$input'
|
||||
#else
|
||||
cat '$input'
|
||||
#end if
|
||||
| head -n $lineNum
|
||||
>> '$out_file1'
|
||||
]]></command>
|
||||
<inputs>
|
||||
<param name="lineNum" type="integer" value="10" label="Select first" help="lines"/>
|
||||
<param name="lineNum" type="integer" value="10" min="1" label="Select first" help="lines"/>
|
||||
<param name="input" type="data" format="txt" label="from"/>
|
||||
<param name="header" type="boolean" label="Dataset has a header" help="if checked the header is kept and is not counted as a kept line"/>
|
||||
</inputs>
|
||||
<outputs>
|
||||
<data name="out_file1" format="input" metadata_source="input"/>
|
||||
<data name="out_file1" format_source="input" metadata_source="input"/>
|
||||
</outputs>
|
||||
<tests>
|
||||
<test>
|
||||
<param name="lineNum" value="10"/>
|
||||
<param name="input" value="1.bed"/>
|
||||
<output name="out_file1" file="eq-showbeginning.dat"/>
|
||||
<output name="out_file1" file="eq-showbeginning.dat">
|
||||
<assert_contents>
|
||||
<not_has_text text="CCDS9526.1_cds_0_0_chr13_112381695_f"/>
|
||||
<has_n_lines n="10"/>
|
||||
</assert_contents>
|
||||
</output>
|
||||
</test>
|
||||
<test>
|
||||
<param name="lineNum" value="10"/>
|
||||
<param name="input" value="1.bed"/>
|
||||
<param name="header" value="true"/>
|
||||
<output name="out_file1">
|
||||
<assert_contents>
|
||||
<has_text text="CCDS9526.1_cds_0_0_chr13_112381695_f"/>
|
||||
<has_n_lines n="11"/>
|
||||
</assert_contents>
|
||||
</output>
|
||||
</test>
|
||||
</tests>
|
||||
<help>
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
#! /usr/bin/env perl
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
# a wrapper for tail for use in galaxy
|
||||
# lessWrapper.pl [filename] [# lines to show] [output]
|
||||
|
||||
die "Check arguments" unless @ARGV == 3;
|
||||
die "Line number should be an integer\n" unless $ARGV[1]=~ m/^\d+$/;
|
||||
|
||||
open (OUT, ">$ARGV[2]") or die "Cannot create $ARGV[2]:$!\n";
|
||||
open (TAIL, "tail -n $ARGV[1] $ARGV[0]|") or die "Cannot run tail:$!\n";
|
||||
while (<TAIL>) {
|
||||
print OUT;
|
||||
}
|
||||
close OUT;
|
||||
close TAIL;
|
||||
|
||||
@@ -1,10 +1,24 @@
|
||||
<tool id="Show tail1" name="Select last" version="1.0.0">
|
||||
<tool id="Show tail1" name="Select last" version="1.0.1" profile="20.05">
|
||||
<description>lines from a dataset</description>
|
||||
<command detect_errors="exit_code">
|
||||
perl '$__tool_directory__/tailWrapper.pl' '$input' $lineNum '$out_file1'</command>
|
||||
<requirements>
|
||||
<requirement type="package" version="8.31">coreutils</requirement>
|
||||
</requirements>
|
||||
<command detect_errors="exit_code"><![CDATA[
|
||||
set -eo pipefail;
|
||||
(
|
||||
#if $header
|
||||
head -n 1 '$input' &&
|
||||
tail -n +2 '$input'
|
||||
#else
|
||||
cat '$input'
|
||||
#end if
|
||||
| tail -n $lineNum
|
||||
) > '$out_file1'
|
||||
]]></command>
|
||||
<inputs>
|
||||
<param name="lineNum" type="integer" value="10" label="Select last" help="lines"/>
|
||||
<param name="lineNum" type="integer" value="10" min="1" label="Select last" help="lines"/>
|
||||
<param name="input" type="data" format="txt" label="from"/>
|
||||
<param name="header" type="boolean" label="Dataset has a header" help="if checked the header is kept and is not counted as a kept line"/>
|
||||
</inputs>
|
||||
<outputs>
|
||||
<data name="out_file1" format_source="input" metadata_source="input"/>
|
||||
@@ -13,7 +27,23 @@ perl '$__tool_directory__/tailWrapper.pl' '$input' $lineNum '$out_file1'</comman
|
||||
<test>
|
||||
<param name="lineNum" value="10"/>
|
||||
<param name="input" value="1.bed"/>
|
||||
<output name="out_file1" file="eq-showtail.dat"/>
|
||||
<output name="out_file1" file="eq-showtail.dat">
|
||||
<assert_contents>
|
||||
<not_has_text text="CCDS989.1_cds_0_0_chr1_147962193_r"/>
|
||||
<has_n_lines n="10"/>
|
||||
</assert_contents>
|
||||
</output>
|
||||
</test>
|
||||
<test>
|
||||
<param name="lineNum" value="10"/>
|
||||
<param name="input" value="1.bed"/>
|
||||
<param name="header" value="true"/>
|
||||
<output name="out_file1">
|
||||
<assert_contents>
|
||||
<has_text text="CCDS989.1_cds_0_0_chr1_147962193_r"/>
|
||||
<has_n_lines n="11"/>
|
||||
</assert_contents>
|
||||
</output>
|
||||
</test>
|
||||
</tests>
|
||||
<help>
|
||||
|
||||
Reference in New Issue
Block a user