diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index d3a88844ba0..53933175d6c 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -178,7 +178,7 @@ def add_file( dataset, json_file, output_path ): # See if we have a gzipped file, which, if it passes our restrictions, we'll uncompress is_gzipped, is_valid = check_gzip( dataset.path ) if is_gzipped and not is_valid: - file_err( 'The uploaded file contains inappropriate content', dataset, json_file ) + file_err( 'The gzipped uploaded file contains inappropriate content', dataset, json_file ) return elif is_gzipped and is_valid: # We need to uncompress the temp_name file, but BAM files must remain compressed in the BGZF format @@ -206,7 +206,7 @@ def add_file( dataset, json_file, output_path ): # See if we have a zip archive is_zipped, is_valid, test_ext = check_zip( dataset.path ) if is_zipped and not is_valid: - file_err( 'The uploaded file contains inappropriate content', dataset, json_file ) + file_err( 'The zipped uploaded file contains inappropriate content', dataset, json_file ) return elif is_zipped and is_valid: # Currently, we force specific tools to handle this case. We also require the user @@ -231,7 +231,7 @@ def add_file( dataset, json_file, output_path ): if len( parts ) > 1: ext = parts[1].strip().lower() if ext not in unsniffable_binary_formats: - file_err( 'The uploaded file contains inappropriate content', dataset, json_file ) + file_err( 'The uploaded binary file contains inappropriate content', dataset, json_file ) return elif ext in unsniffable_binary_formats and dataset.file_type != ext: err_msg = "You must manually set the 'File Format' to '%s' when uploading %s files." % ( ext.capitalize(), ext ) @@ -240,7 +240,7 @@ def add_file( dataset, json_file, output_path ): if not data_type: # We must have a text file if check_html( dataset.path ): - file_err( 'The uploaded file contains inappropriate content', dataset, json_file ) + file_err( 'The uploaded file contains inappropriate HTML content', dataset, json_file ) return if data_type != 'binary' and data_type != 'zip': # don't convert newlines on data we're only going to symlink diff --git a/tools/rgenetics/rgManQQ.py b/tools/rgenetics/rgManQQ.py index 5b41dc42a9e..f549820a47d 100644 --- a/tools/rgenetics/rgManQQ.py +++ b/tools/rgenetics/rgManQQ.py @@ -124,7 +124,14 @@ qq = function(pvector, title=NULL, spartan=F) { rcode2 = """rgqqMan = function(infile="%s",chromcolumn=%d, offsetcolumn=%d, pvalscolumns=%s, title="%s",grey=%d) { -d = read.table(infile,head=T,sep='\t') +rawd = read.table(infile,head=T,sep='\t') +dn = names(rawd) +cc = dn[chromcolumn] +oc = dn[offsetcolumn] +nams = c(cc,oc) +d = rawd[do.call(order,rawd[nams]),] +# mmmf - suggested by http://onertipaday.blogspot.com/2007/08/sortingordering-dataframe-according.html +# in case not yet ordered print(paste('###',length(d[,1]),'values read from',infile,'read - now running plots',sep=' ')) for (pvalscolumn in pvalscolumns) { if (pvalscolumn > 0) diff --git a/tools/rgenetics/rgQC.py b/tools/rgenetics/rgQC.py index a5ba96a7626..bfcc098a545 100644 --- a/tools/rgenetics/rgQC.py +++ b/tools/rgenetics/rgQC.py @@ -994,7 +994,7 @@ def markerRep(froot='cleantest',outfname="mrep",newfpath='.',logf=None,maplist=N mapdict = {} if maplist <> None: rslist = [x[1] for x in maplist] - offset = [x[3] for x in maplist] + offset = [(x[0],x[3]) for x in maplist] mapdict = dict(zip(rslist,offset)) hwefile = '%s.hwe' % froot lmissfile = '%s.lmiss' % froot @@ -1009,8 +1009,9 @@ def markerRep(froot='cleantest',outfname="mrep",newfpath='.',logf=None,maplist=N lmenddict = {} Tops = {} Tnames = ['Ranked Marker MAF', 'Ranked Marker Missing Genotype', 'Ranked Marker HWE', 'Ranked Marker Mendel'] - Tsorts = [2,5,9,10] + Tsorts = [3,6,10,11] Treverse = [False,True,True,True] # so first values are worse(r) + #res.append([rs,chrom,offset,maf,a1,a2,f_missing,hwe_all[0],hwe_all[1],hwe_unaff[0],hwe_unaff[1],nmend]) #rhead = ['snp','chrom','maf','a1','a2','missfrac','p_hwe_all','logp_hwe_all','p_hwe_unaff','logp_hwe_unaff','N_Mendel'] # -------------------hwe-------------------------- # hwe has SNP TEST GENO O(HET) E(HET) P_HWD @@ -1057,12 +1058,13 @@ def markerRep(froot='cleantest',outfname="mrep",newfpath='.',logf=None,maplist=N hwedict[rs] = {} markerlist.append(rs) chromlist.append(chrom) # one place to find it? + lpvals = 0 if ps.upper() <> 'NA' and ps.upper() <> 'NAN': # worth keeping lpvals = '0' if ps <> '1': try: pval = float(ps) - lpvals = '%f6' % -math.log10(pval) + lpvals = '%f' % -math.log10(pval) except: pass hwedict[rs][test] = (ps,lpvals) @@ -1147,49 +1149,33 @@ def markerRep(froot='cleantest',outfname="mrep",newfpath='.',logf=None,maplist=N rhead = ['snp','chromosome','offset','maf','a1','a2','missfrac','p_hwe_all','logp_hwe_all','p_hwe_unaff','logp_hwe_unaff','N_Mendel'] res = [] fres = [] - for i in xrange(len(markerlist)): # for each snp in found order - chrom = chromlist[i] - rs = markerlist[i] + for rs in markerlist: # for each snp in found order f_missing = lmissdict.get(rs,'NA') maf,a1,a2 = freqdict.get(rs,('NA','NA','NA')) hwe_all = hwedict[rs].get('ALL',('NA','NA')) # hope this doesn't change... hwe_unaff = hwedict[rs].get('UNAFF',('NA','NA')) nmend = lmenddict.get(rs,'NA') - offset=mapdict.get(rs,'0') + (chrom,offset)=mapdict.get(rs,('?','0')) res.append([rs,chrom,offset,maf,a1,a2,f_missing,hwe_all[0],hwe_all[1],hwe_unaff[0],hwe_unaff[1],nmend]) - try: - fmaf = '%f' % float(maf) - except: - fmaf = 'NA' - try: - inmend = '%d' % int(nmend) - except: - inmend = 'NA' - try: - fhweall = '%f' % float(hwe_all[1]) # the log value - except: - fhweall = 'NA' - try: - fhweunaff = '%f' % float(hwe_unaff[1]) # the log value - except: - fhweunaff = 'NA' - try: - ff_missing = '%f' % float(f_missing) # the log value - except: - ff_missing = 'NA' - #fres.append([rs,chrom,fmaf,a1,a2,ff_missing,hwe_all[0],hwe_all[1],hwe_unaff[0],fhwe,inmend]) - arow = [rs,chrom,offset,fmaf,a1,a2,ff_missing,hwe_all[0],fhweall,hwe_unaff[0],fhweunaff,inmend] - fres.append(arow) ntokeep = max(10,len(res)/keepfrac) + + def msortk(item=None): + """ + deal with non numeric sorting + """ + try: + return float(item) + except: + return item + for i,col in enumerate(Tsorts): - fres.sort(key=operator.itemgetter(col)) + res.sort(key=msortk(lambda x:x[col])) if Treverse[i]: - fres.reverse() + res.reverse() repname = Tnames[i] - Tops[repname] = fres[0:ntokeep] - #Tops[repname] = [map(str,x) for x in Tops[repname]] + Tops[repname] = res[0:ntokeep] Tops[repname].insert(0,rhead) - res.sort() + res.sort(key=lambda x: '%s_%10d' % (x[1].ljust(4,'0'),int(x[2]))) # in chrom offset order res.insert(0,rhead) f = open(outfile,'w') f.write('\n'.join(['\t'.join(x) for x in res])) @@ -1315,10 +1301,6 @@ if __name__ == "__main__": dat = subjectTops.get(ttitle,None) if not dat: dat = markerTops.get(ttitle,None) - if not dat: - print '## iterating plotpage - cannot find title=%s/ttitle=%s' % (title,ttitle) - else: - print '## iterating plotpage - found title=%s/ttitle=%s' % (title,ttitle) imghref = '%s.jpg' % os.path.splitext(url)[0] # removes .pdf thumbnail = os.path.join(newfpath,imghref) if not os.path.exists(thumbnail): # for multipage pdfs, mogrify makes multiple jpgs - fugly hack diff --git a/tools/rgenetics/rgtest_one_tool.sh b/tools/rgenetics/rgtest_one_tool.sh new file mode 100644 index 00000000000..c12d32f9dde --- /dev/null +++ b/tools/rgenetics/rgtest_one_tool.sh @@ -0,0 +1,218 @@ +#!/bin/sh +# script to generate all functional test outputs for each rgenetics tool +# could be run at installation to ensure all dependencies are in place? +case $# in 0) echo "USAGE: ${0##*/} TooltoTest"; exit 1;; + [2-10]*) echo "Too many arguments - name of tool only"; exit 2;; + *) +esac +GALAXYROOT=`pwd` +echo "using $GALAXYROOT" +# change this as needed for your local install +INPATH="${GALAXYROOT}/test-data" +BINPATH="${GALAXYROOT}/tool-data/rg/bin" +TOOLPATH="${GALAXYROOT}/tools/rgenetics" +OROOT="${GALAXYROOT}/test-data/rgtestouts" +NORMALOROOT="${GALAXYROOT}/test-data" +case "$1" in +'rgManQQ') + +TOOL="rgManQQ" +NPRE=${TOOL}test1 +OUTPATH="$OROOT/$TOOL" +rm -rf $OUTPATH/* +CL="python $TOOLPATH/$TOOL.py "$INPATH/smallwgaP.xls" $NPRE ${OUTPATH}/${NPRE}.html $OUTPATH 1 2 5,7 0" +# rgManQQ.py '$input_file' "$name" '$out_html' '$out_html.files_path' '$chrom_col' '$offset_col' +# '$pval_col' +#python /opt/galaxy/tools/rgenetics/rgManQQ.py /opt/galaxy/test-data/smallwgaP.xls rgManQQtest1 +#/opt/galaxy/test-data/rgtestouts/rgManQQ/rgManQQtest1.html /opt/galaxy/test-data/rgtestouts/rgManQQ 1 2 5,7 +echo "Testing $TOOL using $CL" +python $TOOLPATH/$TOOL.py "$INPATH/smallwgaP.xls" $NPRE ${OUTPATH}/${NPRE}.html $OUTPATH 1 2 5,7 0 +;; + +'rgfakePhe') +TOOL="rgfakePhe" +NPRE=${TOOL}test1 +OUTPATH="$OROOT/$TOOL" +rm -rf $OUTPATH/* +PSSCRIPT="$OUTPATH/script_file" +echo "{'pN':'normtest','pT':'rnorm','pP':\"{'Mean':'100', 'SD':'10'}\"}" > $PSSCRIPT +echo "{'pN':'cattest','pT':'cat','pP':\"{'values':'red,green,blue'}\"}" >> $PSSCRIPT +echo "{'pN':'uniftest','pT':'$f.series.phetype','pP':\"{'low':'1','hi':'100'}\"}" >> $PSSCRIPT +echo "{'pN':'gammatest','pT':'rgamma','pP':\"{'Alpha':'1', 'Beta':'0.1'}\"}" >> $PSSCRIPT +echo "{'pN':'poissontest','pT':'poisson','pP':\"{'lamb':'1.0',}\"}" >> $PSSCRIPT +echo "{'pN':'exptest','pT':'exponential','pP':\"{'Mean':'100.0',}\"}" >> $PSSCRIPT +echo "{'pN':'weibtest','pT':'weibull','pP':\"{'Alpha':'1.0', 'Beta':'0.1'}\"}" >> $PSSCRIPT +echo "now doing $TOOL" +python $TOOLPATH/$TOOL.py ${INPATH}/tinywga $NPRE $NPRE.pphe $OUTPATH $PSSCRIPT +# rgfakePhe.py '$infile1.extra_files_path/$infile1.metadata.base_name' +# "$title1" '$ppheout' '$ppheout.files_path' '$script_file' +# +;; +'rgQC') + +TOOL="rgQC" +NPRE=${TOOL}test1 +echo "now doing $TOOL" +OUTPATH="$OROOT/$TOOL" +rm -rf $OUTPATH/* +python $TOOLPATH/$TOOL.py -i "$INPATH/tinywga" -o $NPRE -s ${OUTPATH}/${NPRE}.html -p $OUTPATH +# rgQC.py -i '$input_file.extra_files_path/$input_file.metadata.base_name' -o "$out_prefix" +# -s '$html_file' -p '$html_file.files_path' +# +;; + +'rgGRR') +TOOL="rgGRR" +NPRE=${TOOL}test1 +echo "now doing $TOOL" +OUTPATH="$OROOT/$TOOL" +rm -rf $OUTPATH/* +python $TOOLPATH/$TOOL.py "$INPATH/tinywga" "tinywga" $OUTPATH/${NPRE}.html $OUTPATH "$NPRE" '100' '6' +# rgGRR.py $i.extra_files_path/$i.metadata.base_name "$i.metadata.base_name" +#'$out_file1' '$out_file1.files_path' "$title" '$n' '$Z' +;; +'rgLDIndep') +TOOL="rgLDIndep" +NPRE=${TOOL}test1 +OUTPATH="$OROOT/$TOOL" +rm -rf $OUTPATH/* +python $TOOLPATH/$TOOL.py "$INPATH" "tinywga" "$NPRE" 1 1 0 0 1 1 $OUTPATH/${NPRE}.pbed $OUTPATH 10000 5000 0.1 +#rgLDIndep.py '$input_file.extra_files_path' '$input_file.metadata.base_name' '$title' '$mind' +# '$geno' '$hwe' '$maf' '$mef' '$mei' '$out_file1' +#'$out_file1.files_path' '$window' '$step' '$r2' +;; + +'rgPedSub') +TOOL="rgPedSub" +NPRE=${TOOL}test1 +OUTPATH="$OROOT/$TOOL" +rm -rf $OUTPATH/* +PSSCRIPT="$OUTPATH/pedsub.script" +echo "title~~~~$NPRE" > $PSSCRIPT +echo "output1~~~~${OUTPATH}/${NPRE}.lped" >> $PSSCRIPT +echo "outformat~~~~lped" >> $PSSCRIPT +echo "basename~~~~tinywga" >> $PSSCRIPT +echo "inped~~~~$INPATH/tinywga" >> $PSSCRIPT +echo "outdir~~~~$OUTPATH" >> $PSSCRIPT +echo "region~~~~" >> $PSSCRIPT +echo "relfilter~~~~all" >> $PSSCRIPT +echo "rslist~~~~rs2283802Xrs2267000Xrs16997606Xrs4820537Xrs3788347Xrs756632Xrs4820539Xrs2283804Xrs2267006Xrs4822363X" >> $PSSCRIPT +echo "now doing $TOOL" +python $TOOLPATH/$TOOL.py $PSSCRIPT +rm -rf $PSSCRIPT +;; + +'rgfakePhe') + +TOOL="rgfakePhe" +NPRE=${TOOL}test1 +OUTPATH="$OROOT/$TOOL" +rm -rf $OUTPATH/* +PSSCRIPT="$OUTPATH/script_file" +echo "{'pN':'normtest','pT':'rnorm','pP':\"{'Mean':'100', 'SD':'10'}\"}" > $PSSCRIPT +echo "{'pN':'cattest','pT':'cat','pP':\"{'values':'red,green,blue'}\"}" >> $PSSCRIPT +echo "{'pN':'uniftest','pT':'$f.series.phetype','pP':\"{'low':'1','hi':'100'}\"}" >> $PSSCRIPT +echo "{'pN':'gammatest','pT':'rgamma','pP':\"{'Alpha':'1', 'Beta':'0.1'}\"}" >> $PSSCRIPT +echo "{'pN':'poissontest','pT':'poisson','pP':\"{'lamb':'1.0',}\"}" >> $PSSCRIPT +echo "{'pN':'exptest','pT':'exponential','pP':\"{'Mean':'100.0',}\"}" >> $PSSCRIPT +echo "{'pN':'weibtest','pT':'weibull','pP':\"{'Alpha':'1.0', 'Beta':'0.1'}\"}" >> $PSSCRIPT +echo "now doing $TOOL" +python $TOOLPATH/$TOOL.py $PSSCRIPT +;; + +'rgClean') +TOOL="rgClean" +NPRE=${TOOL}test1 +OUTPATH="$OROOT/$TOOL" +rm -rf $OUTPATH/* +python $TOOLPATH/$TOOL.py $INPATH "tinywga" "$NPRE" 1 1 0 0 1 1 $OUTPATH/${NPRE}.pbed $OUTPATH 0 0 0 0 +# rgClean.py '$input_file.extra_files_path' '$input_file.metadata.base_name' '$title' '$mind' +# '$geno' '$hwe' '$maf' '$mef' '$mei' '$out_file1' '$out_file1.files_path' +# '${GALAXY_DATA_INDEX_DIR}/rg/bin/plink' '$relfilter' '$afffilter' '$sexfilter' '$fixaff' +# +;; + +'rgEigPCA') + +TOOL="rgEigPCA" +NPRE=${TOOL}test1 +OUTPATH="$OROOT/$TOOL" +rm -rf $OUTPATH/* +python $TOOLPATH/$TOOL.py "$INPATH/tinywga" "$NPRE" ${OUTPATH}/${NPRE}.html $OUTPATH 4 2 2 2 $OUTPATH/rgEigPCAtest1.txt +# rgEigPCA.py "$i.extra_files_path/$i.metadata.base_name" "$title" "$out_file1" +# "$out_file1.files_path" "$k" "$m" "$t" "$s" "$pca" +# +;; + +'rgfakePed') +TOOL="rgfakePed" +NPRE=${TOOL}test1 +OUTPATH="$OROOT/$TOOL" +rm -rf $OUTPATH/* +echo "now doing $TOOL" +python $TOOLPATH/$TOOL.py --title "$NPRE" -o $OUTPATH/${NPRE}.lped -p $OUTPATH -c "20" -n "40" -s "10" -w "0" -v "0" -l "pbed" -d "T" -m "0" -M "0" +#rgfakePed.py --title '$title1' +# -o '$out_file1' -p '$out_file1.extra_files_path' -c '$ncases' -n '$ntotal' +# -s '$nsnp' -w '$lowmaf' -v '$missingValue' -l '$outFormat' +# -d '$mafdist' -m '$missingRate' -M '$mendelRate' +;; + +'rgHaploView') + +TOOL="rgHaploView" +NPRE=${TOOL}test1 +OUTPATH="$OROOT/$TOOL" +rm -rf $OUTPATH/* +python $TOOLPATH/$TOOL.py "" "rs2283802Xrs2267000Xrs16997606Xrs4820537Xrs3788347Xrs756632Xrs4820539Xrs2283804Xrs2267006Xrs4822363X" \ +"$NPRE" $OUTPATH/${NPRE}.html "$INPATH" "tinywga" 0.0 200000 "RSQ" "lo" "2048" "$OUTPATH" "noinfo" "0.8" \ +"YRI" $BINPATH/haploview.jar +# rgHaploView.py "$ucsc_region" "$rslist" "$title" "$output1" +# "$lhistIn.extra_files_path" "$lhistIn.metadata.base_name" +# "$minmaf" "$maxdist" "$ldtype" "$hires" "$memsize" "$output1.files_path" +# "$infoTrack" "$tagr2" "$hmpanel" ${GALAXY_DATA_INDEX_DIR}/rg/bin/haploview.jar +# note these statistical tools do NOT generate composite outputs +;; + +'rgGLM') +TOOL="rgGLM" +NPRE=${TOOL}test1 +OUTPATH=$NORMALOROOT +python $TOOLPATH/$TOOL.py "$INPATH/tinywga" $INPATH/tinywga "$NPRE" "c1" "" $OUTPATH/${NPRE}_GLM.xls \ +$OUTPATH/${NPRE}_GLM_log.txt "tinywga" "" "" "" 1 1 0 0 $OUTPATH/${NPRE}_GLM_topTable.gff +## rgGLM.py '$i.extra_files_path/$i.metadata.base_name' '$phef.extra_files_path/$phef.metadata.base_name' +## "$title1" '$predvar' '$covar' '$out_file1' '$logf' '$dbkey' '$i.metadata.base_name' +## '$inter' '$cond' '$gender' '$mind' '$geno' '$maf' '$logistic' '$gffout' +;; + +'rgTDT') +TOOL="rgTDT" +NPRE=${TOOL}test1 +OUTPATH=$NORMALOROOT +python $TOOLPATH/$TOOL.py -i "$INPATH/tinywga" -o "$NPRE" -r $OUTPATH/${NPRE}_TDT.xls \ +-l $OUTPATH/${NPRE}_TDT_log.txt -g $OUTPATH/${NPRE}_TDT_topTable.gff +## rgTDT.py -i '$infile.extra_files_path/$infile.metadata.base_name' -o '$title' +## -r '$out_file1' -l '$logf' -x '${GALAXY_DATA_INDEX_DIR}/rg/bin/plink' +## -g '$gffout' +;; + +'rgCaCo') +TOOL="rgCaCo" +NPRE=${TOOL}test1 +OUTPATH=$NORMALOROOT +echo "now doing $TOOL" +python $TOOLPATH/rgCaCo.py $INPATH/tinywga "$NPRE" $OUTPATH/${NPRE}_CaCo.xls $OUTPATH/${NPRE}_CaCo_log.txt $OUTPATH $OUTPATH/${NPRE}_CaCo_topTable.gff +# echo tp=$TOOLPATH t=$TOOL op=$OUTPATH b=$BINPATH +# rgCaCo.py '$i.extra_files_path/$i.metadata.base_name' "$name" '$out_file1' '$logf' '$logf.files_path' '$gffout' +;; + +'rgQQ') +TOOL="rgQQ" +echo "now doing $TOOL" +NPRE=${TOOL}test1 +OUTPATH=$NORMALOROOT +CL="python $TOOLPATH/$TOOL.py "$INPATH/tinywga.pphe" "$NPRE" 1 3 $OUTPATH/$NPRE.pdf 8 10 "false" 1 $OUTPATH" +echo "running $TOOL using $CL" +python $TOOLPATH/$TOOL.py "$INPATH/tinywga.pphe" "$NPRE" 1 3 $OUTPATH/$NPRE.pdf 8 10 "false" 1 $OUTPATH +# rgQQ.py "$input1" "$name" $sample "$cols" $allqq $height $width $log $allqq.id $__new_file_path__ +;; +esac