diff --git a/.hgignore b/.hgignore index 4189d7b6255..bddfc20a828 100644 --- a/.hgignore +++ b/.hgignore @@ -13,6 +13,7 @@ scripts/scramble/archives # Database stuff database/beaker_sessions +database/community_files database/compiled_templates database/files database/pbs diff --git a/README.txt b/README.txt index 9496410db7b..fd2f7f88bb9 100644 --- a/README.txt +++ b/README.txt @@ -28,4 +28,4 @@ on adding tools can be found on the Galaxy website (linked above). Not all dependencies are included for the tools provided in the sample tool_conf.xml. A full list of external dependencies is available at: -http://bitbucket.org/galaxy/galaxy-central/wiki/ToolDependencies +http://wiki.g2.bx.psu.edu/Admin/Tools/Tool%20Dependencies diff --git a/community_datatypes_conf.xml.sample b/community_datatypes_conf.xml.sample deleted file mode 100644 index 5373e562f0b..00000000000 --- a/community_datatypes_conf.xml.sample +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - diff --git a/community_wsgi.ini.sample b/community_wsgi.ini.sample index 9326a5551fa..55b67ce5d4c 100644 --- a/community_wsgi.ini.sample +++ b/community_wsgi.ini.sample @@ -12,9 +12,6 @@ threadpool_workers = 10 [app:main] -# Enable next-gen tool shed features -#enable_next_gen_tool_shed = True - # Specifies the factory for the universe WSGI application paste.app_factory = galaxy.webapps.community.buildapp:app_factory log_level = DEBUG @@ -58,6 +55,13 @@ require_login = False # path to sendmail sendmail_path = /usr/sbin/sendmail +# For use by email messages sent from the tool shed +#smtp_server = smtp.your_tool_shed_server +#email_from = your_tool_shed_email@server + +# The URL linked by the "Support" link in the "Help" menu. +#support_url = http://wiki.g2.bx.psu.edu/Support + # Write thread status periodically to 'heartbeat.log' (careful, uses disk space rapidly!) ## use_heartbeat = True diff --git a/contrib/README b/contrib/README index f3868af6129..8bd81b76e51 100644 --- a/contrib/README +++ b/contrib/README @@ -23,3 +23,9 @@ galaxy.solaris-smf.xml: SMF Manifest for Solaris 10 and OpenSolaris. Import with `svccfg import galaxy.solaris-smf.xml`. + +gls.pl: + + "Galaxy ls", for sites where Galaxy logins match system logins, this script + can be used to list the filesystem paths to a user's history datasets. + Requires site modifications. Written and submitted by Simon McGowan. diff --git a/contrib/collect_sge_job_timings.sh b/contrib/collect_sge_job_timings.sh new file mode 100644 index 00000000000..60ac8433ca3 --- /dev/null +++ b/contrib/collect_sge_job_timings.sh @@ -0,0 +1,126 @@ +#!/bin/sh + +## +## CHANGE ME to galaxy's database name +## +DATABASE=galaxyprod + +## +## AWK script to extract the relevant fields of SGE's qacct report +## and write them all in one line. +AWKSCRIPT=' +$1=="jobnumber" { job_number = $2 } +$1=="qsub_time" { qsub_time = $2 } +$1=="start_time" { start_time = $2 } +$1=="end_time" { end_time = $2 + print job_number, qsub_time, start_time, end_time +} +' + +FIFO=$(mktemp -u) || exit 1 +mkfifo "$FIFO" || exit 1 + +## +## Write the SGE/QACCT job report into a pipe +## (later will be loaded into a temporary table) +qacct -j | + egrep "jobnumber|qsub_time|start_time|end_time" | + sed 's/ */\t/' | + awk -v FS="\t" -v OFS="\t" "$AWKSCRIPT" | + grep -v -- "-/-" > "$FIFO" & + +## +## The SQL to generate the report +## +SQL=" +-- +-- Temporary table which contains the qsub/start/end times, based on SGE's qacct report. +-- +CREATE TEMPORARY TABLE sge_times ( + sge_job_id INTEGER PRIMARY KEY, + qsub_time TIMESTAMP WITHOUT TIME ZONE, + start_time TIMESTAMP WITHOUT TIME ZONE, + end_time TIMESTAMP WITHOUT TIME ZONE +); + +COPY sge_times FROM '$FIFO' ; + +-- +-- Temporary table which contains a unified view of all galaxy jobs. +-- for each job: +-- the user name, total input size (bytes), and input file types, DBKEY +-- creation time, update time, SGE job runner parameters +-- If a job had more than one input file, then some parameters might not be accurate (e.g. DBKEY) +-- as one will be chosen arbitrarily +CREATE TEMPORARY TABLE job_input_sizes AS +SELECT + job.job_runner_external_id as job_runner_external_id, + min(job.id) as job_id, + min(job.create_time) as job_create_time, + min(job.update_time) as job_update_time, + min(galaxy_user.email) as email, + min(job.tool_id) as tool_name, +-- This hack requires a user-custom aggregate function, comment it out for now +-- textcat_all(hda.extension || ' ') as file_types, + sum(dataset.file_size) as total_input_size, + count(dataset.file_size) as input_dataset_count, + min(job.job_runner_name) as job_runner_name, +-- This hack tries to extract the DBKEY attribute from the metadata JSON string + min(substring(encode(metadata,'escape') from '\"dbkey\": \\\\[\"(.*?)\"\\\\]')) as dbkey +FROM + job, + galaxy_user, + job_to_input_dataset, + history_dataset_association hda, + dataset +WHERE + job.user_id = galaxy_user.id + AND + job.id = job_to_input_dataset.job_id + AND + hda.id = job_to_input_dataset.dataset_id + AND + dataset.id = hda.dataset_id + AND + job.job_runner_external_id is not NULL +GROUP BY + job.job_runner_external_id; + + +-- +-- Join the two temporary tables, create a nice report +-- +SELECT + job_input_sizes.job_runner_external_id as sge_job_id, + job_input_sizes.job_id as galaxy_job_id, + job_input_sizes.email, + job_input_sizes.tool_name, +-- ## SEE previous query for commented-out filetypes field +-- job_input_sizes.file_types, + job_input_sizes.job_runner_name as sge_params, + job_input_sizes.dbkey, + job_input_sizes.total_input_size, + job_input_sizes.input_dataset_count, + job_input_sizes.job_update_time - job_input_sizes.job_create_time as galaxy_total_time, + sge_times.end_time - sge_times.qsub_time as sge_total_time, + sge_times.start_time - sge_times.qsub_time as sge_waiting_time, + sge_times.end_time - sge_times.start_time as sge_running_time, + job_input_sizes.job_create_time as galaxy_job_create_time +-- ## no need to show the exact times, the deltas (above) are informative enough +-- job_input_sizes.job_update_time as galaxy_job_update_time, +-- sge_times.qsub_time as sge_qsub_time, +-- sge_times.start_time as sge_start_time, +-- sge_times.end_time as sge_end_time +FROM + job_input_sizes +LEFT OUTER JOIN + SGE_TIMES +ON (job_input_sizes.job_runner_external_id = sge_times.sge_job_id) +ORDER BY + galaxy_job_create_time + +" + +echo "$SQL" | psql --pset "footer=off" -F" " -A --quiet "$DATABASE" + + diff --git a/contrib/gls.pl b/contrib/gls.pl new file mode 100644 index 00000000000..75995dab43e --- /dev/null +++ b/contrib/gls.pl @@ -0,0 +1,204 @@ +#/!/usr/bin/env perl -w + +=head1 NAME + + gls + +=head1 DESCRIPTION + + Display the files generated by the current user within the local instance of Galaxy. + Information is grouped by user's Galaxy histories and ordered by date/time + +=head1 OPTIONS + + - i|info = show more info about file [default = no] + - e|error = show error files [default = no] + - d|dirname = show files from this dirname (ie, history name) only (NOTE. if history name contains spaces, it should be quoted) + - n|nocontent = show empty files [default = no] + - a|altuser = supply an alternative username (nb, admin only) + - h|help = help + - m|man = man + +=head1 AUTHOR + + Simon McGowan, CBRG [Computational Biology Research Group, Oxford University, UK] + +=head1 Update Record + + 23/09/2010 001 S.McGowan first written + +=cut + + + +use strict; +use Data::Dumper; +use DBI; +use Getopt::Long; +use Pod::Usage; + +my $show_file_info = 0; +my $show_error_files = 0; +my $show_empty_files = 0; +my $help = 0; +my $man = 0; +my $alt_user; +my $selected_dir; + +GetOptions( + 'h|help'=>\$help, + 'm|man'=>\$man, + 'i|info'=>\$show_file_info, + 'e|error'=>\$show_error_files, + 'a|altuser=s'=>\$alt_user, + 'n|nocontent'=>\$show_empty_files, + 'd|dirname=s'=>\$selected_dir +); + +pod2usage(1) if $help; +pod2usage(-verbose=>2) if $man; + +my %history_data; +my %file_data; + + +############ CONFIG ######################################## +# list of admin usernames: +my %admin; +$admin{simonmcg} = ''; +$admin{stevetay} = ''; + +# institute email domain +my $email_domain = '@molbiol.ox.ac.uk'; + +# mysql db +my $mysql_database = 'galaxy'; +my $mysql_host = 'xxxxxxxx'; +my $mysql_username = 'xxxxxxxx'; +my $mysql_password = 'xxxxxxxx'; + +# file path +my $db_root_dir = '/wwwdata/galaxy-prod/database/files/'; +############################################################## + +#----------------------------------------------------------------------------------- + +my $current_user = getlogin(); + +# allow admin to list any user's galaxy files: +if (exists($admin{$current_user})) +{ + if ($alt_user) {$current_user = $alt_user;} +} + +&get_data; + +&print_galaxy_data; + +exit(); + +#----------------------------------------------------------------------------------- + +sub get_data +{ + my $dbh = DBI->connect("DBI:mysql:database=$mysql_database;host=" . $mysql_host, $mysql_username, $mysql_password, {'RaiseError' => 1}); + + my $sql = "SELECT h.id, h.name, h.create_time, hda.dataset_id, hda.update_time, hda.name, hda.info, hda.blurb, hda.extension, d.file_size + FROM history h, history_dataset_association hda, galaxy_user g, dataset d + WHERE g.email = '$current_user$email_domain' + AND g.id = h.user_id + AND h.id = hda.history_id + AND hda.dataset_id = d.id"; + + my $sth = $dbh->prepare($sql) or die("Failed to prepare statement $sql\n"); + $sth->execute() or die("Can't perform SQL $sql : $DBI::errstr\n"); + while (my $ref = $sth->fetch) + { + my ($history_id, $history_name, $create_time, $dataset_id, $dataset_time, $dataset_name, $info, $blurb, $ext, $file_size) = @{$ref}; + + #print "$history_id, $history_name, $create_time, $dataset_id, $dataset_time, $dataset_name, $info, $blurb, $ext, $file_size\n\n"; + + $history_data{$history_id}{create_time} = $create_time; + $history_data{$history_id}{history_name} = $history_name; + + $file_data{$history_id}{$dataset_id}{dataset_update_time} = $dataset_time; + $file_data{$history_id}{$dataset_id}{dataset_name} = $dataset_name; + $file_data{$history_id}{$dataset_id}{info} = $info; + $file_data{$history_id}{$dataset_id}{blurb} = $blurb; + $file_data{$history_id}{$dataset_id}{file_size} = $file_size; + $file_data{$history_id}{$dataset_id}{ext} = $ext; + } + $sth->finish; +} + +sub print_galaxy_data +{ + foreach my $hist_id (sort numerically keys %history_data) + { + my $hist_name = $history_data{$hist_id}{history_name}; + if ($selected_dir) + { + # if the user has opted to see just one dir... + unless($hist_name eq $selected_dir) {next;} + } + + my $hist_date = $history_data{$hist_id}{create_time}; + + print "\n"; + print "$hist_date - $hist_name\n"; + + foreach my $dataset_id (sort numerically keys %{$file_data{$hist_id}}) + { + my $dataset_time = $file_data{$hist_id}{$dataset_id}{dataset_update_time}; + my $dataset_name = $file_data{$hist_id}{$dataset_id}{dataset_name}; + my $info = $file_data{$hist_id}{$dataset_id}{info}; + my $blurb = $file_data{$hist_id}{$dataset_id}{blurb}; + my $file_size = $file_data{$hist_id}{$dataset_id}{file_size}; + my $ext = $file_data{$hist_id}{$dataset_id}{ext}; + my $file_path = &derive_file_path($dataset_id); + + if (($blurb) and ($blurb eq 'empty')) + { + unless ($show_empty_files) {next;} + } + if (($blurb) and ($blurb eq 'error')) + { + unless ($show_error_files) {next;} + } + + print "\t$dataset_time - $dataset_name"; + print " $file_path"; + if ($show_file_info) + { + print " [size:$file_size; type:$ext;"; + if (($info) and ($blurb)) { print " $info; $blurb"; } + elsif ($info) { print " $info"; } + elsif ($blurb) { print " $blurb"; } + print ']'; + } + print "\n"; + } + } +} + +sub derive_file_path +{ + my ($dataset_id) = @_; + my $dir = sprintf("%06d", $dataset_id); + $dir =~ s/\d\d\d$//; + my $full_path = $db_root_dir . $dir . '/dataset_' . $dataset_id . '.dat'; + return ($full_path); +} + +sub numerically +{ + $a <=> $b; +} + + + + + + + + diff --git a/datatypes_conf.xml.sample b/datatypes_conf.xml.sample index ed57048ab2a..053009b4905 100644 --- a/datatypes_conf.xml.sample +++ b/datatypes_conf.xml.sample @@ -1,311 +1,352 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dist-eggs.ini b/dist-eggs.ini index 8c68c59f249..969050fa676 100644 --- a/dist-eggs.ini +++ b/dist-eggs.ini @@ -3,7 +3,7 @@ ; eggs.g2.bx.psu.edu). Probably only useful to Galaxy developers at ; Penn State. This file is used by scripts/dist-scramble.py ; -; More information: http://bitbucket.org/galaxy/galaxy-central/wiki/Config/Eggs +; More information: http://wiki.g2.bx.psu.edu/Admin/Config/Eggs ; [hosts] @@ -13,33 +13,44 @@ py2.5-linux-i686-ucs2 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/lin py2.5-linux-i686-ucs4 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs4/bin/python2.5 py2.6-linux-i686-ucs2 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs2/bin/python2.6 py2.6-linux-i686-ucs4 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs4/bin/python2.6 +py2.7-linux-i686-ucs2 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs2/bin/python2.7 +py2.7-linux-i686-ucs4 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs4/bin/python2.7 py2.4-linux-x86_64-ucs2 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs2/bin/python2.4 py2.4-linux-x86_64-ucs4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.4 py2.5-linux-x86_64-ucs2 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs2/bin/python2.5 py2.5-linux-x86_64-ucs4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.5 py2.6-linux-x86_64-ucs2 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs2/bin/python2.6 py2.6-linux-x86_64-ucs4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.6 -py2.4-macosx-10.3-fat-ucs2 = lonnie.bx.psu.edu /usr/local/bin/python2.4 -py2.5-macosx-10.3-fat-ucs2 = lonnie.bx.psu.edu /usr/local/bin/python2.5 -py2.6-macosx-10.3-fat-ucs2 = lonnie.bx.psu.edu /usr/local/bin/python2.6 -py2.6-macosx-10.6-universal-ucs2 = fanty.bx.psu.edu /usr/bin/python2.6 +py2.7-linux-x86_64-ucs2 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs2/bin/python2.7 +py2.7-linux-x86_64-ucs4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.7 +py2.4-macosx-10.3-fat-ucs2 = weyerbacher.bx.psu.edu /usr/local/bin/python2.4 +py2.5-macosx-10.3-fat-ucs2 = weyerbacher.bx.psu.edu /usr/local/bin/python2.5 +py2.6-macosx-10.3-fat-ucs2 = weyerbacher.bx.psu.edu /usr/local/bin/python2.6 +py2.7-macosx-10.3-fat-ucs2 = weyerbacher.bx.psu.edu /usr/local/bin/python2.7 +py2.6-macosx-10.6-universal-ucs2 = lion.bx.psu.edu /usr/bin/python2.6 +py2.7-macosx-10.6-intel-ucs2 = lion.bx.psu.edu /usr/local/bin/python2.7 py2.4-solaris-2.10-i86pc_32-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_32-ucs2/bin/python2.4 py2.5-solaris-2.10-i86pc_32-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_32-ucs2/bin/python2.5 py2.6-solaris-2.10-i86pc_32-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_32-ucs2/bin/python2.6 +py2.7-solaris-2.10-i86pc_32-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_32-ucs2/bin/python2.7 py2.4-solaris-2.10-i86pc_64-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_64-ucs2/bin/python2.4 py2.5-solaris-2.10-i86pc_64-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_64-ucs2/bin/python2.5 py2.6-solaris-2.10-i86pc_64-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_64-ucs2/bin/python2.6 +py2.7-solaris-2.10-i86pc_64-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_64-ucs2/bin/python2.7 py2.4-solaris-2.10-sun4u_32-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_32-ucs2/bin/python2.4 py2.5-solaris-2.10-sun4u_32-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_32-ucs2/bin/python2.5 py2.6-solaris-2.10-sun4u_32-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_32-ucs2/bin/python2.6 +py2.7-solaris-2.10-sun4u_32-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_32-ucs2/bin/python2.7 py2.4-solaris-2.10-sun4u_64-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_64-ucs2/bin/python2.4 py2.5-solaris-2.10-sun4u_64-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_64-ucs2/bin/python2.5 py2.6-solaris-2.10-sun4u_64-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_64-ucs2/bin/python2.6 +py2.7-solaris-2.10-sun4u_64-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_64-ucs2/bin/python2.7 ; these hosts are used to build eggs with no C extensions py2.4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.4 py2.5 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.5 py2.6 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.6 +py2.7 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.7 [groups] py2.4-linux-i686 = py2.4-linux-i686-ucs2 py2.4-linux-i686-ucs4 @@ -48,34 +59,42 @@ py2.5-linux-i686 = py2.5-linux-i686-ucs2 py2.5-linux-i686-ucs4 py2.5-linux-x86_64 = py2.5-linux-x86_64-ucs2 py2.5-linux-x86_64-ucs4 py2.6-linux-i686 = py2.6-linux-i686-ucs2 py2.6-linux-i686-ucs4 py2.6-linux-x86_64 = py2.6-linux-x86_64-ucs2 py2.6-linux-x86_64-ucs4 +py2.7-linux-i686 = py2.7-linux-i686-ucs2 py2.7-linux-i686-ucs4 +py2.7-linux-x86_64 = py2.7-linux-x86_64-ucs2 py2.7-linux-x86_64-ucs4 py2.4-linux = py2.4-linux-i686 py2.4-linux-x86_64 py2.5-linux = py2.5-linux-i686 py2.5-linux-x86_64 py2.6-linux = py2.6-linux-i686 py2.6-linux-x86_64 -linux-i686 = py2.4-linux-i686 py2.5-linux-i686 py2.6-linux-i686 -linux-x86_64 = py2.4-linux-x86_64 py2.5-linux-x86_64 py2.6-linux-x86_64 +py2.7-linux = py2.7-linux-i686 py2.7-linux-x86_64 +linux-i686 = py2.4-linux-i686 py2.5-linux-i686 py2.6-linux-i686 py2.7-linux-i686 +linux-x86_64 = py2.4-linux-x86_64 py2.5-linux-x86_64 py2.6-linux-x86_64 py2.7-linux-x86_64 linux = linux-i686 linux-x86_64 py2.4-macosx = py2.4-macosx-10.3-fat-ucs2 py2.5-macosx = py2.5-macosx-10.3-fat-ucs2 py2.6-macosx = py2.6-macosx-10.3-fat-ucs2 py2.6-macosx-10.6-universal-ucs2 -macosx = py2.4-macosx py2.5-macosx py2.6-macosx +py2.7-macosx = py2.7-macosx-10.3-fat-ucs2 py2.7-macosx-10.6-intel-ucs2 +macosx = py2.4-macosx py2.5-macosx py2.6-macosx py2.7-macosx py2.4-solaris-i86pc = py2.4-solaris-2.10-i86pc_32-ucs2 py2.4-solaris-2.10-i86pc_64-ucs2 py2.5-solaris-i86pc = py2.5-solaris-2.10-i86pc_32-ucs2 py2.5-solaris-2.10-i86pc_64-ucs2 py2.6-solaris-i86pc = py2.6-solaris-2.10-i86pc_32-ucs2 py2.6-solaris-2.10-i86pc_64-ucs2 +py2.7-solaris-i86pc = py2.7-solaris-2.10-i86pc_32-ucs2 py2.7-solaris-2.10-i86pc_64-ucs2 py2.4-solaris-sun4u = py2.4-solaris-2.10-sun4u_32-ucs2 py2.4-solaris-2.10-sun4u_64-ucs2 py2.5-solaris-sun4u = py2.5-solaris-2.10-sun4u_32-ucs2 py2.5-solaris-2.10-sun4u_64-ucs2 py2.6-solaris-sun4u = py2.6-solaris-2.10-sun4u_32-ucs2 py2.6-solaris-2.10-sun4u_64-ucs2 +py2.7-solaris-sun4u = py2.7-solaris-2.10-sun4u_32-ucs2 py2.7-solaris-2.10-sun4u_64-ucs2 py2.4-solaris = py2.4-solaris-i86pc py2.4-solaris-sun4u py2.5-solaris = py2.5-solaris-i86pc py2.5-solaris-sun4u py2.6-solaris = py2.6-solaris-i86pc py2.6-solaris-sun4u -solaris-i86pc = py2.4-solaris-i86pc py2.5-solaris-i86pc py2.6-solaris-i86pc -solaris-sun4u = py2.4-solaris-sun4u py2.5-solaris-sun4u py2.6-solaris-sun4u +py2.7-solaris = py2.7-solaris-i86pc py2.7-solaris-sun4u +solaris-i86pc = py2.4-solaris-i86pc py2.5-solaris-i86pc py2.6-solaris-i86pc py2.7-solaris-i86pc +solaris-sun4u = py2.4-solaris-sun4u py2.5-solaris-sun4u py2.6-solaris-sun4u py2.7-solaris-sun4u solaris = solaris-i86pc solaris-sun4u py2.4-all = py2.4-linux py2.4-macosx py2.4-solaris py2.5-all = py2.5-linux py2.5-macosx py2.5-solaris py2.6-all = py2.6-linux py2.6-macosx py2.6-solaris +py2.7-all = py2.7-linux py2.7-macosx py2.7-solaris ; group for building pysam on solaris 10 sparc -solaris-2.10-sun4u = py2.4-solaris-2.10-sun4u_32-ucs2 py2.5-solaris-2.10-sun4u_32-ucs2 py2.6-solaris-2.10-sun4u_32-ucs2 py2.4-solaris-2.10-sun4u_64-ucs2 py2.5-solaris-2.10-sun4u_64-ucs2 py2.6-solaris-2.10-sun4u_64-ucs2 +;solaris-2.10-sun4u = py2.4-solaris-2.10-sun4u_32-ucs2 py2.5-solaris-2.10-sun4u_32-ucs2 py2.6-solaris-2.10-sun4u_32-ucs2 py2.4-solaris-2.10-sun4u_64-ucs2 py2.5-solaris-2.10-sun4u_64-ucs2 py2.6-solaris-2.10-sun4u_64-ucs2 ; the 'all' key is used internally by the build system to specify which hosts ; to build on when no hosts are specified on the dist-eggs.py command line. @@ -83,10 +102,10 @@ all = linux macosx solaris ; the 'noplatform' key, likewise, is for which build hosts should be used when ; building pure python (noplatform) eggs. -noplatform = py2.4 py2.5 py2.6 +noplatform = py2.4 py2.5 py2.6 py2.7 ; don't build these eggs on these platforms: [ignore] GeneTrack = py2.4 python-daemon = py2.4 -ctypes = py2.5-linux-i686-ucs2 py2.5-linux-i686-ucs4 py2.6-linux-i686-ucs2 py2.6-linux-i686-ucs4 py2.5-linux-x86_64-ucs2 py2.5-linux-x86_64-ucs4 py2.6-linux-x86_64-ucs2 py2.6-linux-x86_64-ucs4 py2.5-macosx-10.3-fat-ucs2 py2.6-macosx-10.3-fat-ucs2 py2.6-macosx-10.6-universal-ucs2 py2.5-solaris-2.10-i86pc_32-ucs2 py2.6-solaris-2.10-i86pc_32-ucs2 py2.5-solaris-2.10-i86pc_64-ucs2 py2.6-solaris-2.10-i86pc_64-ucs2 py2.5-solaris-2.10-sun4u_32-ucs2 py2.6-solaris-2.10-sun4u_32-ucs2 py2.5-solaris-2.10-sun4u_64-ucs2 py2.6-solaris-2.10-sun4u_64-ucs2 +ctypes = py2.5-linux-i686-ucs2 py2.5-linux-i686-ucs4 py2.6-linux-i686-ucs2 py2.6-linux-i686-ucs4 py2.7-linux-i686-ucs2 py2.7-linux-i686-ucs4 py2.5-linux-x86_64-ucs2 py2.5-linux-x86_64-ucs4 py2.6-linux-x86_64-ucs2 py2.6-linux-x86_64-ucs4 py2.7-linux-x86_64-ucs2 py2.7-linux-x86_64-ucs4 py2.5-macosx-10.3-fat-ucs2 py2.6-macosx-10.3-fat-ucs2 py2.6-macosx-10.6-universal-ucs2 py2.7-macosx-10.3-fat-ucs2 py2.5-solaris-2.10-i86pc_32-ucs2 py2.6-solaris-2.10-i86pc_32-ucs2 py2.7-solaris-2.10-i86pc_32-ucs2 py2.5-solaris-2.10-i86pc_64-ucs2 py2.6-solaris-2.10-i86pc_64-ucs2 py2.7-solaris-2.10-i86pc_64-ucs2 py2.5-solaris-2.10-sun4u_32-ucs2 py2.6-solaris-2.10-sun4u_32-ucs2 py2.7-solaris-2.10-sun4u_32-ucs2 py2.5-solaris-2.10-sun4u_64-ucs2 py2.6-solaris-2.10-sun4u_64-ucs2 py2.7-solaris-2.10-sun4u_64-ucs2 diff --git a/eggs.ini b/eggs.ini index 47428e9df85..7bec53d8b6f 100644 --- a/eggs.ini +++ b/eggs.ini @@ -3,7 +3,7 @@ ; ; This file is version controlled and should not be edited by hand! ; For more information, see: -; http://bitbucket.org/galaxy/galaxy-central/wiki/Config/Eggs +; http://wiki.g2.bx.psu.edu/Admin/Config/Eggs ; [general] @@ -18,7 +18,7 @@ ctypes = 1.0.2 DRMAA_python = 0.2 MarkupSafe = 0.12 MySQL_python = 1.2.3c1 -numpy = 1.3.0 +numpy = 1.6.0 pbs_python = 4.1.0 psycopg2 = 2.0.13 pycrypto = 2.0.1 @@ -38,7 +38,7 @@ drmaa = 0.4b3 elementtree = 1.2.6_20050316 GeneTrack = 2.0.0_beta_1 lrucache = 0.2 -Mako = 0.2.5 +Mako = 0.4.1 nose = 0.11.1 NoseHTML = 0.4.1 NoseTestDiff = 0.1 @@ -67,7 +67,7 @@ Whoosh = 0.3.18 psycopg2 = _8.4.2_static pysqlite = _3.6.17_static MySQL_python = _5.1.41_static -bx_python = _494c2d1d68b3 +bx_python = _494c2d1d68b3_rebuild1 GeneTrack = _dev_48da9e998f0caf01c5be731e926f4b0481f658f0 SQLAlchemy = _dev_r6498 pysam = _kanwei_b10f6e722e9a diff --git a/lib/galaxy/actions/__init__.py b/lib/galaxy/actions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/galaxy/actions/admin.py b/lib/galaxy/actions/admin.py new file mode 100644 index 00000000000..1bb21f5858c --- /dev/null +++ b/lib/galaxy/actions/admin.py @@ -0,0 +1,196 @@ +""" +Contains administrative functions +""" +import logging +from galaxy import util +from galaxy.exceptions import * + +log = logging.getLogger( __name__ ) + +class AdminActions( object ): + """ + Mixin for controllers that provide administrative functionality. + """ + def _create_quota( self, params ): + if params.amount.lower() in ( 'unlimited', 'none', 'no limit' ): + create_amount = None + else: + try: + create_amount = util.size_to_bytes( params.amount ) + except AssertionError: + create_amount = False + if not params.name or not params.description: + raise MessageException( "Enter a valid name and a description.", type='error' ) + elif self.sa_session.query( self.app.model.Quota ).filter( self.app.model.Quota.table.c.name==params.name ).first(): + raise MessageException( "Quota names must be unique and a quota with that name already exists, so choose another name.", type='error' ) + elif not params.get( 'amount', None ): + raise MessageException( "Enter a valid quota amount.", type='error' ) + elif create_amount is False: + raise MessageException( "Unable to parse the provided amount.", type='error' ) + elif params.operation not in self.app.model.Quota.valid_operations: + raise MessageException( "Enter a valid operation.", type='error' ) + elif params.default != 'no' and params.default not in self.app.model.DefaultQuotaAssociation.types.__dict__.values(): + raise MessageException( "Enter a valid default type.", type='error' ) + elif params.default != 'no' and params.operation != '=': + raise MessageException( "Operation for a default quota must be '='.", type='error' ) + elif create_amount is None and params.operation != '=': + raise MessageException( "Operation for an unlimited quota must be '='.", type='error' ) + else: + # Create the quota + quota = self.app.model.Quota( name=params.name, description=params.description, amount=create_amount, operation=params.operation ) + self.sa_session.add( quota ) + # If this is a default quota, create the DefaultQuotaAssociation + if params.default != 'no': + self.app.quota_agent.set_default_quota( params.default, quota ) + else: + # Create the UserQuotaAssociations + for user in [ self.sa_session.query( self.app.model.User ).get( x ) for x in params.in_users ]: + uqa = self.app.model.UserQuotaAssociation( user, quota ) + self.sa_session.add( uqa ) + # Create the GroupQuotaAssociations + for group in [ self.sa_session.query( self.app.model.Group ).get( x ) for x in params.in_groups ]: + gqa = self.app.model.GroupQuotaAssociation( group, quota ) + self.sa_session.add( gqa ) + self.sa_session.flush() + message = "Quota '%s' has been created with %d associated users and %d associated groups." % \ + ( quota.name, len( params.in_users ), len( params.in_groups ) ) + return quota, message + + def _rename_quota( self, quota, params ): + if not params.name: + raise MessageException( 'Enter a valid name', type='error' ) + elif params.name != quota.name and self.sa_session.query( self.app.model.Quota ).filter( self.app.model.Quota.table.c.name==params.name ).first(): + raise MessageException( 'A quota with that name already exists', type='error' ) + else: + old_name = quota.name + quota.name = params.name + quota.description = params.description + self.sa_session.add( quota ) + self.sa_session.flush() + message = "Quota '%s' has been renamed to '%s'" % ( old_name, params.name ) + return message + + def _manage_users_and_groups_for_quota( self, quota, params ): + if quota.default: + raise MessageException( 'Default quotas cannot be associated with specific users and groups', type='error' ) + else: + in_users = [ self.sa_session.query( self.app.model.User ).get( x ) for x in util.listify( params.in_users ) ] + in_groups = [ self.sa_session.query( self.app.model.Group ).get( x ) for x in util.listify( params.in_groups ) ] + self.app.quota_agent.set_entity_quota_associations( quotas=[ quota ], users=in_users, groups=in_groups ) + self.sa_session.refresh( quota ) + message = "Quota '%s' has been updated with %d associated users and %d associated groups" % ( quota.name, len( in_users ), len( in_groups ) ) + return message + + def _edit_quota( self, quota, params ): + if params.amount.lower() in ( 'unlimited', 'none', 'no limit' ): + new_amount = None + else: + try: + new_amount = util.size_to_bytes( params.amount ) + except AssertionError: + new_amount = False + if not params.amount: + raise MessageException( 'Enter a valid amount', type='error' ) + elif new_amount is False: + raise MessageException( 'Unable to parse the provided amount', type='error' ) + elif params.operation not in self.app.model.Quota.valid_operations: + raise MessageException( 'Enter a valid operation', type='error' ) + else: + quota.amount = new_amount + quota.operation = params.operation + self.sa_session.add( quota ) + self.sa_session.flush() + message = "Quota '%s' is now '%s'" % ( quota.name, quota.operation + quota.display_amount ) + return message + + def _set_quota_default( self, quota, params ): + if params.default != 'no' and params.default not in self.app.model.DefaultQuotaAssociation.types.__dict__.values(): + raise MessageException( 'Enter a valid default type.', type='error' ) + else: + if params.default != 'no': + self.app.quota_agent.set_default_quota( params.default, quota ) + message = "Quota '%s' is now the default for %s users" % ( quota.name, params.default ) + else: + if quota.default: + message = "Quota '%s' is no longer the default for %s users." % ( quota.name, quota.default[0].type ) + for dqa in quota.default: + self.sa_session.delete( dqa ) + self.sa_session.flush() + else: + message = "Quota '%s' is not a default." % quota.name + return message + + def _unset_quota_default( self, quota, params ): + if not quota.default: + raise MessageException( "Quota '%s' is not a default." % quota.name, type='error' ) + else: + message = "Quota '%s' is no longer the default for %s users." % ( quota.name, quota.default[0].type ) + for dqa in quota.default: + self.sa_session.delete( dqa ) + self.sa_session.flush() + return message + + def _mark_quota_deleted( self, quota, params ): + quotas = util.listify( quota ) + names = [] + for q in quotas: + if q.default: + names.append( q.name ) + if len( names ) == 1: + raise MessageException( "Quota '%s' is a default, please unset it as a default before deleting it" % ( names[0] ), type='error' ) + elif len( names ) > 1: + raise MessageException( "Quotas are defaults, please unset them as defaults before deleting them: " + ', '.join( names ), type='error' ) + message = "Deleted %d quotas: " % len( quotas ) + for q in quotas: + q.deleted = True + self.sa_session.add( q ) + names.append( q.name ) + self.sa_session.flush() + message += ', '.join( names ) + return message + + def _undelete_quota( self, quota, params ): + quotas = util.listify( quota ) + names = [] + for q in quotas: + if not q.deleted: + names.append( q.name ) + if len( names ) == 1: + raise MessageException( "Quota '%s' has not been deleted, so it cannot be undeleted." % ( names[0] ), type='error' ) + elif len( names ) > 1: + raise MessageException( "Quotas have not been deleted so they cannot be undeleted: " + ', '.join( names ), type='error' ) + message = "Undeleted %d quotas: " % len( quotas ) + for q in quotas: + q.deleted = False + self.sa_session.add( q ) + names.append( q.name ) + self.sa_session.flush() + message += ', '.join( names ) + return message + + def _purge_quota( self, quota, params ): + # This method should only be called for a Quota that has previously been deleted. + # Purging a deleted Quota deletes all of the following from the database: + # - UserQuotaAssociations where quota_id == Quota.id + # - GroupQuotaAssociations where quota_id == Quota.id + quotas = util.listify( quota ) + names = [] + for q in quotas: + if not q.deleted: + names.append( q.name ) + if len( names ) == 1: + raise MessageException( "Quota '%s' has not been deleted, so it cannot be purged." % ( names[0] ), type='error' ) + elif len( names ) > 1: + raise MessageException( "Quotas have not been deleted so they cannot be undeleted: " + ', '.join( names ), type='error' ) + message = "Purged %d quotas: " % len( quotas ) + for q in quotas: + # Delete UserQuotaAssociations + for uqa in q.users: + self.sa_session.delete( uqa ) + # Delete GroupQuotaAssociations + for gqa in q.groups: + self.sa_session.delete( gqa ) + names.append( q.name ) + self.sa_session.flush() + message += ', '.join( names ) + return message diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 86b01d4cd19..be16f355b1f 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -3,10 +3,12 @@ import sys, os, atexit from galaxy import config, jobs, util, tools, web import galaxy.tools.search import galaxy.tools.data +import galaxy.tools.tool_shed_registry from galaxy.web import security import galaxy.model import galaxy.datatypes.registry import galaxy.security +import galaxy.quota from galaxy.tags.tag_handler import GalaxyTagHandler from galaxy.tools.imp_exp import load_history_imp_exp_tools from galaxy.sample_tracking import external_service_types @@ -22,6 +24,11 @@ class UniverseApplication( object ): # Set up datatypes registry self.datatypes_registry = galaxy.datatypes.registry.Registry( self.config.root, self.config.datatypes_config ) galaxy.model.set_datatypes_registry( self.datatypes_registry ) + # Set up the tool sheds registry + if os.path.isfile( self.config.tool_sheds_config ): + self.tool_shed_registry = galaxy.tools.tool_shed_registry.Registry( self.config.root, self.config.tool_sheds_config ) + else: + self.tool_shed_registry = None # Determine the database url if self.config.database_connection: db_url = self.config.database_connection @@ -29,7 +36,7 @@ class UniverseApplication( object ): db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database # Initialize database / check for appropriate schema version from galaxy.model.migrate.check import create_or_verify_database - create_or_verify_database( db_url, self.config.database_engine_options ) + create_or_verify_database( db_url, kwargs.get( 'global_conf', {} ).get( '__file__', None ), self.config.database_engine_options ) # Setup the database engine and ORM from galaxy.model import mapping self.model = mapping.init( self.config.file_path, @@ -43,7 +50,7 @@ class UniverseApplication( object ): # Tool data tables self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( self.config.tool_data_table_config_path ) # Initialize the tools - self.toolbox = tools.ToolBox( self.config.tool_config, self.config.tool_path, self ) + self.toolbox = tools.ToolBox( self.config.tool_configs, self.config.tool_path, self ) # Search support for tools self.toolbox_search = galaxy.tools.search.ToolBoxSearch( self.toolbox ) # Load datatype converters @@ -57,6 +64,11 @@ class UniverseApplication( object ): #Load security policy self.security_agent = self.model.security_agent self.host_security_agent = galaxy.security.HostAgent( model=self.security_agent.model, permitted_actions=self.security_agent.permitted_actions ) + # Load quota management + if self.config.enable_quotas: + self.quota_agent = galaxy.quota.QuotaAgent( self.model ) + else: + self.quota_agent = galaxy.quota.NoQuotaAgent( self.model ) # Heartbeat and memdump for thread / heap profiling self.heartbeat = None self.memdump = None diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 091683ca770..22fa920d52d 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -5,7 +5,7 @@ Universe configuration builder. import sys, os, tempfile import logging, logging.config import ConfigParser -from galaxy.util import string_as_bool +from galaxy.util import string_as_bool, listify, parse_xml from galaxy import eggs import pkg_resources @@ -45,11 +45,13 @@ class Configuration( object ): # web API self.enable_api = string_as_bool( kwargs.get( 'enable_api', False ) ) self.enable_openid = string_as_bool( kwargs.get( 'enable_openid', False ) ) + self.enable_quotas = string_as_bool( kwargs.get( 'enable_quotas', False ) ) + self.tool_sheds_config = kwargs.get( 'tool_sheds_config_file', 'tool_sheds_conf.xml' ) self.tool_path = resolve_path( kwargs.get( "tool_path", "tools" ), self.root ) self.tool_data_path = resolve_path( kwargs.get( "tool_data_path", "tool-data" ), os.getcwd() ) self.len_file_path = kwargs.get( "len_file_path", resolve_path(os.path.join(self.tool_data_path, 'shared','ucsc','chrom'), self.root) ) self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root ) - self.tool_config = resolve_path( kwargs.get( 'tool_config_file', 'tool_conf.xml' ), self.root ) + self.tool_configs = [ resolve_path( p, self.root ) for p in listify( kwargs.get( 'tool_config_file', 'tool_conf.xml' ) ) ] self.tool_data_table_config_path = resolve_path( kwargs.get( 'tool_data_table_config_path', 'tool_data_table_conf.xml' ), self.root ) self.tool_secret = kwargs.get( "tool_secret", "" ) self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!" ) @@ -61,6 +63,7 @@ class Configuration( object ): self.require_login = string_as_bool( kwargs.get( "require_login", "False" ) ) self.allow_user_creation = string_as_bool( kwargs.get( "allow_user_creation", "True" ) ) self.allow_user_deletion = string_as_bool( kwargs.get( "allow_user_deletion", "False" ) ) + self.allow_user_dataset_purge = string_as_bool( kwargs.get( "allow_user_dataset_purge", "False" ) ) self.new_user_dataset_access_role_default_private = string_as_bool( kwargs.get( "new_user_dataset_access_role_default_private", "False" ) ) self.template_path = resolve_path( kwargs.get( "template_path", "templates" ), self.root ) self.template_cache = resolve_path( kwargs.get( "template_cache_path", "database/compiled_templates" ), self.root ) @@ -76,15 +79,19 @@ class Configuration( object ): self.mailing_join_addr = kwargs.get('mailing_join_addr',"galaxy-user-join@bx.psu.edu") self.error_email_to = kwargs.get( 'error_email_to', None ) self.smtp_server = kwargs.get( 'smtp_server', None ) + self.smtp_username = kwargs.get( 'smtp_username', None ) + self.smtp_password = kwargs.get( 'smtp_password', None ) self.start_job_runners = kwargs.get( 'start_job_runners', None ) # External Service types used in sample tracking self.external_service_type_config_file = resolve_path( kwargs.get( 'external_service_type_config_file', 'external_service_types_conf.xml' ), self.root ) self.external_service_type_path = resolve_path( kwargs.get( 'external_service_type_path', 'external_service_types' ), self.root ) # Tasked job runner. self.use_tasked_jobs = string_as_bool( kwargs.get( 'use_tasked_jobs', False ) ) + self.local_task_queue_workers = int(kwargs.get("local_task_queue_workers", 2)) # The transfer manager and deferred job queue self.enable_beta_job_managers = string_as_bool( kwargs.get( 'enable_beta_job_managers', 'False' ) ) - self.local_task_queue_workers = int(kwargs.get("local_task_queue_workers", 2)) + # Per-user Job concurrency limitations + self.user_job_limit = int( kwargs.get( 'user_job_limit', 0 ) ) self.default_cluster_job_runner = kwargs.get( 'default_cluster_job_runner', 'local:///' ) self.pbs_application_server = kwargs.get('pbs_application_server', "" ) self.pbs_dataset_server = kwargs.get('pbs_dataset_server', "" ) @@ -98,8 +105,8 @@ class Configuration( object ): self.gbrowse_display_sites = kwargs.get( 'gbrowse_display_sites', "wormbase,tair,modencode_worm,modencode_fly,sgd_yeast" ).lower().split(",") self.genetrack_display_sites = kwargs.get( 'genetrack_display_sites', "main,test" ).lower().split(",") self.brand = kwargs.get( 'brand', None ) + self.support_url = kwargs.get( 'support_url', 'http://wiki.g2.bx.psu.edu/Support' ) self.wiki_url = kwargs.get( 'wiki_url', 'http://g2.trac.bx.psu.edu/' ) - self.bugs_email = kwargs.get( 'bugs_email', None ) self.blog_url = kwargs.get( 'blog_url', None ) self.screencasts_url = kwargs.get( 'screencasts_url', None ) self.library_import_dir = kwargs.get( 'library_import_dir', None ) @@ -112,9 +119,12 @@ class Configuration( object ): self.ftp_upload_site = kwargs.get( 'ftp_upload_site', None ) self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False ) self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' ) - # Location for dependencies + # Location for tool dependencies. if 'tool_dependency_dir' in kwargs: self.tool_dependency_dir = resolve_path( kwargs.get( "tool_dependency_dir" ), self.root ) + # Setting the following flag to true will ultimately cause tool dependencies + # to be located in the shell environment and used by the job that is executing + # the tool. self.use_tool_dependencies = True else: self.tool_dependency_dir = None @@ -165,10 +175,21 @@ class Configuration( object ): else: return default def check( self ): + paths_to_check = [ self.root, self.tool_path, self.tool_data_path, self.template_path ] + # Look for any tool shed configs and retrieve the tool_path attribute from the tag. + for config_filename in self.tool_configs: + tree = parse_xml( config_filename ) + root = tree.getroot() + tool_path = root.get( 'tool_path' ) + if tool_path not in [ None, False ]: + paths_to_check.append( resolve_path( tool_path, self.root ) ) # Check that required directories exist - for path in self.root, self.tool_path, self.tool_data_path, self.template_path: - if not os.path.isdir( path ): - raise ConfigurationError("Directory does not exist: %s" % path ) + for path in paths_to_check: + if path not in [ None, False ] and not os.path.isdir( path ): + try: + os.makedirs( path ) + except Exception, e: + raise ConfigurationError( "Unable to create missing directory: %s\n%s" % ( path, e ) ) # Create the directories that it makes sense to create for path in self.file_path, \ self.new_file_path, \ @@ -188,9 +209,11 @@ class Configuration( object ): except Exception, e: raise ConfigurationError( "Unable to create missing directory: %s\n%s" % ( path, e ) ) # Check that required files exist - for path in self.tool_config, self.datatypes_config: + for path in self.tool_configs: if not os.path.isfile(path): raise ConfigurationError("File not found: %s" % path ) + if not os.path.isfile( self.datatypes_config ): + raise ConfigurationError("File not found: %s" % path ) # Check for deprecated options. for key in self.config_dict.keys(): if key in self.deprecated_options: diff --git a/lib/galaxy/datatypes/assembly.py b/lib/galaxy/datatypes/assembly.py index f4bbd5075a2..caa000f6aea 100644 --- a/lib/galaxy/datatypes/assembly.py +++ b/lib/galaxy/datatypes/assembly.py @@ -143,7 +143,6 @@ class Velvet( Html ): def __init__( self, **kwd ): Html.__init__( self, **kwd ) - log.debug( "Velvet log info %s" % 'JJ __init__') self.add_composite_file( 'Sequences', mimetype = 'text/html', description = 'Sequences', substitute_name_with_metadata = None, is_binary = False ) self.add_composite_file( 'Roadmaps', mimetype = 'text/html', description = 'Roadmaps', substitute_name_with_metadata = None, is_binary = False ) self.add_composite_file( 'Log', mimetype = 'text/html', description = 'Log', optional = 'True', substitute_name_with_metadata = None, is_binary = False ) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 02364fc156c..59c4213f46e 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -6,6 +6,10 @@ import data, logging, binascii from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes import metadata from galaxy.datatypes.sniff import * +from galaxy import eggs +import pkg_resources +pkg_resources.require( "bx-python" ) +from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_MAGIC_SIZE from urllib import urlencode, quote_plus import zipfile, gzip import os, subprocess, tempfile @@ -14,7 +18,7 @@ import struct log = logging.getLogger(__name__) # Currently these supported binary data types must be manually set on upload -unsniffable_binary_formats = [ 'ab1', 'scf' ] +unsniffable_binary_formats = [ 'ab1', 'scf', 'h5' ] class Binary( data.Data ): """Binary data""" @@ -50,7 +54,7 @@ class Ab1( Binary ): class Bam( Binary ): """Class describing a BAM binary file""" file_ext = "bam" - MetadataElement( name="bam_index", desc="BAM Index File", param=metadata.FileParameter, readonly=True, no_value=None, visible=False, optional=True ) + MetadataElement( name="bam_index", desc="BAM Index File", param=metadata.FileParameter, file_ext="bai", readonly=True, no_value=None, visible=False, optional=True ) def _get_samtools_version( self ): # Determine the version of samtools being used. Wouldn't it be nice if @@ -202,7 +206,24 @@ class Bam( Binary ): return "Binary bam alignments file (%s)" % ( data.nice_size( dataset.get_size() ) ) def get_track_type( self ): return "ReadTrack", {"data": "bai", "index": "summary_tree"} - + +class H5( Binary ): + """Class describing an HDF5 file""" + file_ext = "h5" + + def set_peek( self, dataset, is_multi_byte=False ): + if not dataset.dataset.purged: + dataset.peek = "Binary h5 file" + dataset.blurb = data.nice_size( dataset.get_size() ) + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' + def display_peek( self, dataset ): + try: + return dataset.peek + except: + return "Binary h5 sequence file (%s)" % ( data.nice_size( dataset.get_size() ) ) + class Scf( Binary ): """Class describing an scf binary sequence file""" file_ext = "scf" @@ -288,7 +309,30 @@ class BigBed(BigWig): Binary.__init__( self, **kwd ) self._magic = 0x8789F2EB self._name = "BigBed" - def get_track_type( self ): return "LineTrack", {"data_standalone": "bigbed"} +class TwoBit (Binary): + """Class describing a TwoBit format nucleotide file""" + + file_ext = "twobit" + + def sniff(self, filename): + try: + input = file(filename) + magic = struct.unpack(">L", input.read(TWOBIT_MAGIC_SIZE))[0] + if magic == TWOBIT_MAGIC_NUMBER or magic == TWOBIT_MAGIC_NUMBER_SWAP: + return True + except IOError: + return False + def set_peek(self, dataset, is_multi_byte=False): + if not dataset.dataset.purged: + dataset.peek = "Binary TwoBit format nucleotide file" + dataset.blurb = data.nice_size(dataset.get_size()) + else: + return super(TwoBit, self).set_peek(dataset, is_multi_byte) + def display_peek(self, dataset): + try: + return dataset.peek + except: + return "Binary TwoBit format nucleotide file (%s)" % (data.nice_size(dataset.get_size())) diff --git a/lib/galaxy/datatypes/checkers.py b/lib/galaxy/datatypes/checkers.py new file mode 100644 index 00000000000..4561631a777 --- /dev/null +++ b/lib/galaxy/datatypes/checkers.py @@ -0,0 +1,130 @@ +import os, gzip, re, gzip, zipfile, binascii, bz2, imghdr +from galaxy import util + +try: + import Image as PIL +except ImportError: + try: + from PIL import Image as PIL + except: + PIL = None + +def check_image( file_path ): + if PIL != None: + try: + im = PIL.open( file_path ) + except: + return False + if im: + return im + return False + else: + if imghdr.what( file_path ) != None: + return True + return False + +def check_html( file_path, chunk=None ): + if chunk is None: + temp = open( file_path, "U" ) + else: + temp = chunk + regexp1 = re.compile( "]*HREF[^>]+>", re.I ) + regexp2 = re.compile( "]*>", re.I ) + regexp3 = re.compile( "]*>", re.I ) + regexp4 = re.compile( "]*>", re.I ) + regexp5 = re.compile( "]*>", re.I ) + lineno = 0 + for line in temp: + lineno += 1 + matches = regexp1.search( line ) or regexp2.search( line ) or regexp3.search( line ) or regexp4.search( line ) or regexp5.search( line ) + if matches: + if chunk is None: + temp.close() + return True + if lineno > 100: + break + if chunk is None: + temp.close() + return False + +def check_binary( name, file_path=True ): + # Handles files if file_path is True or text if file_path is False + is_binary = False + if file_path: + temp = open( name, "U" ) + else: + temp = name + chars_read = 0 + for chars in temp: + for char in chars: + chars_read += 1 + if ord( char ) > 128: + is_binary = True + break + if chars_read > 100: + break + if chars_read > 100: + break + if file_path: + temp.close() + return is_binary + +def check_gzip( file_path ): + # This method returns a tuple of booleans representing ( is_gzipped, is_valid ) + # Make sure we have a gzipped file + try: + temp = open( file_path, "U" ) + magic_check = temp.read( 2 ) + temp.close() + if magic_check != util.gzip_magic: + return ( False, False ) + except: + return ( False, False ) + # We support some binary data types, so check if the compressed binary file is valid + # If the file is Bam, it should already have been detected as such, so we'll just check + # for sff format. + try: + header = gzip.open( file_path ).read(4) + if binascii.b2a_hex( header ) == binascii.hexlify( '.sff' ): + return ( True, True ) + except: + return( False, False ) + CHUNK_SIZE = 2**15 # 32Kb + gzipped_file = gzip.GzipFile( file_path, mode='rb' ) + chunk = gzipped_file.read( CHUNK_SIZE ) + gzipped_file.close() + # See if we have a compressed HTML file + if check_html( file_path, chunk=chunk ): + return ( True, False ) + return ( True, True ) + +def check_bz2( file_path ): + try: + temp = open( file_path, "U" ) + magic_check = temp.read( 3 ) + temp.close() + if magic_check != util.bz2_magic: + return ( False, False ) + except: + return( False, False ) + CHUNK_SIZE = 2**15 # reKb + bzipped_file = bz2.BZ2File( file_path, mode='rb' ) + chunk = bzipped_file.read( CHUNK_SIZE ) + bzipped_file.close() + # See if we have a compressed HTML file + if check_html( file_path, chunk=chunk ): + return ( True, False ) + return ( True, True ) + +def check_zip( file_path ): + if zipfile.is_zipfile( file_path ): + return True + return False + +def is_bz2( file_path ): + is_bz2, is_valid = check_bz2( file_path ) + return is_bz2 + +def is_gzip( file_path ): + is_gzipped, is_valid = check_gzip( file_path ) + return is_gzipped diff --git a/lib/galaxy/datatypes/converters/__init__.py b/lib/galaxy/datatypes/converters/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/galaxy/datatypes/converters/fasta_to_2bit.xml b/lib/galaxy/datatypes/converters/fasta_to_2bit.xml new file mode 100644 index 00000000000..b4890808a13 --- /dev/null +++ b/lib/galaxy/datatypes/converters/fasta_to_2bit.xml @@ -0,0 +1,13 @@ + + + + faToTwoBit $input $output + + + + + + + + + diff --git a/lib/galaxy/datatypes/converters/fasta_to_len.py b/lib/galaxy/datatypes/converters/fasta_to_len.py new file mode 100644 index 00000000000..26f0856bd4e --- /dev/null +++ b/lib/galaxy/datatypes/converters/fasta_to_len.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +""" +Input: fasta, int +Output: tabular +Return titles with lengths of corresponding seq +""" + +import sys, os + +assert sys.version_info[:2] >= ( 2, 4 ) + +def compute_fasta_length( fasta_file, out_file, keep_first_char ): + + infile = fasta_file + out = open( out_file, 'w') + keep_first_char = int( keep_first_char ) + + fasta_title = '' + seq_len = 0 + + # number of char to keep in the title + if keep_first_char == 0: + keep_first_char = None + else: + keep_first_char += 1 + + first_entry = True + + for line in open( infile ): + line = line.strip() + if not line or line.startswith( '#' ): + continue + if line[0] == '>': + if first_entry == False: + out.write( "%s\t%d\n" % ( fasta_title[ 1:keep_first_char ], seq_len ) ) + else: + first_entry = False + fasta_title = line + seq_len = 0 + else: + seq_len += len(line) + + # last fasta-entry + out.write( "%s\t%d\n" % ( fasta_title[ 1:keep_first_char ], seq_len ) ) + out.close() + +if __name__ == "__main__" : + compute_fasta_length( sys.argv[1], sys.argv[2], sys.argv[3] ) \ No newline at end of file diff --git a/lib/galaxy/datatypes/converters/fasta_to_len.xml b/lib/galaxy/datatypes/converters/fasta_to_len.xml new file mode 100644 index 00000000000..8e1a1a0b966 --- /dev/null +++ b/lib/galaxy/datatypes/converters/fasta_to_len.xml @@ -0,0 +1,13 @@ + + + + fasta_to_len.py $input $output 0 + + + + + + + + + diff --git a/lib/galaxy/datatypes/converters/len_to_linecount.xml b/lib/galaxy/datatypes/converters/len_to_linecount.xml new file mode 100644 index 00000000000..9a91d8723b8 --- /dev/null +++ b/lib/galaxy/datatypes/converters/len_to_linecount.xml @@ -0,0 +1,13 @@ + + + + wc -l $input | awk '{print $1}' > $output + + + + + + + + + diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 77983cc59dc..b26670c41a0 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -430,13 +430,13 @@ class Text( Data ): if line and not line.startswith( '#' ): data_lines += 1 return data_lines - def set_peek( self, dataset, line_count=None, is_multi_byte=False ): + def set_peek( self, dataset, line_count=None, is_multi_byte=False, skipchars=[] ): """ Set the peek. This method is used by various subclasses of Text. """ if not dataset.dataset.purged: # The file must exist on disk for the get_file_peek() method - dataset.peek = get_file_peek( dataset.file_name, is_multi_byte=is_multi_byte ) + dataset.peek = get_file_peek( dataset.file_name, is_multi_byte=is_multi_byte, skipchars=skipchars ) if line_count is None: # See if line_count is stored in the metadata if dataset.metadata.data_lines: @@ -548,7 +548,7 @@ def get_test_fname( fname ): path, name = os.path.split(__file__) full_path = os.path.join( path, 'test', fname ) return full_path -def get_file_peek( file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5 ): +def get_file_peek( file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skipchars=[] ): """ Returns the first LINE_COUNT lines wrapped to WIDTH @@ -576,8 +576,14 @@ def get_file_peek( file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5 ): data_checked = True if file_type in [ 'gzipped', 'binary' ]: break - lines.append( line ) - count += 1 + skip_line = False + for skipchar in skipchars: + if line.startswith( skipchar ): + skip_line = True + break + if not skip_line: + lines.append( line ) + count += 1 temp.close() if file_type in [ 'gzipped', 'binary' ]: text = "%s file" % file_type diff --git a/lib/galaxy/datatypes/display_applications/parameters.py b/lib/galaxy/datatypes/display_applications/parameters.py index cd8a2f5d0c0..a22af845bcf 100644 --- a/lib/galaxy/datatypes/display_applications/parameters.py +++ b/lib/galaxy/datatypes/display_applications/parameters.py @@ -92,6 +92,8 @@ class DisplayApplicationDataParameter( DisplayApplicationParameter ): #find target ext target_ext, converted_dataset = data.find_conversion_destination( self.formats, converter_safe = True ) if target_ext and not converted_dataset: + if isinstance( data, DisplayDataValueWrapper ): + data = data.value assoc = trans.app.model.ImplicitlyConvertedDatasetAssociation( parent = data, file_type = target_ext, metadata_safe = False ) new_data = data.datatype.convert_dataset( trans, data, target_ext, return_output = True, visible = False ).values()[0] new_data.hid = data.hid diff --git a/lib/galaxy/datatypes/genetics.py b/lib/galaxy/datatypes/genetics.py index 5f9d8b4f712..48fafada24a 100644 --- a/lib/galaxy/datatypes/genetics.py +++ b/lib/galaxy/datatypes/genetics.py @@ -53,9 +53,9 @@ class GenomeGraphs( Tabular ): header = file(dataset.file_name,'r').readlines()[0].strip().split('\t') dataset.metadata.columns = len(header) t = ['numeric' for x in header] - t[0] = 'string' + t[0] = 'string' dataset.metadata.column_types = t - return True + return True def as_ucsc_display_file( self, dataset, **kwd ): """ @@ -113,8 +113,8 @@ class GenomeGraphs( Tabular ): f = open(dataset.file_name,'r') d = f.readlines()[:5] if len(d) == 0: - out = "Cannot find anything to parse in %s" % dataset.name - return out + out = "Cannot find anything to parse in %s" % dataset.name + return out hasheader = 0 try: test = ['%f' % x for x in d[0][1:]] # first is name - see if starts all numerics @@ -123,7 +123,7 @@ class GenomeGraphs( Tabular ): try: # Generate column header out.append( '' ) - if hasheader: + if hasheader: for i, name in enumerate(d[0].split() ): out.append( '%s.%s' % ( str( i+1 ), name ) ) d.pop(0) @@ -285,9 +285,9 @@ class Rgenetics(Html): if composite_file.optional: opt_text = ' (optional)' if composite_file.get('description'): - rval.append( '
  • %s (%s)%s
    • ' % ( fn, fn, composite_file.get('description'), opt_text ) ) + rval.append( '
  • %s (%s)%s
    • ' % ( fn, fn, composite_file.get('description'), opt_text ) ) else: - rval.append( '
  • %s%s
    • ' % ( fn, fn, opt_text ) ) + rval.append( '
  • %s%s
    • ' % ( fn, fn, opt_text ) ) rval.append( '' ) return "\n".join( rval ) @@ -641,9 +641,9 @@ class RexpBase( Html ): if not dataset.dataset.purged: pp = os.path.join(dataset.extra_files_path,'%s.pheno' % dataset.metadata.base_name) try: - p = file(pp,'r').readlines() + p = file(pp,'r').readlines() except: - p = ['##failed to find %s' % pp,] + p = ['##failed to find %s' % pp,] dataset.peek = ''.join(p[:5]) dataset.blurb = 'Galaxy Rexpression composite file' else: diff --git a/lib/galaxy/datatypes/images.py b/lib/galaxy/datatypes/images.py index f0446088a80..d20321d9421 100644 --- a/lib/galaxy/datatypes/images.py +++ b/lib/galaxy/datatypes/images.py @@ -7,12 +7,31 @@ import logging from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes import metadata from galaxy.datatypes.sniff import * +from galaxy.datatypes.util.image_util import * from urllib import urlencode, quote_plus import zipfile -import os, subprocess, tempfile +import os, subprocess, tempfile, imghdr + +try: + import Image as PIL +except ImportError: + try: + from PIL import Image as PIL + except: + PIL = None log = logging.getLogger(__name__) +# TODO: Uploading image files of various types is supported in Galaxy, but on +# the main public instance, the display_in_upload is not set for these data +# types in datatypes_conf.xml because we do not allow image files to be uploaded +# there. There is currently no API feature that allows uploading files outside +# of a data library ( where it requires either the upload_paths or upload_directory +# option to be enabled, which is not the case on the main public instance ). Because +# of this, we're currently safe, but when the api is enhanced to allow other uploads, +# we need to ensure that the implementation is such that image files cannot be uploaded +# to our main public instance. + class Image( data.Data ): """Class describing an image""" def set_peek( self, dataset, is_multi_byte=False ): @@ -22,11 +41,110 @@ class Image( data.Data ): else: dataset.peek = 'file does not exist' dataset.blurb = 'file purged from disk' + def sniff( self, filename ): + # First check if we can use PIL + if PIL is not None: + try: + im = PIL.open( filename ) + im.close() + return True + except: + return False + else: + if imghdr.what( filename ) is not None: + return True + else: + return False + +class Jpg( Image ): + def sniff(self, filename, image=None): + """Determine if the file is in jpg format.""" + return check_image_type( filename, ['JPEG'], image ) + +class Png( Image ): + def sniff(self, filename, image=None): + """Determine if the file is in png format.""" + return check_image_type( filename, ['PNG'], image ) + +class Tiff( Image ): + def sniff(self, filename, image=None): + """Determine if the file is in tiff format.""" + return check_image_type( filename, ['TIFF'], image ) + +class Bmp( Image ): + def sniff(self, filename, image=None): + """Determine if the file is in bmp format.""" + return check_image_type( filename, ['BMP'], image ) + +class Gif( Image ): + def sniff(self, filename, image=None): + """Determine if the file is in gif format.""" + return check_image_type( filename, ['GIF'], image ) + +class Im( Image ): + def sniff(self, filename, image=None): + """Determine if the file is in im format.""" + return check_image_type( filename, ['IM'], image ) + +class Pcd( Image ): + def sniff(self, filename, image=None): + """Determine if the file is in pcd format.""" + return check_image_type( filename, ['PCD'], image ) + +class Pcx( Image ): + def sniff(self, filename, image=None): + """Determine if the file is in pcx format.""" + return check_image_type( filename, ['PCX'], image ) + +class Ppm( Image ): + def sniff(self, filename, image=None): + """Determine if the file is in ppm format.""" + return check_image_type( filename, ['PPM'], image ) + +class Psd( Image ): + def sniff(self, filename, image=None): + """Determine if the file is in psd format.""" + return check_image_type( filename, ['PSD'], image ) + +class Xbm( Image ): + def sniff(self, filename, image=None): + """Determine if the file is in XBM format.""" + return check_image_type( filename, ['XBM'], image ) + +class Xpm( Image ): + def sniff(self, filename, image=None): + """Determine if the file is in XPM format.""" + return check_image_type( filename, ['XPM'], image ) + +class Rgb( Image ): + def sniff(self, filename, image=None): + """Determine if the file is in RGB format.""" + return check_image_type( filename, ['RGB'], image ) + +class Pbm( Image ): + def sniff(self, filename, image=None): + """Determine if the file is in PBM format""" + return check_image_type( filename, ['PBM'], image ) + +class Pgm( Image ): + def sniff(self, filename, image=None): + """Determine if the file is in PGM format""" + return check_image_type( filename, ['PGM'], image ) + +class Eps( Image ): + def sniff(self, filename, image=None): + """Determine if the file is in eps format.""" + return check_image_type( filename, ['EPS'], image ) + + +class Rast( Image ): + def sniff(self, filename, image=None): + """Determine if the file is in rast format""" + return check_image_type( filename, ['RAST'], image ) class Pdf( Image ): def sniff(self, filename): - """Determine if the file is in pdf format. - """ + """Determine if the file is in pdf format.""" headers = get_headers(filename, None, 1) try: if headers[0][0].startswith("%PDF"): @@ -73,7 +191,7 @@ class Gmaj( data.Data ): "nobutton": "false", "urlpause" :"100", "debug": "false", - "posturl": quote_plus( "history_add_to?%s" % "&".join( [ "%s=%s" % ( key, value ) for key, value in { 'history_id': dataset.history_id, 'ext': 'maf', 'name': 'GMAJ Output on data %s' % dataset.hid, 'info': 'Added by GMAJ', 'dbkey': dataset.dbkey, 'copy_access_from': dataset.id }.items() ] ) ) + "posturl": "history_add_to?%s" % "&".join( map( lambda x: "%s=%s" % ( x[0], quote_plus( str( x[1] ) ) ), [ ( 'copy_access_from', dataset.id), ( 'history_id', dataset.history_id ), ( 'ext', 'maf' ), ( 'name', 'GMAJ Output on data %s' % dataset.hid ), ( 'info', 'Added by GMAJ' ), ( 'dbkey', dataset.dbkey ) ] ) ) } class_name = "edu.psu.bx.gmaj.MajApplet.class" archive = "/static/gmaj/gmaj.jar" diff --git a/lib/galaxy/datatypes/interval.py b/lib/galaxy/datatypes/interval.py index a64062bec68..0c7693e6c5b 100644 --- a/lib/galaxy/datatypes/interval.py +++ b/lib/galaxy/datatypes/interval.py @@ -939,23 +939,11 @@ class Gtf( Gff ): return False # Check attributes for gene_id, transcript_id - attributes = hdr[8].split(";") + attributes = parse_gff_attributes( hdr[8] ) if len( attributes ) >= 2: - try: - # Imprecise: should check for a single space per the spec. - # strip() needed b/c Ensembl GTF files include an (illegal) - # space before attributes string. - attr_name, attr_value = attributes[0].strip().split(" ") - if attr_name != 'gene_id': - return False - except: + if 'gene_id' not in attributes: return False - try: - # Imprecise: should check for a single space per the spec. - attr_name, attr_value = attributes[1][1:].split(" ") - if attr_name != 'transcript_id': - return False - except: + if 'transcript_id' not in attributes: return False else: return False diff --git a/lib/galaxy/datatypes/metadata.py b/lib/galaxy/datatypes/metadata.py index 443bfa88a2f..174aae0c5c4 100644 --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -228,8 +228,8 @@ class MetadataElementSpec( object ): #set up param last, as it uses values set above self.param = param( self ) datatype.metadata_spec.append( self ) #add spec element to the spec - def get( self, name ): - return self.__dict__.get(name, None) + def get( self, name, default=None ): + return self.__dict__.get(name, default) def wrap( self, value ): """ Turns a stored value into its usable form. diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 7e7b18bc858..9d3f8d02c6f 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -122,32 +122,22 @@ class Registry( object ): if current_app is None and isinstance( d_type1, type( d_type2 ) ): d_type1.add_display_application( display_app ) # Load datatype sniffers from the config - sniff_order = [] sniffers = root.find( 'sniffers' ) for elem in sniffers.findall( 'sniffer' ): dtype = elem.get( 'type', None ) if dtype: - sniff_order.append( dtype ) - for dtype in sniff_order: - try: - fields = dtype.split( ":" ) - datatype_module = fields[0] - datatype_class = fields[1] - fields = datatype_module.split( "." ) - module = __import__( fields.pop(0) ) - for mod in fields: - module = getattr( module, mod ) - aclass = getattr( module, datatype_class )() - included = False - for atype in self.sniff_order: - if not issubclass( atype.__class__, aclass.__class__ ) and isinstance( atype, aclass.__class__ ): - included = True - break - if not included: + try: + fields = dtype.split( ":" ) + datatype_module = fields[0] + datatype_class = fields[1] + module = __import__( datatype_module ) + for comp in datatype_module.split('.')[1:]: + module = getattr(module, comp) + aclass = getattr( module, datatype_class )() self.sniff_order.append( aclass ) self.log.debug( 'Loaded sniffer for datatype: %s' % dtype ) - except Exception, exc: - self.log.warning( 'Error appending datatype %s to sniff_order, problem: %s' % ( dtype, str( exc ) ) ) + except Exception, exc: + self.log.warning( 'Error appending datatype %s to sniff_order, problem: %s' % ( dtype, str( exc ) ) ) #default values if len(self.datatypes_by_extension) < 1: self.datatypes_by_extension = { @@ -180,14 +170,15 @@ class Registry( object ): 'tabular' : tabular.Tabular(), 'taxonomy' : tabular.Taxonomy(), 'txt' : data.Text(), - 'wig' : interval.Wiggle() + 'wig' : interval.Wiggle(), + 'xml' : xml.GenericXml(), } self.mimetypes_by_extension = { 'ab1' : 'application/octet-stream', 'axt' : 'text/plain', 'bam' : 'application/octet-stream', 'bed' : 'text/plain', - 'blastxml' : 'text/plain', + 'blastxml' : 'application/xml', 'customtrack' : 'text/plain', 'csfasta' : 'text/plain', 'fasta' : 'text/plain', @@ -200,6 +191,7 @@ class Registry( object ): 'laj' : 'text/plain', 'lav' : 'text/plain', 'maf' : 'text/plain', + 'memexml' : 'application/xml', 'pileup' : 'text/plain', 'qualsolid' : 'text/plain', 'qualsolexa' : 'text/plain', @@ -210,7 +202,8 @@ class Registry( object ): 'tabular' : 'text/plain', 'taxonomy' : 'text/plain', 'txt' : 'text/plain', - 'wig' : 'text/plain' + 'wig' : 'text/plain', + 'xml' : 'application/xml', } # super supertype fix for input steps in workflows. if 'data' not in self.datatypes_by_extension: @@ -223,6 +216,7 @@ class Registry( object ): binary.Bam(), binary.Sff(), xml.BlastXml(), + xml.GenericXml(), sequence.Maf(), sequence.Lav(), sequence.csFasta(), @@ -387,9 +381,9 @@ class Registry( object ): """Returns ( target_ext, existing converted dataset )""" for convert_ext in self.get_converters_by_datatype( dataset.ext ): if isinstance( self.get_datatype_by_extension( convert_ext ), accepted_formats ): - dataset = dataset.get_converted_files_by_type( convert_ext ) - if dataset: - ret_data = dataset + converted_dataset = dataset.get_converted_files_by_type( convert_ext ) + if converted_dataset: + ret_data = converted_dataset elif not converter_safe: continue else: diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 5b697376dd0..379c3713add 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -4,6 +4,7 @@ File format detector import logging, sys, os, csv, tempfile, shutil, re, zipfile, gzip import registry from galaxy import util +from galaxy.datatypes.checkers import * from galaxy.datatypes.binary import unsniffable_binary_formats log = logging.getLogger(__name__) @@ -319,59 +320,6 @@ def guess_ext( fname, sniff_order=None, is_multi_byte=False ): return 'tabular' #default tabular data type file extension return 'txt' #default text data type file extension - -#Methods Used below can be used to upload new datasets into Galaxy. Currently used by the data_source.py script/tools. -#These should be further abstracted and merged with upload.py script/tool functionality. -def is_gzip( filename ): - temp = open( filename, "U" ) - magic_check = temp.read( 2 ) - temp.close() - if magic_check != util.gzip_magic: - return False - return True - - -def is_binary( filename ): - is_binary = False - temp = open( filename, "U" ) - chars_read = 0 - for chars in temp: - for char in chars: - chars_read += 1 - if ord( char ) > 128: - is_binary = True - break - if chars_read > 100: - break - if chars_read > 100: - break - temp.close() - return is_binary - -def is_html( temp_name, chunk=None ): - if chunk is None: - temp = open(temp_name, "U") - else: - temp = chunk - regexp1 = re.compile( "]*HREF[^>]+>", re.I ) - regexp2 = re.compile( "]*>", re.I ) - regexp3 = re.compile( "]*>", re.I ) - regexp4 = re.compile( "]*>", re.I ) - regexp5 = re.compile( "]*>", re.I ) - lineno = 0 - for line in temp: - lineno += 1 - matches = regexp1.search( line ) or regexp2.search( line ) or regexp3.search( line ) or regexp4.search( line ) or regexp5.search( line ) - if matches: - if chunk is None: - temp.close() - return True - if lineno > 100: - break - if chunk is None: - temp.close() - return False - def handle_compressed_file( filename, datatypes_registry, ext = 'auto' ): CHUNK_SIZE = 2**20 # 1Mb is_compressed = False @@ -429,10 +377,10 @@ def handle_uploaded_dataset_file( filename, datatypes_registry, ext = 'auto', is if ext in AUTO_DETECT_EXTENSIONS: ext = guess_ext( filename, sniff_order = datatypes_registry.sniff_order, is_multi_byte=is_multi_byte ) - if is_binary( filename ): + if check_binary( filename ): if ext not in unsniffable_binary_formats and not datatypes_registry.get_datatype_by_extension( ext ).sniff( filename ): raise InappropriateDatasetContentError, 'The binary uploaded file contains inappropriate content.' - elif is_html( filename ): + elif check_html( filename ): raise InappropriateDatasetContentError, 'The uploaded file contains inappropriate HTML content.' return ext @@ -449,4 +397,3 @@ class InappropriateDatasetContentError( Exception ): if __name__ == '__main__': import doctest, sys doctest.testmod(sys.modules[__name__]) - diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index dc0a562b521..c35ad2e7ac3 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -479,13 +479,8 @@ class Vcf( Tabular ): MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[5], param=metadata.ColumnParameter, multiple=True ) def sniff( self, filename ): - try: - # If reader can read and parse file, it's VCF. - for line in list( galaxy_utils.sequence.vcf.Reader( open( filename ) ) ): - pass - return True - except: - return False + headers = get_headers( filename, '\n', count=1 ) + return headers[0][0].startswith("##fileformat=VCF") def make_html_table( self, dataset, skipchars=[] ): """Create HTML table, used for displaying peek""" diff --git a/lib/galaxy/datatypes/test/tblastn_four_human_vs_rhodopsin.xml b/lib/galaxy/datatypes/test/tblastn_four_human_vs_rhodopsin.xml new file mode 100644 index 00000000000..9bccf112f2d --- /dev/null +++ b/lib/galaxy/datatypes/test/tblastn_four_human_vs_rhodopsin.xml @@ -0,0 +1,722 @@ + + + + tblastn + TBLASTN 2.2.25+ + Stephen F. Altschul, Thomas L. Madden, Alejandro A. Sch&auml;ffer, Jinghui Zhang, Zheng Zhang, Webb Miller, and David J. Lipman (1997), "Gapped BLAST and PSI-BLAST: a new generation of protein database search programs", Nucleic Acids Res. 25:3389-3402. + + Query_1 + sp|Q9BS26|ERP44_HUMAN Endoplasmic reticulum resident protein 44 OS=Homo sapiens GN=ERP44 PE=1 SV=1 + 406 + + + BLOSUM80 + 1e-10 + 10 + 1 + F + + + + + 1 + Query_1 + sp|Q9BS26|ERP44_HUMAN Endoplasmic reticulum resident protein 44 OS=Homo sapiens GN=ERP44 PE=1 SV=1 + 406 + + + + 0 + 0 + 19 + 127710 + 0.071 + 0.299 + 0.27 + + + No hits found + + + 2 + Query_1 + sp|Q9BS26|ERP44_HUMAN Endoplasmic reticulum resident protein 44 OS=Homo sapiens GN=ERP44 PE=1 SV=1 + 406 + + + + 0 + 0 + 19 + 127710 + 0.071 + 0.299 + 0.27 + + + No hits found + + + 3 + Query_1 + sp|Q9BS26|ERP44_HUMAN Endoplasmic reticulum resident protein 44 OS=Homo sapiens GN=ERP44 PE=1 SV=1 + 406 + + + + 0 + 0 + 19 + 127710 + 0.071 + 0.299 + 0.27 + + + No hits found + + + 4 + Query_1 + sp|Q9BS26|ERP44_HUMAN Endoplasmic reticulum resident protein 44 OS=Homo sapiens GN=ERP44 PE=1 SV=1 + 406 + + + + 0 + 0 + 19 + 127710 + 0.071 + 0.299 + 0.27 + + + No hits found + + + 5 + Query_1 + sp|Q9BS26|ERP44_HUMAN Endoplasmic reticulum resident protein 44 OS=Homo sapiens GN=ERP44 PE=1 SV=1 + 406 + + + + 0 + 0 + 19 + 127710 + 0.071 + 0.299 + 0.27 + + + No hits found + + + 6 + Query_1 + sp|Q9BS26|ERP44_HUMAN Endoplasmic reticulum resident protein 44 OS=Homo sapiens GN=ERP44 PE=1 SV=1 + 406 + + + + 0 + 0 + 19 + 127710 + 0.071 + 0.299 + 0.27 + + + No hits found + + + 7 + Query_2 + sp|Q9NSY1|BMP2K_HUMAN BMP-2-inducible protein kinase OS=Homo sapiens GN=BMP2K PE=1 SV=2 + 1161 + + + + 0 + 0 + 23 + 370988 + 0.071 + 0.299 + 0.27 + + + No hits found + + + 8 + Query_2 + sp|Q9NSY1|BMP2K_HUMAN BMP-2-inducible protein kinase OS=Homo sapiens GN=BMP2K PE=1 SV=2 + 1161 + + + + 0 + 0 + 23 + 370988 + 0.071 + 0.299 + 0.27 + + + No hits found + + + 9 + Query_2 + sp|Q9NSY1|BMP2K_HUMAN BMP-2-inducible protein kinase OS=Homo sapiens GN=BMP2K PE=1 SV=2 + 1161 + + + + 0 + 0 + 23 + 370988 + 0.071 + 0.299 + 0.27 + + + No hits found + + + 10 + Query_2 + sp|Q9NSY1|BMP2K_HUMAN BMP-2-inducible protein kinase OS=Homo sapiens GN=BMP2K PE=1 SV=2 + 1161 + + + + 0 + 0 + 23 + 370988 + 0.071 + 0.299 + 0.27 + + + No hits found + + + 11 + Query_2 + sp|Q9NSY1|BMP2K_HUMAN BMP-2-inducible protein kinase OS=Homo sapiens GN=BMP2K PE=1 SV=2 + 1161 + + + + 0 + 0 + 23 + 370988 + 0.071 + 0.299 + 0.27 + + + No hits found + + + 12 + Query_2 + sp|Q9NSY1|BMP2K_HUMAN BMP-2-inducible protein kinase OS=Homo sapiens GN=BMP2K PE=1 SV=2 + 1161 + + + + 0 + 0 + 23 + 370988 + 0.071 + 0.299 + 0.27 + + + No hits found + + + 13 + Query_3 + sp|P06213|INSR_HUMAN Insulin receptor OS=Homo sapiens GN=INSR PE=1 SV=4 + 1382 + + + + 0 + 0 + 24 + 441350 + 0.071 + 0.299 + 0.27 + + + No hits found + + + 14 + Query_3 + sp|P06213|INSR_HUMAN Insulin receptor OS=Homo sapiens GN=INSR PE=1 SV=4 + 1382 + + + + 0 + 0 + 24 + 441350 + 0.071 + 0.299 + 0.27 + + + No hits found + + + 15 + Query_3 + sp|P06213|INSR_HUMAN Insulin receptor OS=Homo sapiens GN=INSR PE=1 SV=4 + 1382 + + + + 0 + 0 + 24 + 441350 + 0.071 + 0.299 + 0.27 + + + No hits found + + + 16 + Query_3 + sp|P06213|INSR_HUMAN Insulin receptor OS=Homo sapiens GN=INSR PE=1 SV=4 + 1382 + + + + 0 + 0 + 24 + 441350 + 0.071 + 0.299 + 0.27 + + + No hits found + + + 17 + Query_3 + sp|P06213|INSR_HUMAN Insulin receptor OS=Homo sapiens GN=INSR PE=1 SV=4 + 1382 + + + + 0 + 0 + 24 + 441350 + 0.071 + 0.299 + 0.27 + + + No hits found + + + 18 + Query_3 + sp|P06213|INSR_HUMAN Insulin receptor OS=Homo sapiens GN=INSR PE=1 SV=4 + 1382 + + + + 0 + 0 + 24 + 441350 + 0.071 + 0.299 + 0.27 + + + No hits found + + + 19 + Query_4 + sp|P08100|OPSD_HUMAN Rhodopsin OS=Homo sapiens GN=RHO PE=1 SV=1 + 348 + + + 1 + Subject_1 + gi|57163782|ref|NM_001009242.1| Felis catus rhodopsin (RHO), mRNA + Subject_1 + 1047 + + + 1 + 732.392902459534 + 1689 + 0 + 1 + 348 + 1 + 1044 + 0 + 1 + 336 + 343 + 0 + 348 + MNGTEGPNFYVPFSNATGVVRSPFEYPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMVLGGFTSTLYTSLHGYFVFGPTGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGVAFTWVMALACAAPPLAGWSRYIPEGLQCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMIIIFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMVIAFLICWVPYASVAFYIFTHQGSNFGPIFMTIPAFFAKSAAIYNPVIYIMMNKQFRNCMLTTICCGKNPLGDDEASATVSKTETSQVAPA + MNGTEGPNFYVPFSNKTGVVRSPFEYPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMVFGGFTTTLYTSLHGYFVFGPTGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGVAFTWVMALACAAPPLVGWSRYIPEGMQCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMIVIFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMVIAFLICWVPYASVAFYIFTHQGSNFGPIFMTLPAFFAKSSSIYNPVIYIMMNKQFRNCMLTTLCCGKNPLGDDEASTTGSKTETSQVAPA + MNGTEGPNFYVPFSN TGVVRSPFEYPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMV GGFT+TLYTSLHGYFVFGPTGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGVAFTWVMALACAAPPL GWSRYIPEG+QCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMI+IFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMVIAFLICWVPYASVAFYIFTHQGSNFGPIFMT+PAFFAKS++IYNPVIYIMMNKQFRNCMLTT+CCGKNPLGDDEAS T SKTETSQVAPA + + + + + + + 0 + 0 + 18 + 109230 + 0.071 + 0.299 + 0.27 + + + + + 20 + Query_4 + sp|P08100|OPSD_HUMAN Rhodopsin OS=Homo sapiens GN=RHO PE=1 SV=1 + 348 + + + 1 + Subject_2 + gi|2734705|gb|U59921.1|BBU59921 Bufo bufo rhodopsin mRNA, complete cds + Subject_2 + 1574 + + + 1 + 646.119739014374 + 1489 + 0 + 1 + 341 + 42 + 1067 + 0 + 3 + 290 + 320 + 1 + 342 + MNGTEGPNFYVPFSNATGVVRSPFEYPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMVLGGFTSTLYTSLHGYFVFGPTGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGVAFTWVMALACAAPPLAGWSRYIPEGLQCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMIIIFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMVIAFLICWVPYASVAFYIFTHQGSNFGPIFMTIPAFFAKSAAIYNPVIYIMMNKQFRNCMLTTICCGKNPLGDDEA-SATVSKTE + MNGTEGPNFYIPMSNKTGVVRSPFEYPQYYLAEPWQYSILCAYMFLLILLGFPINFMTLYVTIQHKKLRTPLNYILLNLAFANHFMVLCGFTVTMYSSMNGYFILGATGCYVEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFSENHAVMGVAFTWIMALSCAVPPLLGWSRYIPEGMQCSCGVDYYTLKPEVNNESFVIYMFVVHFTIPLIIIFFCYGRLVCTVKEAAAQQQESATTQKAEKEVTRMVIIMVVFFLICWVPYASVAFFIFSNQGSEFGPIFMTVPAFFAKSSSIYNPVIYIMLNKQFRNCMITTLCCGKNPFGEDDASSAATSKTE + MNGTEGPNFY+P SN TGVVRSPFEYPQYYLAEPWQ+S+L AYMFLLI+LGFPINF+TLYVT+QHKKLRTPLNYILLNLA A+ FMVL GFT T+Y+S+ GYF+ G TGC +EGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRF ENHA+MGVAFTW+MAL+CA PPL GWSRYIPEG+QCSCG+DYYTLKPEVNNESFVIYMFVVHFTIP+IIIFFCYG+LV TVKEAAAQQQESATTQKAEKEVTRMVIIMV+ FLICWVPYASVAF+IF+ QGS FGPIFMT+PAFFAKS++IYNPVIYIM+NKQFRNCM+TT+CCGKNP G+D+A SA SKTE + + + + + + + 0 + 0 + 18 + 109230 + 0.071 + 0.299 + 0.27 + + + + + 21 + Query_4 + sp|P08100|OPSD_HUMAN Rhodopsin OS=Homo sapiens GN=RHO PE=1 SV=1 + 348 + + + 1 + Subject_3 + gi|283855845|gb|GQ290303.1| Cynopterus brachyotis voucher 20020434 rhodopsin (RHO) gene, exons 1 through 5 and partial cds + Subject_3 + 4301 + + + 1 + 151.343146656381 + 342 + 1.39566684546685e-72 + 239 + 312 + 3147 + 3368 + 0 + 3 + 69 + 73 + 0 + 74 + ESATTQKAEKEVTRMVIIMVIAFLICWVPYASVAFYIFTHQGSNFGPIFMTIPAFFAKSAAIYNPVIYIMMNKQ + ESATTQKAEKEVTRMVIIMVIAFLICWLPYAGVAFYIFTHQGSNFGPIFMTLPAFFAKSSSIYNPVIYIMMNKQ + ESATTQKAEKEVTRMVIIMVIAFLICW+PYA VAFYIFTHQGSNFGPIFMT+PAFFAKS++IYNPVIYIMMNKQ + + + 2 + 126.323929257285 + 284 + 1.39566684546685e-72 + 177 + 235 + 2855 + 3031 + 0 + 2 + 54 + 57 + 0 + 59 + RYIPEGLQCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMIIIFFCYGQLVFTVKEAAA + RYIPEGMQCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMIVIFFCYGQLVFTVKEVRS + RYIPEG+QCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMI+IFFCYGQLVFTVKE + + + + 3 + 229.420359574251 + 523 + 9.84654801241353e-65 + 11 + 121 + 1 + 333 + 0 + 1 + 107 + 109 + 0 + 111 + VPFSNATGVVRSPFEYPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMVLGGFTSTLYTSLHGYFVFGPTGCNLEGFFATLGG + VPFSNKTGVVRSPFEHPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMVFGGFTTTLYTSLHGYFVFGPTGCNLEGFFATLGG + VPFSN TGVVRSPFE+PQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMV GGFT+TLYTSLHGYFVFGPTGCNLEGFFATLGG + + + 4 + 122.873002719478 + 276 + 1.40732096096596e-32 + 119 + 177 + 1404 + 1580 + 0 + 3 + 55 + 56 + 0 + 59 + LGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGVAFTWVMALACAAPPLAGWSR + LAGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGLALTWVMALACAAPPLVGWSR + L GEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMG+A TWVMALACAAPPL GWSR + + + 5 + 57.7367643183824 + 125 + 5.60065526485586e-13 + 312 + 337 + 4222 + 4299 + 0 + 1 + 23 + 24 + 0 + 26 + QFRNCMLTTICCGKNPLGDDEASATV + QFRNCMLTTLCCGKNPLGDDEASTTA + QFRNCMLTT+CCGKNPLGDDEAS T + + + + + + + 0 + 0 + 18 + 109230 + 0.071 + 0.299 + 0.27 + + + + + 22 + Query_4 + sp|P08100|OPSD_HUMAN Rhodopsin OS=Homo sapiens GN=RHO PE=1 SV=1 + 348 + + + 1 + Subject_4 + gi|283855822|gb|GQ290312.1| Myotis ricketti voucher GQX10 rhodopsin (RHO) mRNA, partial cds + Subject_4 + 983 + + + 1 + 658.197981896696 + 1517 + 0 + 11 + 336 + 1 + 978 + 0 + 1 + 310 + 322 + 0 + 326 + VPFSNATGVVRSPFEYPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMVLGGFTSTLYTSLHGYFVFGPTGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGVAFTWVMALACAAPPLAGWSRYIPEGLQCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMIIIFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMVIAFLICWVPYASVAFYIFTHQGSNFGPIFMTIPAFFAKSAAIYNPVIYIMMNKQFRNCMLTTICCGKNPLGDDEASAT + VPFSNKTGVVRSPFEYPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVANLFMVFGGFTTTLYTSMHGYFVFGATGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGLAFTWVMALACAAPPLAGWSRYIPEGMQCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMIVIFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMVVAFLICWLPYASVAFYIFTHQGSNFGPVFMTIPAFFAKSSSIYNPVIYIMMNKQFRNCMLTTLCCGKNPLGDDEASTT + VPFSN TGVVRSPFEYPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVA+LFMV GGFT+TLYTS+HGYFVFG TGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMG+AFTWVMALACAAPPLAGWSRYIPEG+QCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMI+IFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMV+AFLICW+PYASVAFYIFTHQGSNFGP+FMTIPAFFAKS++IYNPVIYIMMNKQFRNCMLTT+CCGKNPLGDDEAS T + + + + + + + 0 + 0 + 18 + 109230 + 0.071 + 0.299 + 0.27 + + + + + 23 + Query_4 + sp|P08100|OPSD_HUMAN Rhodopsin OS=Homo sapiens GN=RHO PE=1 SV=1 + 348 + + + 1 + Subject_5 + gi|18148870|dbj|AB062417.1| Synthetic construct Bos taurus gene for rhodopsin, complete cds + Subject_5 + 1047 + + + 1 + 711.255977415469 + 1640 + 0 + 1 + 348 + 1 + 1044 + 0 + 1 + 325 + 337 + 0 + 348 + MNGTEGPNFYVPFSNATGVVRSPFEYPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMVLGGFTSTLYTSLHGYFVFGPTGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGVAFTWVMALACAAPPLAGWSRYIPEGLQCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMIIIFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMVIAFLICWVPYASVAFYIFTHQGSNFGPIFMTIPAFFAKSAAIYNPVIYIMMNKQFRNCMLTTICCGKNPLGDDEASATVSKTETSQVAPA + MNGTEGPNFYVPFSNKTGVVRSPFEAPQYYLAEPWQFSMLAAYMFLLIMLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMVFGGFTTTLYTSLHGYFVFGPTGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGVAFTWVMALACAAPPLVGWSRYIPEGMQCSCGIDYYTPHEETNNESFVIYMFVVHFIIPLIVIFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMVIAFLICWLPYAGVAFYIFTHQGSDFGPIFMTIPAFFAKTSAVYNPVIYIMMNKQFRNCMVTTLCCGKNPLGDDEASTTVSKTETSQVAPA + MNGTEGPNFYVPFSN TGVVRSPFE PQYYLAEPWQFSMLAAYMFLLI+LGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMV GGFT+TLYTSLHGYFVFGPTGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGVAFTWVMALACAAPPL GWSRYIPEG+QCSCGIDYYT E NNESFVIYMFVVHF IP+I+IFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMVIAFLICW+PYA VAFYIFTHQGS+FGPIFMTIPAFFAK++A+YNPVIYIMMNKQFRNCM+TT+CCGKNPLGDDEAS TVSKTETSQVAPA + + + + + + + 0 + 0 + 18 + 109230 + 0.071 + 0.299 + 0.27 + + + + + 24 + Query_4 + sp|P08100|OPSD_HUMAN Rhodopsin OS=Homo sapiens GN=RHO PE=1 SV=1 + 348 + + + 1 + Subject_6 + gi|12583664|dbj|AB043817.1| Conger myriaster conf gene for fresh water form rod opsin, complete cds + Subject_6 + 1344 + + + 1 + 626.708277239213 + 1444 + 0 + 1 + 341 + 23 + 1048 + 0 + 2 + 281 + 311 + 1 + 342 + MNGTEGPNFYVPFSNATGVVRSPFEYPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMVLGGFTSTLYTSLHGYFVFGPTGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGVAFTWVMALACAAPPLAGWSRYIPEGLQCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMIIIFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMVIAFLICWVPYASVAFYIFTHQGSNFGPIFMTIPAFFAKSAAIYNPVIYIMMNKQFRNCMLTTICCGKNPL-GDDEASATVSKTE + MNGTEGPNFYIPMSNATGVVRSPFEYPQYYLAEPWAFSALSAYMFFLIIAGFPINFLTLYVTIEHKKLRTPLNYILLNLAVADLFMVFGGFTTTMYTSMHGYFVFGPTGCNIEGFFATLGGEIALWCLVVLAIERWMVVCKPVTNFRFGESHAIMGVMVTWTMALACALPPLFGWSRYIPEGLQCSCGIDYYTRAPGINNESFVIYMFTCHFSIPLAVISFCYGRLVCTVKEAAAQQQESETTQRAEREVTRMVVIMVISFLVCWVPYASVAWYIFTHQGSTFGPIFMTIPSFFAKSSALYNPMIYICMNKQFRHCMITTLCCGKNPFEEEDGASATSSKTE + MNGTEGPNFY+P SNATGVVRSPFEYPQYYLAEPW FS L+AYMF LI+ GFPINFLTLYVT++HKKLRTPLNYILLNLAVADLFMV GGFT+T+YTS+HGYFVFGPTGCN+EGFFATLGGEIALW LVVLAIER++VVCKP++NFRFGE HAIMGV TW MALACA PPL GWSRYIPEGLQCSCGIDYYT P +NNESFVIYMF HF+IP+ +I FCYG+LV TVKEAAAQQQES TTQ+AE+EVTRMV+IMVI+FL+CWVPYASVA YIFTHQGS FGPIFMTIP+FFAKS+A+YNP+IYI MNKQFR CM+TT+CCGKNP +D ASAT SKTE + + + + + + + 0 + 0 + 18 + 109230 + 0.071 + 0.299 + 0.27 + + + + + diff --git a/lib/galaxy/datatypes/util/gff_util.py b/lib/galaxy/datatypes/util/gff_util.py index 2d1a9c6828e..0a960e9e47a 100644 --- a/lib/galaxy/datatypes/util/gff_util.py +++ b/lib/galaxy/datatypes/util/gff_util.py @@ -67,11 +67,18 @@ class GFFFeature( GFFInterval ): def name( self ): """ Returns feature's name. """ - name = self.attributes.get( 'transcript_id', None ) - if not name: - name = self.attributes.get( 'id', None ) - if not name: - name = self.attributes.get( 'group', None ) + name = None + # Preference for name: GTF, GFF3, GFF. + for attr_name in [ + # GTF: + 'transcript_id', 'gene_id', + # GFF3: + 'ID', 'id', + # GFF (TODO): + 'group' ]: + name = self.attributes.get( attr_name, None ) + if name is not None: + break return name def copy( self ): @@ -80,6 +87,13 @@ class GFFFeature( GFFInterval ): intervals_copy.append( interval.copy() ) return GFFFeature(self.reader, self.chrom_col, self.feature_col, self.start_col, self.end_col, self.strand_col, self.score_col, self.strand, intervals=intervals_copy ) + + def lines( self ): + lines = [] + for interval in self.intervals: + lines.append( '\t'.join( interval.fields ) ) + return lines + class GFFIntervalToBEDReaderWrapper( NiceReaderWrapper ): """ @@ -119,6 +133,7 @@ class GFFReaderWrapper( NiceReaderWrapper ): self.last_line = None self.cur_offset = 0 self.seed_interval = None + self.seed_interval_line_len = 0 def parse_row( self, line ): interval = GFFInterval( self, line.split( "\t" ), self.chrom_col, self.feature_col, \ @@ -146,12 +161,12 @@ class GFFReaderWrapper( NiceReaderWrapper ): # For debugging, uncomment this to propogate parsing exceptions up. # I.e. the underlying reason for an unexpected StopIteration exception # can be found by uncommenting this. - #raise e + # raise e # # Get next GFFFeature # - raw_size = 0 + raw_size = self.seed_interval_line_len # If there is no seed interval, set one. Also, if there are no more # intervals to read, this is where iterator dies. @@ -170,13 +185,17 @@ class GFFReaderWrapper( NiceReaderWrapper ): return_val = self.seed_interval return_val.raw_size = len( self.current_line ) self.seed_interval = None + self.seed_interval_line_len = 0 return return_val - - # Initialize feature name from seed. + + # Initialize feature identifier from seed. feature_group = self.seed_interval.attributes.get( 'group', None ) # For GFF - feature_id = self.seed_interval.attributes.get( 'ID', None ) # For GFF3 - feature_gene_id = self.seed_interval.attributes.get( 'gene_id', None ) # For GTF - feature_transcript_id = self.seed_interval.attributes.get( 'transcript_id', None ) # For GTF + # For GFF3 + feature_id = self.seed_interval.attributes.get( 'ID', None ) + feature_parent_id = self.seed_interval.attributes.get( 'Parent', None ) + # For GTF. + feature_gene_id = self.seed_interval.attributes.get( 'gene_id', None ) + feature_transcript_id = self.seed_interval.attributes.get( 'transcript_id', None ) # Read all intervals associated with seed. feature_intervals = [] @@ -199,20 +218,32 @@ class GFFReaderWrapper( NiceReaderWrapper ): #finally: #raw_size += len( self.current_line ) - # If interval not associated with feature, break. + # Ignore comments. + if isinstance( interval, Comment ): + continue + + # Determine if interval is part of feature. + part_of = True group = interval.attributes.get( 'group', None ) # GFF test: if group and feature_group != group: - break + part_of = False # GFF3 test: - parent = interval.attributes.get( 'Parent', None ) - if feature_id and feature_id != parent: - break + parent_id = interval.attributes.get( 'Parent', None ) + cur_id = interval.attributes.get( 'ID', None ) + if ( cur_id and cur_id != feature_id ) or ( parent_id and parent_id != feature_id ): + part_of = False # GTF test: gene_id = interval.attributes.get( 'gene_id', None ) transcript_id = interval.attributes.get( 'transcript_id', None ) if ( transcript_id and transcript_id != feature_transcript_id ) or \ ( gene_id and gene_id != feature_gene_id ): + part_of = False + + # If interval is not part of feature, clean up and break. + if not part_of: + # Adjust raw size because current line is not part of feature. + raw_size -= len( self.current_line ) break # Interval associated with feature. @@ -220,6 +251,7 @@ class GFFReaderWrapper( NiceReaderWrapper ): # Last interval read is the seed for the next interval. self.seed_interval = interval + self.seed_interval_line_len = len( self.current_line ) # Return feature. feature = GFFFeature( self, self.chrom_col, self.feature_col, self.start_col, \ @@ -230,10 +262,9 @@ class GFFReaderWrapper( NiceReaderWrapper ): # Convert to BED coords? if self.convert_to_bed_coord: convert_gff_coords_to_bed( feature ) - + return feature - def convert_bed_coords_to_gff( interval ): """ Converts an interval object's coordinates from BED format to GFF format. @@ -280,10 +311,11 @@ def parse_gff_attributes( attr_str ): attributes_list = attr_str.split(";") attributes = {} for name_value_pair in attributes_list: - # Try splitting by space and, if necessary, by '=' sign. - pair = name_value_pair.strip().split(" ") + # Try splitting by '=' (GFF3) first because spaces are allowed in GFF3 + # attribute; next, try double quotes for GTF. + pair = name_value_pair.strip().split("=") if len( pair ) == 1: - pair = name_value_pair.strip().split("=") + pair = name_value_pair.strip().split("\"") if len( pair ) == 1: # Could not split for some reason -- raise exception? continue diff --git a/lib/galaxy/datatypes/util/image_util.py b/lib/galaxy/datatypes/util/image_util.py new file mode 100644 index 00000000000..5e2e7964d2a --- /dev/null +++ b/lib/galaxy/datatypes/util/image_util.py @@ -0,0 +1,76 @@ +""" +Provides utilities for working with image files. +""" +import logging, imghdr + +try: + import Image as PIL +except ImportError: + try: + from PIL import Image as PIL + except: + PIL = None + +log = logging.getLogger(__name__) + +def image_type( filename, image=None ): + format = '' + if PIL is not None: + if image is not None: + format = image.format + else: + try: + im = PIL.open( filename ) + format = im.format + im.close() + except: + return False + else: + format = imghdr.what( filename ) + if format is not None: + format = format.upper() + else: + return False + return format +def check_image_type( filename, types, image=None ): + format = image_type( filename, image ) + # First check if we can use PIL + if format in types: + return True + return False +def get_image_ext ( file_path, image ): + #determine ext + format = image_type( file_path, image ) + if format in [ 'JPG','JPEG' ]: + return 'jpg' + if format == 'PNG': + return 'png' + if format == 'TIFF': + return 'tiff' + if format == 'BMP': + return 'bmp' + if format == 'GIF': + return 'gif' + if format == 'IM': + return 'im' + if format == 'PCD': + return 'pcd' + if format == 'PCX': + return 'pcx' + if format == 'PPM': + return 'ppm' + if format == 'PSD': + return 'psd' + if format == 'XBM': + return 'xbm' + if format == 'XPM': + return 'xpm' + if format == 'RGB': + return 'rgb' + if format == 'PBM': + return 'pbm' + if format == 'PGM': + return 'pgm' + if format == 'EPS': + return 'eps' + return None diff --git a/lib/galaxy/datatypes/wsf.py b/lib/galaxy/datatypes/wsf.py index 3610c1e199a..551c10515d2 100644 --- a/lib/galaxy/datatypes/wsf.py +++ b/lib/galaxy/datatypes/wsf.py @@ -2,6 +2,7 @@ SnpFile datatype """ +import re import data from galaxy import util from galaxy.datatypes.sniff import * @@ -9,108 +10,147 @@ from galaxy.datatypes.tabular import Tabular from galaxy.datatypes import metadata from galaxy.datatypes.metadata import MetadataElement -snp_type_dict = {} -snp_required_columns = ('scaffold', 'pos') -snp_required_column_count = len( snp_required_columns ) -snp_required_index = {} -snp_column_set = set( snp_required_columns ) - -""" check for duplicate required columns """ -for i, column in enumerate( snp_required_columns ): - assert column not in snp_required_index, \ - "duplicate required column: '%s'" % column - snp_required_index[column] = i - -def add_type( type_list=None, name_list=None, species=None, comment=None ): - type_list_len = len( type_list ) - name_list_len = len( name_list ) - assert type_list_len == name_list_len, \ - "type length mismatch: '%s' has %d names and %d types" % ( comment, name_list_len, type_list_len ) - for column in snp_required_columns: - assert column in name_list, \ - "type missing required column: '%s' missing %s column" % ( comment, column ) - type_key = tuple( type_list ) - type_data = tuple( [tuple( name_list ), species, comment] ) - if type_list_len in snp_type_dict: - assert type_key not in snp_type_dict[type_list_len], \ - "type collision: column count and types for '%s' and '%s' match" % ( comment, snp_type_dict[type_list_len][type_key][2] ) - snp_type_dict[type_list_len][type_key] = type_data - else: - snp_type_dict[type_list_len] = { type_key:type_data } - snp_column_set.update( name_list ) - -""" add our types """ -add_type( - species='tasmanian_devil', - comment='tasmanian devil coding snps', - type_list=['str', 'int', 'str', 'str', 'str', 'str', 'str', 'int', 'str', 'int', 'int', 'int', 'int', 'int', 'int', 'int', 'int', 'int', 'str', 'int', 'float', 'int'], - name_list=['scaffold', 'pos', 'A', 'B', 'aa1', 'aa2', 'ref', 'rPos', 'rAA', '#CA', '#CB', 'CQ', '#SA', '#SB', 'SQ', '#TA', '#TB', 'GQ', 'pair', 'sep', 'prim', '#RFLP'] -) -add_type( - species='tasmanian_devil', - comment='tasmanian devil noncoding snps', - type_list=['str', 'int', 'str', 'str', 'int', 'int', 'int', 'int', 'int', 'int', 'int', 'int', 'int', 'str', 'int', 'float', 'int'], - name_list=['scaffold', 'pos', 'A', 'B', '#CA', '#CB', 'CQ', '#SA', '#SB', 'SQ', '#TA', '#TB', 'GQ', 'pair', 'sep', 'prim', '#RFLP'] -) -add_type( - species='bighorn', - comment='bighorn sheep coding snps', - type_list=['str', 'int', 'str', 'str', 'str', 'str', 'str', 'int', 'str', 'str', 'int', 'int', 'int', 'int', 'int', 'int', 'str', 'int', 'float', 'float', 'float', 'int'], - name_list=['scaffold', 'pos', 'A', 'B', 'aa1', 'aa2', 'ref', 'rPos', 'rNuc', 'rAA', '#desA', '#desB', 'desQ', '#mtA', '#mtB', 'mtQ', 'pair', 'sep', 'Fst', 'cons', 'prim', '#RFLP'] -) -add_type( - species='bighorn', - comment='bighorn sheep noncoding snps', - type_list=['str', 'int', 'str', 'str', 'str', 'int', 'str', 'int', 'int', 'int', 'int', 'int', 'int', 'str', 'int', 'float', 'float', 'float', 'int'], - name_list=['scaffold', 'pos', 'A', 'B', 'ref', 'rPos', 'rNuc', '#desA', '#desB', 'desQ', '#mtA', '#mtB', 'mtQ', 'pair', 'sep', 'Fst', 'cons', 'prim', '#RFLP'] -) - - class SnpFile( Tabular ): """ Webb's SNP file format """ file_ext = 'wsf' + species_regex = re.compile('species=(\S+)') + MetadataElement( name="species", desc="species", default='', no_value='', visible=False, readonly=True ) + MetadataElement( name="scaffold", desc="scaffold column", param=metadata.ColumnParameter, default=0 ) + MetadataElement( name="pos", desc="pos column", param=metadata.ColumnParameter, default=0 ) + MetadataElement( name="ref", desc="ref column", param=metadata.ColumnParameter, default=0 ) + MetadataElement( name="rPos", desc="rPos column", param=metadata.ColumnParameter, default=0 ) + MetadataElement( name="labels", desc="Number of labels", default=0, no_value=0, visible=False, readonly=True ) + MetadataElement( name="label_for_column", desc="Mapping from column to label", default=[], no_value=[], visible=False, readonly=True ) + MetadataElement( name="columns_with_label", desc="Mapping from label to columns", param=metadata.DictParameter, default={}, no_value={}, visible=False, readonly=True ) + MetadataElement( name="column_headers", desc="Column headers", default=[], no_value=[], visible=False, readonly=True ) - """ add metadata elements """ - MetadataElement( name="species", desc="species", readonly=True, no_value=None ) - for name in sorted( snp_column_set ): - default = 0 - desc = "%s column" % name - optional = False - if name in snp_required_columns: - default = snp_required_index[name] + 1 - optional = True - MetadataElement( name=name, default=default, desc=desc, param=metadata.ColumnParameter, optional=optional ) def set_meta( self, dataset, overwrite = True, **kwd ): - Tabular.set_meta( self, dataset, overwrite = overwrite, **kwd ) - if dataset.has_data(): - if dataset.metadata.columns in snp_type_dict: - type_key = tuple( dataset.metadata.column_types ) - if type_key in snp_type_dict[dataset.metadata.columns]: - name_list, dataset.metadata.species, comment = snp_type_dict[dataset.metadata.columns][type_key] - for i, name in enumerate( name_list[snp_required_column_count:] ): - setattr( dataset.metadata, name, snp_required_column_count + i + 1 ) + Tabular.set_meta( self, dataset, overwrite=overwrite, max_data_lines=None, **kwd ) + # these two if statements work around a potential bug in metadata.py + if dataset.metadata.labels is None or dataset.metadata.labels == dataset.metadata.spec['labels'].no_value: + self._set_column_labels_metadata( dataset ) + if dataset.metadata.column_headers is None or dataset.metadata.column_headers == dataset.metadata.spec['column_headers'].no_value: + self._set_column_headers_metadata( dataset ) + self._set_columnParameter_metadata( dataset ) - def make_html_table(self, dataset, skipchars=[] ): + + def _set_column_labels_metadata( self, dataset ): + def build_map_from_label_to_comma_separated_column_list( labels ): + map = {} + for index, label in enumerate( labels ): + map.setdefault( label, [] ).append( index ) + + for label in map: + map[label] = ','.join( [ str( index + 1 ) for index in map[label] ] ) + return map + + def strip_list_elements( list ): + return [ element.strip() for element in list ] + + def initial_comment_lines_of_dataset( dataset ): + comment_lines = [] + if dataset.has_data(): + try: + fh = open( dataset.file_name, 'r' ) + for line in fh: + if not line.startswith('#'): + break + line = line[1:] + line = line.rstrip( '\r\n' ) + if line: + comment_lines.append( line ) + fh.close() + except: + pass + return comment_lines + + def set_metadata_from_comment_lines( dataset ): + labels = [] + comment_lines = initial_comment_lines_of_dataset( dataset ) + + for line in comment_lines: + match = SnpFile.species_regex.match( line ) + if match: + dataset.metadata.species = match.group(1) + continue + elems = line.split( '\t' ) + if len(elems) > 1: + labels = strip_list_elements( elems ) + + dataset.metadata.labels = len( labels ) + dataset.metadata.label_for_column = labels[:] + if labels: + dataset.metadata.label_for_column.insert(0, '') + dataset.metadata.columns_with_label = build_map_from_label_to_comma_separated_column_list( labels ) + + set_metadata_from_comment_lines( dataset ) + + + def _set_column_headers_metadata( self, dataset ): + if dataset.metadata.labels < dataset.metadata.columns: + column_headers = dataset.metadata.label_for_column[1:] + [ '' ] * ( dataset.metadata.columns - dataset.metadata.labels ) + else: + column_headers = dataset.metadata.label_for_column[1:dataset.metadata.columns+1] + + dataset.metadata.column_headers = column_headers + + + def _set_columnParameter_metadata( self, dataset ): + def unique_column_number_or_zero( string ): + try: + val = int( string ) + except: + val = 0 + return val + + for name in self._metadata_columnParameter_names( dataset ): + if name in dataset.metadata.columns_with_label: + if dataset.metadata.columns_with_label[name]: + column = unique_column_number_or_zero( dataset.metadata.columns_with_label[name] ) + if column: + setattr( dataset.metadata, name, column ) + + + def _metadata_columnParameter_names( self, dataset ): + for name, spec in dataset.metadata.spec.items(): + if isinstance( spec.param, metadata.ColumnParameter ): + yield name + + + def set_peek( self, dataset, line_count=None, is_multi_byte=False ): + super(Tabular, self).set_peek( dataset, line_count=line_count, is_multi_byte=is_multi_byte, skipchars=[ '#' ]) + + + def make_html_table( self, dataset, skipchars=[ '#' ] ): """Create HTML table, used for displaying peek""" - out = [''] - try: - out.append( '' ) - col_names = range( 0, dataset.metadata.columns + 1 ) - for name, spec in dataset.metadata.spec.items(): - if isinstance( spec.param, metadata.ColumnParameter ): - col = getattr( dataset.metadata, name ) - if col > 0 and col_names[col] == col: - col_names[col] = name - for i, name in enumerate( col_names[1:] ): - if col_names[ i + 1 ] == i + 1: - out.append( '' % ( i + 1 ) ) + def table_header_values( dataset ): + headers = dataset.metadata.column_headers[:] + for name in self._metadata_columnParameter_names( dataset ): + col = getattr( dataset.metadata, name ) + assert col <= dataset.metadata.columns, Exception( 'ColumnParameter %s %d > %d columns for dataset %s.' % ( name, col, dataset.metadata.columns, dataset.id ) ) + if col > 0: + headers[ col - 1 ] = name + return headers + + def table_headers( dataset ): + out = [ '' ] + headers = table_header_values( dataset ) + for index, header in enumerate( headers ): + column = index + 1 + if header: + out.append( "" % ( column, header ) ) else: - out.append( '' % ( i + 1, name ) ) + out.append( "" % column ) + out.append( '' ) + return out + + try: + out = ['
    %d.
    %d.%s%d.%s%d.
    '] + out.extend( table_headers( dataset ) ) out.append( self.make_html_peek_rows( dataset, skipchars=skipchars ) ) out.append( '
    ' ) out = "".join( out ) except Exception, exc: out = "Can't create peek %s" % exc return out - diff --git a/lib/galaxy/datatypes/xml.py b/lib/galaxy/datatypes/xml.py index 6fc25c34320..2766982af26 100644 --- a/lib/galaxy/datatypes/xml.py +++ b/lib/galaxy/datatypes/xml.py @@ -7,7 +7,43 @@ from galaxy.datatypes.sniff import * log = logging.getLogger(__name__) -class BlastXml( data.Text ): +class GenericXml( data.Text ): + """Base format class for any XML file.""" + file_ext = "xml" + + def set_peek( self, dataset, is_multi_byte=False ): + """Set the peek and blurb text""" + if not dataset.dataset.purged: + dataset.peek = data.get_file_peek( dataset.file_name, is_multi_byte=is_multi_byte ) + dataset.blurb = 'XML data' + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' + + def sniff( self, filename ): + """ + Determines whether the file is XML or not + + >>> fname = get_test_fname( 'megablast_xml_parser_test1.blastxml' ) + >>> GenericXml().sniff( fname ) + True + >>> fname = get_test_fname( 'tblastn_four_human_vs_rhodopsin.xml' ) + >>> BlastXml().sniff( fname ) + True + >>> fname = get_test_fname( 'interval.interval' ) + >>> GenericXml().sniff( fname ) + False + """ + #TODO - Use a context manager on Python 2.5+ to close handle + handle = open(filename) + line = handle.readline() + handle.close() + + #TODO - Is there a more robust way to do this? + return line.startswith('>> fname = get_test_fname( 'megablast_xml_parser_test1.blastxml' ) >>> BlastXml().sniff( fname ) True + >>> fname = get_test_fname( 'tblastn_four_human_vs_rhodopsin.xml' ) + >>> BlastXml().sniff( fname ) + True >>> fname = get_test_fname( 'interval.interval' ) >>> BlastXml().sniff( fname ) False """ - blastxml_header = [ '', - '', - '' ] - for i, line in enumerate( file( filename ) ): - if i >= len( blastxml_header ): - return True - line = line.rstrip( '\n\r' ) - if line != blastxml_header[ i ]: - return False + #TODO - Use a context manager on Python 2.5+ to close handle + handle = open(filename) + line = handle.readline() + if line.strip() != '': + handle.close() + return False + line = handle.readline() + if line.strip() not in ['', + '']: + handle.close() + return False + line = handle.readline() + if line.strip() != '': + handle.close() + return False + handle.close() + return True + -class MEMEXml( data.Text ): +class MEMEXml( GenericXml ): """MEME XML Output data""" file_ext = "memexml" @@ -54,3 +102,18 @@ class MEMEXml( data.Text ): dataset.blurb = 'file purged from disk' def sniff( self, filename ): return False + +class CisML( GenericXml ): + """CisML XML data""" #see: http://www.ncbi.nlm.nih.gov/pubmed/15001475 + file_ext = "cisml" + + def set_peek( self, dataset, is_multi_byte=False ): + """Set the peek and blurb text""" + if not dataset.dataset.purged: + dataset.peek = data.get_file_peek( dataset.file_name, is_multi_byte=is_multi_byte ) + dataset.blurb = 'CisML data' + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' + def sniff( self, filename ): + return False diff --git a/lib/galaxy/eggs/__init__.py b/lib/galaxy/eggs/__init__.py index 01402dde8fb..eecf0896680 100644 --- a/lib/galaxy/eggs/__init__.py +++ b/lib/galaxy/eggs/__init__.py @@ -247,7 +247,7 @@ class Crate( object ): Reads the eggs.ini file for use with checking and fetching. """ config_file = os.path.join( galaxy_dir, 'eggs.ini' ) - def __init__( self, platform=None ): + def __init__( self, galaxy_config_file, platform=None ): self.eggs = {} self.config = CaseSensitiveConfigParser() self.repo = None @@ -256,7 +256,7 @@ class Crate( object ): self.py_platform = None if platform is not None: self.py_platform = platform.split( '-' )[0] - self.galaxy_config = GalaxyConfig() + self.galaxy_config = GalaxyConfig( galaxy_config_file ) self.parse() def parse( self ): self.config.read( Crate.config_file ) @@ -349,12 +349,14 @@ class Crate( object ): raise EggNotFetchable( missing ) class GalaxyConfig( object ): - config_file = os.path.join( galaxy_dir, "universe_wsgi.ini" ) always_conditional = ( 'GeneTrack', 'pysam', 'ctypes', 'python_daemon' ) - def __init__( self ): - self.config = ConfigParser.ConfigParser() - if self.config.read( GalaxyConfig.config_file ) == []: - raise Exception( "error: unable to read Galaxy config from %s" % GalaxyConfig.config_file ) + def __init__( self, config_file ): + if config_file is None: + self.config = None + else: + self.config = ConfigParser.ConfigParser() + if self.config.read( config_file ) == []: + raise Exception( "error: unable to read Galaxy config from %s" % config_file ) def check_conditional( self, egg_name ): def check_pysam(): # can't build pysam on solaris < 10 @@ -364,6 +366,10 @@ class GalaxyConfig( object ): if int( minor ) < 10: return False return True + # If we're using require() we may not have a Galaxy config file, but if + # we're using require(), we don't care about conditionals. + if self.config is None: + return True if egg_name == "pysqlite": # SQLite is different since it can be specified in two config vars and defaults to True try: @@ -396,7 +402,7 @@ def get_env(): env = get_env() def require( req_str ): - c = Crate() + c = Crate( None ) req = pkg_resources.Requirement.parse( req_str ) # TODO: This breaks egg version requirements. Not currently a problem, but # it could become one. diff --git a/lib/galaxy/eggs/dist.py b/lib/galaxy/eggs/dist.py index 70947d7aae5..e058d5f2a04 100644 --- a/lib/galaxy/eggs/dist.py +++ b/lib/galaxy/eggs/dist.py @@ -39,10 +39,10 @@ class DistScrambleCrate( ScrambleCrate ): Holds eggs with info on how to build them for distribution. """ dist_config_file = os.path.join( galaxy_dir, 'dist-eggs.ini' ) - def __init__( self, build_on='all' ): + def __init__( self, galaxy_config_file, build_on='all' ): self.dist_config = CaseSensitiveConfigParser() self.build_on = build_on - ScrambleCrate.__init__( self ) + ScrambleCrate.__init__( self, galaxy_config_file ) def parse( self ): self.dist_config.read( DistScrambleCrate.dist_config_file ) self.hosts = dict( self.dist_config.items( 'hosts' ) ) diff --git a/lib/galaxy/exceptions/__init__.py b/lib/galaxy/exceptions/__init__.py new file mode 100644 index 00000000000..0b1ea5af683 --- /dev/null +++ b/lib/galaxy/exceptions/__init__.py @@ -0,0 +1,20 @@ +""" +Custom exceptions for Galaxy +""" + +class MessageException( Exception ): + """ + Exception to make throwing errors from deep in controllers easier + """ + def __init__( self, err_msg, type="info" ): + self.err_msg = err_msg + self.type = type + +class ItemDeletionException( MessageException ): + pass + +class ItemAccessibilityException( MessageException ): + pass + +class ItemOwnershipException( MessageException ): + pass diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 64566d3a6d0..65283dcf132 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -11,6 +11,8 @@ from galaxy.util.json import from_json_string from galaxy.util.expressions import ExpressionContext from galaxy.jobs.actions.post import ActionBox +from sqlalchemy.sql.expression import and_, or_ + import pkg_resources pkg_resources.require( "PasteDeploy" ) @@ -30,9 +32,9 @@ TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json' class JobManager( object ): """ Highest level interface to job management. - + TODO: Currently the app accesses "job_queue" and "job_stop_queue" directly. - This should be decoupled. + This should be decoupled. """ def __init__( self, app ): self.app = app @@ -69,7 +71,7 @@ class Sleeper( object ): class JobQueue( object ): """ - Job manager, waits for jobs to be runnable and then dispatches to + Job manager, waits for jobs to be runnable and then dispatches to a JobRunner. """ STOP_SIGNAL = object() @@ -93,7 +95,7 @@ class JobQueue( object ): self.running = True self.dispatcher = dispatcher self.monitor_thread = threading.Thread( target=self.__monitor ) - self.monitor_thread.start() + self.monitor_thread.start() log.info( "job manager started" ) if app.config.get_bool( 'enable_job_recovery', True ): self.__check_jobs_at_startup() @@ -130,7 +132,7 @@ class JobQueue( object ): def __monitor( self ): """ - Continually iterate the waiting jobs, checking is each is ready to + Continually iterate the waiting jobs, checking is each is ready to run and dispatching if so. """ # HACK: Delay until after forking, we need a way to do post fork notification!!! @@ -163,6 +165,10 @@ class JobQueue( object ): .options( lazyload( "external_output_metadata" ), lazyload( "parameters" ) ) \ .filter( model.Job.state == model.Job.states.NEW ).all() else: + # Get job objects and append to watch queue for any which were + # previously waiting + for job_id in self.waiting_jobs: + jobs_to_check.append( self.sa_session.query( model.Job ).get( job_id ) ) try: while 1: message = self.queue.get_nowait() @@ -174,16 +180,12 @@ class JobQueue( object ): jobs_to_check.append( self.sa_session.query( model.Job ).get( job_id ) ) except Empty: pass - # Get job objects and append to watch queue for any which were - # previously waiting - for job_id in self.waiting_jobs: - jobs_to_check.append( self.sa_session.query( model.Job ).get( job_id ) ) - # Iterate over new and waiting jobs and look for any that are + # Iterate over new and waiting jobs and look for any that are # ready to run new_waiting_jobs = [] for job in jobs_to_check: try: - # Check the job's dependencies, requeue if they're not done + # Check the job's dependencies, requeue if they're not done job_state = self.__check_if_ready_to_run( job ) if job_state == JOB_WAIT: if not self.track_jobs_in_database: @@ -203,7 +205,7 @@ class JobQueue( object ): elif job_state == JOB_DELETED: log.info( "job %d deleted by user while still queued" % job.id ) elif job_state == JOB_ADMIN_DELETED: - job.info( "job %d deleted by admin while still queued" % job.id ) + log.info( "job %d deleted by admin while still queued" % job.id ) else: log.error( "unknown job state '%s' for job %d" % ( job_state, job.id ) ) if not self.track_jobs_in_database: @@ -214,7 +216,7 @@ class JobQueue( object ): self.waiting_jobs = new_waiting_jobs # Done with the session self.sa_session.remove() - + def __check_if_ready_to_run( self, job ): """ Check if a job is ready to run by verifying that each of its input @@ -229,7 +231,16 @@ class JobQueue( object ): return JOB_DELETED elif job.state == model.Job.states.ERROR: return JOB_ADMIN_DELETED - for dataset_assoc in job.input_datasets: + elif self.app.config.enable_quotas: + quota = self.app.quota_agent.get_quota( job.user ) + if quota is not None: + try: + usage = self.app.quota_agent.get_usage( user=job.user, history=job.history ) + if usage > quota: + return JOB_WAIT + except AssertionError, e: + pass # No history, should not happen with an anon user + for dataset_assoc in job.input_datasets + job.input_library_datasets: idata = dataset_assoc.dataset if not idata: continue @@ -247,14 +258,36 @@ class JobQueue( object ): elif idata.state != idata.states.OK and not ( idata.state == idata.states.SETTING_METADATA and job.tool_id is not None and job.tool_id == self.app.datatypes_registry.set_external_metadata_tool.id ): # need to requeue return JOB_WAIT + return self.__check_user_jobs( job ) + + def __check_user_jobs( self, job ): + if not self.app.config.user_job_limit: + return JOB_READY + if job.user: + user_jobs = self.sa_session.query( model.Job ) \ + .options( lazyload( "external_output_metadata" ), lazyload( "parameters" ) ) \ + .filter( and_( model.Job.user_id == job.user.id, + or_( model.Job.state == model.Job.states.RUNNING, + model.Job.state == model.Job.states.QUEUED ) ) ).all() + elif job.galaxy_session: + user_jobs = self.sa_session.query( model.Job ) \ + .options( lazyload( "external_output_metadata" ), lazyload( "parameters" ) ) \ + .filter( and_( model.Job.session_id == job.galaxy_session.id, + or_( model.Job.state == model.Job.states.RUNNING, + model.Job.state == model.Job.states.QUEUED ) ) ).all() + else: + log.warning( 'Job %s is not associated with a user or session so job concurrency limit cannot be checked.' % job.id ) + return JOB_READY + if len( user_jobs ) >= self.app.config.user_job_limit: + return JOB_WAIT return JOB_READY - + def put( self, job_id, tool ): """Add a job to the queue (by job identifier)""" if not self.track_jobs_in_database: self.queue.put( ( job_id, tool.id ) ) self.sleeper.wake() - + def shutdown( self ): """Attempts to gracefully shut down the worker thread""" if self.parent_pid != os.getpid(): @@ -271,7 +304,7 @@ class JobQueue( object ): class JobWrapper( object ): """ - Wraps a 'model.Job' with convience methods for running processes and + Wraps a 'model.Job' with convenience methods for running processes and state management. """ def __init__( self, job, queue ): @@ -284,6 +317,9 @@ class JobWrapper( object ): self.sa_session = self.app.model.context self.extra_filenames = [] self.command_line = None + # Tool versioning variables + self.version_string_cmd = None + self.version_string = "" self.galaxy_lib_dir = None # With job outputs in the working directory, we need the working # directory to be set before prepare is run, or else premature deletion @@ -294,15 +330,15 @@ class JobWrapper( object ): self.output_dataset_paths = None self.tool_provided_job_metadata = None # Wrapper holding the info required to restore and clean up from files used for setting metadata externally - self.external_output_metadata = metadata.JobExternalOutputMetadataWrapper( job ) - + self.external_output_metadata = metadata.JobExternalOutputMetadataWrapper( job ) + def get_job( self ): return self.sa_session.query( model.Job ).get( self.job_id ) - + def get_id_tag(self): # For compatability with drmaa, which uses job_id right now, and TaskWrapper return str(self.job_id) - + def get_param_dict( self ): """ Restore the dictionary of parameters from the database. @@ -311,7 +347,10 @@ class JobWrapper( object ): param_dict = dict( [ ( p.name, p.value ) for p in job.parameters ] ) param_dict = self.tool.params_from_strings( param_dict, self.app ) return param_dict - + + def get_version_string_path( self ): + return os.path.abspath(os.path.join(self.app.config.new_file_path, "GALAXY_VERSION_STRING_%s" % self.job_id)) + def prepare( self ): """ Prepare the job to run by creating the working directory and the @@ -331,10 +370,11 @@ class JobWrapper( object ): # Restore input / output data lists inp_data = dict( [ ( da.name, da.dataset ) for da in job.input_datasets ] ) out_data = dict( [ ( da.name, da.dataset ) for da in job.output_datasets ] ) + inp_data.update( [ ( da.name, da.dataset ) for da in job.input_library_datasets ] ) out_data.update( [ ( da.name, da.dataset ) for da in job.output_library_datasets ] ) - - # Set up output dataset association for export history jobs. Because job - # uses a Dataset rather than an HDA or LDA, it's necessary to set up a + + # Set up output dataset association for export history jobs. Because job + # uses a Dataset rather than an HDA or LDA, it's necessary to set up a # fake dataset association that provides the needed attributes for # preparing a job. class FakeDatasetAssociation ( object ): @@ -361,7 +401,7 @@ class JobWrapper( object ): # ( this used to be performed in the "exec_before_job" hook, but hooks are deprecated ). self.tool.exec_before_job( self.queue.app, inp_data, out_data, param_dict ) # Run the before queue ("exec_before_job") hook - self.tool.call_hook( 'exec_before_job', self.queue.app, inp_data=inp_data, + self.tool.call_hook( 'exec_before_job', self.queue.app, inp_data=inp_data, out_data=out_data, tool=self.tool, param_dict=incoming) self.sa_session.flush() # Build any required config files @@ -389,11 +429,12 @@ class JobWrapper( object ): extra_filenames.append( param_filename ) self.param_dict = param_dict self.extra_filenames = extra_filenames + self.version_string_cmd = self.tool.version_string_cmd return extra_filenames def fail( self, message, exception=False ): """ - Indicate job failure by setting state and message on all output + Indicate job failure by setting state and message on all output datasets. """ job = self.get_job() @@ -425,6 +466,7 @@ class JobWrapper( object ): dataset.blurb = 'tool error' dataset.info = message dataset.set_size() + dataset.dataset.set_total_size() if dataset.ext == 'auto': dataset.extension = 'data' self.sa_session.add( dataset ) @@ -438,7 +480,7 @@ class JobWrapper( object ): if self.tool: self.tool.job_failed( self, message, exception ) self.cleanup() - + def change_state( self, state, info = False ): job = self.get_job() self.sa_session.refresh( job ) @@ -468,12 +510,12 @@ class JobWrapper( object ): job.job_runner_external_id = external_id self.sa_session.add( job ) self.sa_session.flush() - + def finish( self, stdout, stderr ): """ - Called to indicate that the associated command has been run. Updates + Called to indicate that the associated command has been run. Updates the output datasets based on stderr and stdout from the command, and - the contents of the output files. + the contents of the output files. """ # default post job setup self.sa_session.expunge_all() @@ -490,6 +532,12 @@ class JobWrapper( object ): job.state = job.states.ERROR else: job.state = job.states.OK + if self.version_string_cmd: + version_filename = self.get_version_string_path() + if os.path.exists(version_filename): + self.version_string = open(version_filename).read() + os.unlink(version_filename) + if self.app.config.outputs_to_working_directory: for dataset_path in self.get_output_fnames(): try: @@ -537,10 +585,11 @@ class JobWrapper( object ): else: # Security violation. log.exception( "from_work_dir specified a location not in the working directory: %s, %s" % ( source_file, self.working_directory ) ) - + dataset.blurb = 'done' dataset.peek = 'no peek' dataset.info = context['stdout'] + context['stderr'] + dataset.tool_version = self.version_string dataset.set_size() if context['stderr']: dataset.blurb = "error" @@ -551,7 +600,7 @@ class JobWrapper( object ): dataset.init_meta( copy_from=dataset ) #if a dataset was copied, it won't appear in our dictionary: #either use the metadata from originating output dataset, or call set_meta on the copies - #it would be quicker to just copy the metadata from the originating output dataset, + #it would be quicker to just copy the metadata from the originating output dataset, #but somewhat trickier (need to recurse up the copied_from tree), for now we'll call set_meta() if not self.app.config.set_metadata_externally or \ ( not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session ) \ @@ -563,7 +612,7 @@ class JobWrapper( object ): #load metadata from file #we need to no longer allow metadata to be edited while the job is still running, #since if it is edited, the metadata changed on the running output will no longer match - #the metadata that was stored to disk for use via the external process, + #the metadata that was stored to disk for use via the external process, #and the changes made by the user will be lost, without warning or notice dataset.metadata.from_JSON_dict( self.external_output_metadata.get_output_filenames_by_dataset( dataset, self.sa_session ).filename_out ) try: @@ -604,16 +653,17 @@ class JobWrapper( object ): # Flush all the dataset and job changes above. Dataset state changes # will now be seen by the user. self.sa_session.flush() - # Save stdout and stderr + # Save stdout and stderr if len( stdout ) > 32768: log.error( "stdout for job %d is greater than 32K, only first part will be logged to database" % job.id ) job.stdout = stdout[:32768] if len( stderr ) > 32768: log.error( "stderr for job %d is greater than 32K, only first part will be logged to database" % job.id ) - job.stderr = stderr[:32768] + job.stderr = stderr[:32768] # custom post process setup inp_data = dict( [ ( da.name, da.dataset ) for da in job.input_datasets ] ) out_data = dict( [ ( da.name, da.dataset ) for da in job.output_datasets ] ) + inp_data.update( [ ( da.name, da.dataset ) for da in job.input_library_datasets ] ) out_data.update( [ ( da.name, da.dataset ) for da in job.output_library_datasets ] ) param_dict = dict( [ ( p.name, p.value ) for p in job.parameters ] ) # why not re-use self.param_dict here? ##dunno...probably should, this causes tools.parameters.basic.UnvalidatedValue to be used in following methods instead of validated and transformed values during i.e. running workflows param_dict = self.tool.params_from_strings( param_dict, self.app ) @@ -626,18 +676,27 @@ class JobWrapper( object ): # ( this used to be performed in the "exec_after_process" hook, but hooks are deprecated ). self.tool.exec_after_process( self.queue.app, inp_data, out_data, param_dict, job = job ) # Call 'exec_after_process' hook - self.tool.call_hook( 'exec_after_process', self.queue.app, inp_data=inp_data, - out_data=out_data, param_dict=param_dict, + self.tool.call_hook( 'exec_after_process', self.queue.app, inp_data=inp_data, + out_data=out_data, param_dict=param_dict, tool=self.tool, stdout=stdout, stderr=stderr ) job.command_line = self.command_line + bytes = 0 + # Once datasets are collected, set the total dataset size (includes extra files) + for dataset_assoc in job.output_datasets: + dataset_assoc.dataset.dataset.set_total_size() + bytes += dataset_assoc.dataset.dataset.get_total_size() + + if job.user: + job.user.total_disk_usage += bytes + # fix permissions for path in [ dp.real_path for dp in self.get_output_fnames() ]: util.umask_fix_perms( path, self.app.config.umask, 0666, self.app.config.gid ) self.sa_session.flush() log.debug( 'job %d ended' % self.job_id ) self.cleanup() - + def cleanup( self ): # remove temporary files try: @@ -651,10 +710,10 @@ class JobWrapper( object ): galaxy.tools.imp_exp.JobImportHistoryArchiveWrapper( self.job_id ).cleanup_after_job( self.sa_session ) except: log.exception( "Unable to cleanup job %d" % self.job_id ) - + def get_command_line( self ): return self.command_line - + def get_session_id( self ): return self.session_id @@ -671,7 +730,7 @@ class JobWrapper( object ): def get_input_fnames( self ): job = self.get_job() filenames = [] - for da in job.input_datasets: #da is JobToInputDatasetAssociation object + for da in job.input_datasets + job.input_library_datasets: #da is JobToInputDatasetAssociation object if da.dataset: filenames.extend(self.get_input_dataset_fnames(da.dataset)) return filenames @@ -771,7 +830,10 @@ class JobWrapper( object ): sizes = [] output_paths = self.get_output_fnames() for outfile in [ str( o ) for o in output_paths ]: - sizes.append( ( outfile, os.stat( outfile ).st_size ) ) + if os.path.exists( outfile ): + sizes.append( ( outfile, os.stat( outfile ).st_size ) ) + else: + sizes.append( ( outfile, 0 ) ) return sizes def setup_external_metadata( self, exec_dir = None, tmp_dir = None, dataset_files_path = None, config_root = None, datatypes_config = None, set_extension = True, **kwds ): @@ -822,7 +884,7 @@ class TaskWrapper(JobWrapper): Should be refactored into a generalized executable unit wrapper parent, then jobs and tasks. """ # Abstract this to be more useful for running tasks that *don't* necessarily compose a job. - + def __init__(self, task, queue): super(TaskWrapper, self).__init__(task.job, queue) self.task_id = task.id @@ -867,9 +929,10 @@ class TaskWrapper(JobWrapper): self.tool.handle_unvalidated_param_values( incoming, self.app ) # Restore input / output data lists inp_data = dict( [ ( da.name, da.dataset ) for da in job.input_datasets ] ) - # DBTODO New method for generating command line for a task? out_data = dict( [ ( da.name, da.dataset ) for da in job.output_datasets ] ) + inp_data.update( [ ( da.name, da.dataset ) for da in job.input_library_datasets ] ) out_data.update( [ ( da.name, da.dataset ) for da in job.output_library_datasets ] ) + # DBTODO New method for generating command line for a task? # These can be passed on the command line if wanted as $userId $userEmail if job.history and job.history.user: # check for anonymous user! userId = '%d' % job.history.user.id @@ -890,7 +953,7 @@ class TaskWrapper(JobWrapper): # ( this used to be performed in the "exec_before_job" hook, but hooks are deprecated ). self.tool.exec_before_job( self.queue.app, inp_data, out_data, param_dict ) # Run the before queue ("exec_before_job") hook - self.tool.call_hook( 'exec_before_job', self.queue.app, inp_data=inp_data, + self.tool.call_hook( 'exec_before_job', self.queue.app, inp_data=inp_data, out_data=out_data, tool=self.tool, param_dict=incoming) self.sa_session.flush() # Build any required config files @@ -937,12 +1000,12 @@ class TaskWrapper(JobWrapper): task.state = state self.sa_session.add( task ) self.sa_session.flush() - + def get_state( self ): task = self.get_task() self.sa_session.refresh( task ) return task.state - + def set_runner( self, runner_url, external_id ): task = self.get_task() self.sa_session.refresh( task ) @@ -951,15 +1014,15 @@ class TaskWrapper(JobWrapper): # DBTODO Check task job_runner_stuff self.sa_session.add( task ) self.sa_session.flush() - + def finish( self, stdout, stderr ): # DBTODO integrate previous finish logic. # Simple finish for tasks. Just set the flag OK. log.debug( 'task %s for job %d ended' % (self.task_id, self.job_id) ) """ - Called to indicate that the associated command has been run. Updates + Called to indicate that the associated command has been run. Updates the output datasets based on stderr and stdout from the command, and - the contents of the output files. + the contents of the output files. """ # default post job setup_external_metadata self.sa_session.expunge_all() @@ -976,7 +1039,7 @@ class TaskWrapper(JobWrapper): task.state = task.states.ERROR else: task.state = task.states.OK - # Save stdout and stderr + # Save stdout and stderr if len( stdout ) > 32768: log.error( "stdout for task %d is greater than 32K, only first part will be logged to database" % task.id ) task.stdout = stdout[:32768] @@ -990,7 +1053,7 @@ class TaskWrapper(JobWrapper): def cleanup( self ): # There is no task cleanup. The job cleans up for all tasks. pass - + def get_command_line( self ): return self.command_line @@ -1000,7 +1063,7 @@ class TaskWrapper(JobWrapper): def get_output_file_id( self, file ): # There is no permanent output file for tasks. return None - + def get_tool_provided_job_metadata( self ): # DBTODO Handle this as applicable for tasks. return None @@ -1013,16 +1076,15 @@ class TaskWrapper(JobWrapper): sizes = [] output_paths = self.get_output_fnames() for outfile in [ str( o ) for o in output_paths ]: - sizes.append( ( outfile, os.stat( outfile ).st_size ) ) + if os.path.exists( outfile ): + sizes.append( ( outfile, os.stat( outfile ).st_size ) ) + else: + sizes.append( ( outfile, 0 ) ) return sizes def setup_external_metadata( self, exec_dir = None, tmp_dir = None, dataset_files_path = None, config_root = None, datatypes_config = None, set_extension = True, **kwds ): # There is no metadata setting for tasks. This is handled after the merge, at the job level. return "" - - @property - def user( self ): - pass class DefaultJobDispatcher( object ): def __init__( self, app ): @@ -1053,13 +1115,19 @@ class DefaultJobDispatcher( object ): runner = getattr( module, obj ) self.job_runners[name] = runner( self.app ) log.debug( 'Loaded job runner: %s' % display_name ) - + def put( self, job_wrapper ): try: - if self.app.config.use_tasked_jobs and job_wrapper.tool.parallelism is not None and not isinstance(job_wrapper, TaskWrapper): - runner_name = "tasks" - log.debug( "dispatching job %d to %s runner" %( job_wrapper.job_id, runner_name ) ) - self.job_runners[runner_name].put( job_wrapper ) + if self.app.config.use_tasked_jobs and job_wrapper.tool.parallelism is not None: + if isinstance(job_wrapper, TaskWrapper): + #DBTODO Refactor + runner_name = ( job_wrapper.tool.job_runner.split(":", 1) )[0] + log.debug( "dispatching task %s, of job %d, to %s runner" %( job_wrapper.task_id, job_wrapper.job_id, runner_name ) ) + self.job_runners[runner_name].put( job_wrapper ) + else: + runner_name = "tasks" + log.debug( "dispatching job %d to %s runner" %( job_wrapper.job_id, runner_name ) ) + self.job_runners[runner_name].put( job_wrapper ) else: runner_name = ( job_wrapper.tool.job_runner.split(":", 1) )[0] log.debug( "dispatching job %d to %s runner" %( job_wrapper.job_id, runner_name ) ) @@ -1115,7 +1183,7 @@ class JobStopQueue( object ): self.sleeper = Sleeper() self.running = True self.monitor_thread = threading.Thread( target=self.monitor ) - self.monitor_thread.start() + self.monitor_thread.start() log.info( "job stopper started" ) def monitor( self ): diff --git a/lib/galaxy/jobs/actions/post.py b/lib/galaxy/jobs/actions/post.py index 1942b526699..934a9710c2c 100644 --- a/lib/galaxy/jobs/actions/post.py +++ b/lib/galaxy/jobs/actions/post.py @@ -1,5 +1,5 @@ -import logging, datetime, smtplib -from email.MIMEText import MIMEText +import logging, datetime +from galaxy.util import send_mail from galaxy.util.json import to_json_string log = logging.getLogger( __name__ ) @@ -10,16 +10,16 @@ def get_form_template(action_type, title, content, help, on_output = True ): if on_output: form = """ if (pja.action_type == "%s"){ - p_str = "
    %s
    on " + pja.output_name + "\ -
    "; + p_str = "
    %s
    on " + pja.output_name + "\ +
    "; %s p_str += "
    %s
    "; }""" % (action_type, title, content, help) else: - form = """ + form = """ if (pja.action_type == "%s"){ - p_str = "
    %s \ -
    "; + p_str = "
    %s \ +
    "; %s p_str += "
    %s
    "; }""" % (action_type, title, content, help) @@ -58,43 +58,25 @@ class EmailAction(DefaultJobAction): @classmethod def execute(cls, app, sa_session, action, job, replacement_dict): - smtp_server = app.config.smtp_server if action.action_arguments and action.action_arguments.has_key('host'): host = action.action_arguments['host'] else: host = 'usegalaxy.org' - if smtp_server is None: - log.error("Mail is not configured for this galaxy instance. Workflow action aborting after logging mail to info.") - frm = 'galaxy-noreply@%s' % host - to = job.user.email - outdata = ', '.join(ds.dataset.display_name() for ds in job.output_datasets) - msg = MIMEText( "Your Galaxy job generating dataset '%s' is complete as of %s." % (outdata, datetime.datetime.now().strftime( "%I:%M" ))) - msg[ 'To' ] = to - msg[ 'From' ] = frm - msg[ 'Subject' ] = "Galaxy notification regarding history '%s'" % (job.history.name) - log.info(msg) - return - # Build the email message frm = 'galaxy-noreply@%s' % host to = job.user.email + subject = "Galaxy workflow step notification '%s'" % (job.history.name) outdata = ', '.join(ds.dataset.display_name() for ds in job.output_datasets) - msg = MIMEText( "Your Galaxy job generating dataset '%s' is complete as of %s." % (outdata, datetime.datetime.now().strftime( "%I:%M" ))) - msg[ 'To' ] = to - msg[ 'From' ] = frm - msg[ 'Subject' ] = "Galaxy workflow step notification '%s'" % (job.history.name) + body = "Your Galaxy job generating dataset '%s' is complete as of %s." % (outdata, datetime.datetime.now().strftime( "%I:%M" )) try: - s = smtplib.SMTP() - s.connect( smtp_server ) - s.sendmail( frm, [ to ], msg.as_string() ) - s.close() + send_mail( frm, to, subject, body, app.config ) except Exception, e: log.error("EmailAction PJA Failed, exception: %s" % e) @classmethod def get_config_form(cls, trans): form = """ - p_str += "\ - "; + p_str += "\ + "; """ % trans.request.host return get_form_template(cls.name, cls.verbose_name, form, "This action will send an email notifying you when the job is done.", on_output = False) @@ -123,14 +105,14 @@ class ChangeDatatypeAction(DefaultJobAction): for dt_name in dtnames: dt_list += """""" % (dt_name, dt_name, dt_name) ps = """ - p_str += "\ - "; - if (pja.action_arguments !== undefined && pja.action_arguments.newtype !== undefined){ + p_str += "\ + "; + if (pja.action_arguments !== undefined && pja.action_arguments.newtype !== undefined){ p_str += "$('#pja__" + pja.output_name + "__ChangeDatatypeAction__newtype').val('" + pja.action_arguments.newtype + "');"; - } - """ % dt_list + } + """ % dt_list # Note the scrip + t hack above. Is there a better way? return get_form_template(cls.name, cls.verbose_name, ps, 'This action will change the datatype of the output to the indicated value.') @@ -158,15 +140,16 @@ class RenameDatasetAction(DefaultJobAction): @classmethod def get_config_form(cls, trans): form = """ - if ((pja.action_arguments !== undefined) && (pja.action_arguments.newname !== undefined)){ - p_str += "\ - "; - } - else{ - p_str += "\ - "; - } - """ + if ((pja.action_arguments !== undefined) && (pja.action_arguments.newname !== undefined)){ + p_str += "\ + "; + } + + else{ + p_str += "\ + "; + } + """ return get_form_template(cls.name, cls.verbose_name, form, "This action will rename the result dataset.") @classmethod @@ -213,8 +196,8 @@ class DeleteDatasetAction(DefaultJobAction): @classmethod def get_config_form(cls, trans): form = """ - p_str += "\ - "; + p_str += "\ + "; """ return get_form_template(cls.name, cls.verbose_name, form, "This action will rename the result dataset.") diff --git a/lib/galaxy/jobs/deferred/__init__.py b/lib/galaxy/jobs/deferred/__init__.py index d38a637692d..6ab341279e6 100644 --- a/lib/galaxy/jobs/deferred/__init__.py +++ b/lib/galaxy/jobs/deferred/__init__.py @@ -98,14 +98,14 @@ class DeferredJobQueue( object ): job_state = self.plugins[job.plugin].check_job( job ) except Exception, e: self.__fail_job( job ) - log.error( 'Set deferred job %s to error because of an exception in check_job(): %s' % ( job.id, str( e ) ) ) + log.exception( 'Set deferred job %s to error because of an exception in check_job(): %s' % ( job.id, str( e ) ) ) continue if job_state == self.job_states.READY: try: self.plugins[job.plugin].run_job( job ) except Exception, e: self.__fail_job( job ) - log.error( 'Set deferred job %s to error because of an exception in run_job(): %s' % ( job.id, str( e ) ) ) + log.exception( 'Set deferred job %s to error because of an exception in run_job(): %s' % ( job.id, str( e ) ) ) continue elif job_state == self.job_states.INVALID: self.__fail_job( job ) @@ -160,8 +160,14 @@ class FakeTrans( object ): self.app = app self.sa_session = app.model.context.current self.dummy = Dummy() - self.history = history - self.user = user + if not history: + self.history = Dummy() + else: + self.history = history + if not user: + self.user = Dummy() + else: + self.user = user self.model = app.model def get_galaxy_session( self ): return self.dummy diff --git a/lib/galaxy/jobs/deferred/data_transfer.py b/lib/galaxy/jobs/deferred/data_transfer.py index eaefaab76d7..6e9dcee3787 100644 --- a/lib/galaxy/jobs/deferred/data_transfer.py +++ b/lib/galaxy/jobs/deferred/data_transfer.py @@ -93,12 +93,18 @@ class DataTransfer( object ): # In this case, job.params will be a dictionary that contains a key named 'result'. The value # of the result key is a dictionary that looks something like: # {'sample_dataset_id': '8', 'status': 'Not started', 'protocol': 'scp', 'name': '3.bed', - # 'file_path': '/tmp/library/3.bed', 'host': '127.0.0.1', 'sample_id': 8, 'external_service_id': 2, - # 'password': 'galaxy', 'user_name': 'gvk', 'error_msg': '', 'size': '8.0K'} - result_dict = job.params[ 'result' ] + # 'file_path': '/data/library/3.bed', 'host': '127.0.0.1', 'sample_id': 8, 'external_service_id': 2, + # 'local_path': '/tmp/kjl2Ss4', 'password': 'galaxy', 'user_name': 'gvk', 'error_msg': '', 'size': '8.0K'} + try: + tj = self.sa_session.query( self.app.model.TransferJob ).get( int( job.params['transfer_job_id'] ) ) + result_dict = tj.params + result_dict['local_path'] = tj.path + except Exception, e: + log.error( "Updated transfer result unavailable, using old result. Error was: %s" % str( e ) ) + result_dict = job.params[ 'result' ] library_dataset_name = result_dict[ 'name' ] # Determine the data format (see the relevant TODO item in the manual_data_transfer plugin).. - extension = sniff.guess_ext( result_dict[ 'file_path' ], sniff_order=self.app.datatypes_registry.sniff_order ) + extension = sniff.guess_ext( result_dict[ 'local_path' ], sniff_order=self.app.datatypes_registry.sniff_order ) self._update_sample_dataset_status( protocol=job.params[ 'protocol' ], sample_id=int( job.params[ 'sample_id' ] ), result_dict=result_dict, @@ -108,7 +114,7 @@ class DataTransfer( object ): ld = self.app.model.LibraryDataset( folder=sample.folder, name=library_dataset_name ) self.sa_session.add( ld ) self.sa_session.flush() - self.app.security_agent.copy_library_permissions( sample.folder, ld ) + self.app.security_agent.copy_library_permissions( FakeTrans( self.app ), sample.folder, ld ) ldda = self.app.model.LibraryDatasetDatasetAssociation( name = library_dataset_name, extension = extension, dbkey = '?', @@ -132,7 +138,9 @@ class DataTransfer( object ): setattr( ldda.metadata, name, spec.unwrap( spec.get( 'default' ) ) ) if self.app.config.set_metadata_externally: self.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( self.app.datatypes_registry.set_external_metadata_tool, - FakeTrans( self.app ), + FakeTrans( self.app, + history=sample.history, + user=sample.request.user ), incoming = { 'input1':ldda } ) else: ldda.set_meta() diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py index 648062d71a8..597d517309f 100644 --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -1,10 +1,9 @@ import os, os.path class BaseJobRunner( object ): - def build_command_line( self, job_wrapper, include_metadata=False ): """ - Compose the sequence of commands neccesary to execute a job. This will + Compose the sequence of commands necessary to execute a job. This will currently include: - environment settings corresponding to any requirement tags - command line taken from job wrapper @@ -15,9 +14,13 @@ class BaseJobRunner( object ): # occur if not commands: return None + # Prepend version string + if job_wrapper.version_string_cmd: + commands = "%s &> %s; " % ( job_wrapper.version_string_cmd, job_wrapper.get_version_string_path() ) + commands # Prepend dependency injection if job_wrapper.dependency_shell_commands: commands = "; ".join( job_wrapper.dependency_shell_commands + [ commands ] ) + # Append metadata setting commands, we don't want to overwrite metadata # that was copied over in init_meta(), as per established behavior if include_metadata and self.app.config.set_metadata_externally: diff --git a/lib/galaxy/jobs/runners/drmaa.py b/lib/galaxy/jobs/runners/drmaa.py index 9a523aedf19..440851e74eb 100644 --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -156,10 +156,15 @@ class DRMAAJobRunner( BaseJobRunner ): jt.nativeSpecification = native_spec script = drm_template % (job_wrapper.galaxy_lib_dir, os.path.abspath( job_wrapper.working_directory ), command_line) - fh = file( jt.remoteCommand, "w" ) - fh.write( script ) - fh.close() - os.chmod( jt.remoteCommand, 0750 ) + try: + fh = file( jt.remoteCommand, "w" ) + fh.write( script ) + fh.close() + os.chmod( jt.remoteCommand, 0750 ) + except: + job_wrapper.fail( "failure preparing job script", exception=True ) + log.exception("failure running job %s" % job_wrapper.get_id_tag()) + return # job was deleted while we were preparing it if job_wrapper.get_state() == model.Job.states.DELETED: diff --git a/lib/galaxy/jobs/runners/pbs.py b/lib/galaxy/jobs/runners/pbs.py index fee798ed069..11c4925a2c5 100644 --- a/lib/galaxy/jobs/runners/pbs.py +++ b/lib/galaxy/jobs/runners/pbs.py @@ -17,7 +17,7 @@ The 'pbs' runner depends on 'pbs_python' which is not installed or not configured properly. Galaxy's "scramble" system should make this installation simple, please follow the instructions found at: - http://bitbucket.org/galaxy/galaxy-central/wiki/Config/Cluster + http://wiki.g2.bx.psu.edu/Admin/Config/Performance/Cluster Additional errors may follow: %s diff --git a/lib/galaxy/jobs/runners/sge.py b/lib/galaxy/jobs/runners/sge.py index 18b36fc8a41..c7b9f3f60ae 100644 --- a/lib/galaxy/jobs/runners/sge.py +++ b/lib/galaxy/jobs/runners/sge.py @@ -14,7 +14,7 @@ The 'sge' runner depends on 'DRMAA_python' which is not installed. Galaxy's "scramble" system should make this installation simple, please follow the instructions found at: - http://bitbucket.org/galaxy/galaxy-central/wiki/Config/Cluster + http://wiki.g2.bx.psu.edu/Admin/Config/Performance/Cluster Additional errors may follow: %s diff --git a/lib/galaxy/jobs/transfer_manager.py b/lib/galaxy/jobs/transfer_manager.py index 2b9ffc3796b..af5f71941a5 100644 --- a/lib/galaxy/jobs/transfer_manager.py +++ b/lib/galaxy/jobs/transfer_manager.py @@ -106,6 +106,7 @@ class TransferManager( object ): error = dict( code=256, message='Error connecting to transfer daemon', data=str( e ) ) rval.append( dict( transfer_job_id=tj.id, state=tj.state, error=error ) ) else: + self.sa_session.refresh( tj ) rval.append( dict( transfer_job_id=tj.id, state=tj.state ) ) for tj_state in rval: if tj_state['state'] in self.app.model.TransferJob.terminal_states: diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index b2be39f43e2..0b887c5137d 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4,7 +4,9 @@ Galaxy data model classes Naming: try to use class names that have a distinct plural form so that the relationship cardinalities are obvious (e.g. prefer Dataset to Data) """ - +import pkg_resources +pkg_resources.require( "simplejson" ) +import simplejson import galaxy.datatypes from galaxy.util.bunch import Bunch from galaxy import util @@ -15,7 +17,11 @@ from galaxy.util.hash_util import * from galaxy.web.form_builder import * from galaxy.model.item_attrs import UsesAnnotations, APIItem from sqlalchemy.orm import object_session -import os.path, os, errno, codecs, operator, smtplib, socket, pexpect, logging, time +from sqlalchemy.sql.expression import func +import os.path, os, errno, codecs, operator, socket, pexpect, logging, time, shutil + +if sys.version_info[:2] < ( 2, 5 ): + from sets import Set as set log = logging.getLogger( __name__ ) @@ -42,7 +48,7 @@ def set_datatypes_registry( d_registry ): class User( object, APIItem ): api_collection_visible_keys = ( 'id', 'email' ) - api_element_visible_keys = ( 'id', 'email', 'username' ) + api_element_visible_keys = ( 'id', 'email', 'username', 'total_disk_usage', 'nice_total_disk_usage' ) def __init__( self, email=None, password=None ): self.email = email self.password = password @@ -53,7 +59,7 @@ class User( object, APIItem ): # Relationships self.histories = [] self.credentials = [] - + def set_password_cleartext( self, cleartext ): """Set 'self.password' to the digest of 'cleartext'.""" self.password = new_secure_hash( text_type=cleartext ) @@ -67,7 +73,31 @@ class User( object, APIItem ): if role not in roles: roles.append( role ) return roles - + def get_disk_usage( self, nice_size=False ): + rval = 0 + if self.disk_usage is not None: + rval = self.disk_usage + if nice_size: + rval = galaxy.datatypes.data.nice_size( rval ) + return rval + def set_disk_usage( self, bytes ): + self.disk_usage = bytes + total_disk_usage = property( get_disk_usage, set_disk_usage ) + @property + def nice_total_disk_usage( self ): + return self.get_disk_usage( nice_size=True ) + def calculate_disk_usage( self ): + dataset_ids = [] + total = 0 + # this can be a huge number and can run out of memory, so we avoid the mappers + db_session = object_session( self ) + for history in db_session.query( History ).enable_eagerloads( False ).filter_by( user_id=self.id, purged=False ).yield_per( 1000 ): + for hda in db_session.query( HistoryDatasetAssociation ).enable_eagerloads( False ).filter_by( history_id=history.id, purged=False ).yield_per( 1000 ): + if not hda.dataset.id in dataset_ids and not hda.dataset.purged and not hda.dataset.library_associations: + dataset_ids.append( hda.dataset.id ) + total += hda.dataset.get_total_size() + return total + class Job( object ): """ A job represents a request to run a tool given input datasets, tool @@ -92,6 +122,7 @@ class Job( object ): self.parameters = [] self.input_datasets = [] self.output_datasets = [] + self.input_library_datasets = [] self.output_library_datasets = [] self.state = Job.states.NEW self.info = None @@ -99,13 +130,15 @@ class Job( object ): self.job_runner_external_id = None self.post_job_actions = [] self.imported = False - + def add_parameter( self, name, value ): self.parameters.append( JobParameter( name, value ) ) def add_input_dataset( self, name, dataset ): self.input_datasets.append( JobToInputDatasetAssociation( name, dataset ) ) def add_output_dataset( self, name, dataset ): self.output_datasets.append( JobToOutputDatasetAssociation( name, dataset ) ) + def add_input_library_dataset( self, name, dataset ): + self.input_library_datasets.append( JobToInputLibraryDatasetAssociation( name, dataset ) ) def add_output_library_dataset( self, name, dataset ): self.output_library_datasets.append( JobToOutputLibraryDatasetAssociation( name, dataset ) ) def add_post_job_action(self, pja): @@ -212,17 +245,22 @@ class JobParameter( object ): def __init__( self, name, value ): self.name = name self.value = value - + class JobToInputDatasetAssociation( object ): def __init__( self, name, dataset ): self.name = name self.dataset = dataset - + class JobToOutputDatasetAssociation( object ): def __init__( self, name, dataset ): self.name = name self.dataset = dataset +class JobToInputLibraryDatasetAssociation( object ): + def __init__( self, name, dataset ): + self.name = name + self.dataset = dataset + class JobToOutputLibraryDatasetAssociation( object ): def __init__( self, name, dataset ): self.name = name @@ -234,7 +272,7 @@ class PostJobAction( object ): self.output_name = output_name self.action_arguments = action_arguments self.workflow_step = workflow_step - + class PostJobActionAssociation( object ): def __init__(self, pja, job): self.job = job @@ -254,7 +292,7 @@ class JobExternalOutputMetadata( object ): elif self.library_dataset_dataset_association: return self.library_dataset_dataset_association return None - + class JobExportHistoryArchive( object ): def __init__( self, job=None, history=None, dataset=None, compressed=False, \ history_attrs_filename=None, datasets_attrs_filename=None, @@ -266,7 +304,7 @@ class JobExportHistoryArchive( object ): self.history_attrs_filename = history_attrs_filename self.datasets_attrs_filename = datasets_attrs_filename self.jobs_attrs_filename = jobs_attrs_filename - + class JobImportHistoryArchive( object ): def __init__( self, job=None, history=None, archive_dir=None ): self.job = job @@ -309,7 +347,7 @@ class DeferredJob( object ): return True else: return False - + class Group( object ): def __init__( self, name = None ): self.name = name @@ -321,6 +359,8 @@ class UserGroupAssociation( object ): self.group = group class History( object, UsesAnnotations ): + api_collection_visible_keys = ( 'id', 'name' ) + api_element_visible_keys = ( 'id', 'name' ) def __init__( self, id=None, name=None, user=None ): self.id = id self.name = name or "Unnamed history" @@ -349,7 +389,7 @@ class History( object, UsesAnnotations ): self.galaxy_sessions.append( GalaxySessionToHistoryAssociation( galaxy_session, self ) ) else: self.galaxy_sessions.append( association ) - def add_dataset( self, dataset, parent_id=None, genome_build=None, set_hid = True ): + def add_dataset( self, dataset, parent_id=None, genome_build=None, set_hid=True, quota=True ): if isinstance( dataset, Dataset ): dataset = HistoryDatasetAssociation(dataset=dataset) object_session( self ).add( dataset ) @@ -367,6 +407,8 @@ class History( object, UsesAnnotations ): else: if set_hid: dataset.hid = self._next_hid() + if quota and self.user: + self.user.total_disk_usage += dataset.quota_amount( self.user ) dataset.history = self if genome_build not in [None, '?']: self.genome_build = genome_build @@ -378,11 +420,14 @@ class History( object, UsesAnnotations ): name = self.name if not target_user: target_user = self.user + quota = True + if target_user == self.user: + quota = False new_history = History( name=name, user=target_user ) db_session = object_session( self ) db_session.add( new_history ) db_session.flush() - + # Copy annotation. self.copy_item_annotation( db_session, self.user, self, target_user, new_history ) @@ -393,10 +438,10 @@ class History( object, UsesAnnotations ): hdas = self.active_datasets for hda in hdas: # Copy HDA. - new_hda = hda.copy( copy_children=True, target_history=new_history ) - new_history.add_dataset( new_hda, set_hid = False ) + new_hda = hda.copy( copy_children=True ) + new_history.add_dataset( new_hda, set_hid = False, quota=quota ) db_session.add( new_hda ) - db_session.flush() + db_session.flush() # Copy annotation. self.copy_item_annotation( db_session, self.user, hda, target_user, new_hda ) new_history.hid_counter = self.hid_counter @@ -414,6 +459,37 @@ class History( object, UsesAnnotations ): history_name = unicode(history_name, 'utf-8') return history_name + def get_api_value( self, view='collection', value_mapper = None ): + if value_mapper is None: + value_mapper = {} + rval = {} + try: + visible_keys = self.__getattribute__( 'api_' + view + '_visible_keys' ) + except AttributeError: + raise Exception( 'Unknown API view: %s' % view ) + for key in visible_keys: + try: + rval[key] = self.__getattribute__( key ) + if key in value_mapper: + rval[key] = value_mapper.get( key )( rval[key] ) + except AttributeError: + rval[key] = None + return rval + @property + def get_disk_size_bytes( self ): + return self.get_disk_size( nice_size=False ) + def get_disk_size( self, nice_size=False ): + # unique datasets only + db_session = object_session( self ) + rval = db_session.query( func.sum( db_session.query( HistoryDatasetAssociation.dataset_id, Dataset.total_size ).join( Dataset ) + .filter( HistoryDatasetAssociation.table.c.history_id == self.id ) + .distinct().subquery().c.total_size ) ).first()[0] + if rval is None: + rval = 0 + if nice_size: + rval = galaxy.datatypes.data.nice_size( rval ) + return rval + class HistoryUserShareAssociation( object ): def __init__( self ): self.history = None @@ -446,6 +522,58 @@ class Role( object, APIItem ): self.type = type self.deleted = deleted +class UserQuotaAssociation( object, APIItem ): + api_element_visible_keys = ( 'user', ) + def __init__( self, user, quota ): + self.user = user + self.quota = quota + +class GroupQuotaAssociation( object, APIItem ): + api_element_visible_keys = ( 'group', ) + def __init__( self, group, quota ): + self.group = group + self.quota = quota + +class Quota( object, APIItem ): + api_collection_visible_keys = ( 'id', 'name' ) + api_element_visible_keys = ( 'id', 'name', 'description', 'bytes', 'operation', 'display_amount', 'default', 'users', 'groups' ) + valid_operations = ( '+', '-', '=' ) + def __init__( self, name="", description="", amount=0, operation="=" ): + self.name = name + self.description = description + if amount is None: + self.bytes = -1 + else: + self.bytes = amount + self.operation = operation + def get_amount( self ): + if self.bytes == -1: + return None + return self.bytes + def set_amount( self, amount ): + if amount is None: + self.bytes = -1 + else: + self.bytes = amount + amount = property( get_amount, set_amount ) + @property + def display_amount( self ): + if self.bytes == -1: + return "unlimited" + else: + return util.nice_size( self.bytes ) + +class DefaultQuotaAssociation( Quota, APIItem ): + api_element_visible_keys = ( 'type', ) + types = Bunch( + UNREGISTERED = 'unregistered', + REGISTERED = 'registered' + ) + def __init__( self, type, quota ): + assert type in self.types.__dict__.values(), 'Invalid type' + self.type = type + self.quota = quota + class DatasetPermissions( object ): def __init__( self, action, dataset, role ): self.action = action @@ -549,7 +677,7 @@ class Dataset( object ): file_name = property( get_file_name, set_file_name ) @property def extra_files_path( self ): - if self._extra_files_path: + if self._extra_files_path: path = self._extra_files_path else: path = os.path.join( self.file_path, "dataset_%d_files" % self.id ) @@ -580,6 +708,22 @@ class Dataset( object ): self.file_size = os.path.getsize( self.file_name ) except OSError: self.file_size = 0 + def get_total_size( self ): + if self.total_size is not None: + return self.total_size + if self.file_size: + # for backwards compatibility, set if unset + self.set_total_size() + db_session = object_session( self ) + db_session.flush() + return self.total_size + return 0 + def set_total_size( self ): + if self.file_size is None: + self.set_size() + self.total_size = self.file_size or 0 + for root, dirs, files in os.walk( self.extra_files_path ): + self.total_size += sum( [ os.path.getsize( os.path.join( root, file ) ) for file in files ] ) def has_data( self ): """Detects whether there is any data""" return self.get_size() > 0 @@ -599,18 +743,42 @@ class Dataset( object ): os.remove(self.data.file_name) except OSError, e: log.critical('%s delete error %s' % (self.__class__.__name__, e)) + @property + def user_can_purge( self ): + return self.purged == False \ + and not bool( self.library_associations ) \ + and len( self.history_associations ) == len( self.purged_history_associations ) + def full_delete( self ): + """Remove the file and extra files, marks deleted and purged""" + os.unlink( self.file_name ) + if os.path.exists( self.extra_files_path ): + shutil.rmtree( self.extra_files_path ) + # TODO: purge metadata files + self.deleted = True + self.purged = True def get_access_roles( self, trans ): roles = [] for dp in self.actions: if dp.action == trans.app.security_agent.permitted_actions.DATASET_ACCESS.action: roles.append( dp.role ) return roles + def get_manage_permissions_roles( self, trans ): + roles = [] + for dp in self.actions: + if dp.action == trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS.action: + roles.append( dp.role ) + return roles + def has_manage_permissions_roles( self, trans ): + for dp in self.actions: + if dp.action == trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS.action: + return True + return False class DatasetInstance( object ): """A base class for all 'dataset instances', HDAs, LDAs, etc""" states = Dataset.states permitted_actions = Dataset.permitted_actions - def __init__( self, id=None, hid=None, name=None, info=None, blurb=None, peek=None, extension=None, + def __init__( self, id=None, hid=None, name=None, info=None, blurb=None, peek=None, tool_version=None, extension=None, dbkey=None, metadata=None, history=None, dataset=None, deleted=False, designation=None, parent_id=None, validation_errors=None, visible=True, create_dataset=False, sa_session=None ): self.name = name or "Unnamed dataset" @@ -618,6 +786,7 @@ class DatasetInstance( object ): self.info = info self.blurb = blurb self.peek = peek + self.tool_version = tool_version self.extension = extension self.designation = designation self.metadata = metadata or dict() @@ -676,9 +845,9 @@ class DatasetInstance( object ): return dbkey[0] def set_dbkey( self, value ): if "dbkey" in self.datatype.metadata_spec: - if not isinstance(value, list): + if not isinstance(value, list): self.metadata.dbkey = [value] - else: + else: self.metadata.dbkey = value dbkey = property( get_dbkey, set_dbkey ) def change_datatype( self, new_ext ): @@ -692,6 +861,10 @@ class DatasetInstance( object ): def set_size( self ): """Returns the size of the data on disk""" return self.dataset.set_size() + def get_total_size( self ): + return self.dataset.get_total_size() + def set_total_size( self ): + return self.dataset.set_total_size() def has_data( self ): """Detects whether there is any data""" return self.dataset.has_data() @@ -706,7 +879,11 @@ class DatasetInstance( object ): self.datatype.set_raw_data(self, data) def get_mime( self ): """Returns the mime type of the data""" - return datatypes_registry.get_mimetype_by_extension( self.extension.lower() ) + try: + return datatypes_registry.get_mimetype_by_extension( self.extension.lower() ) + except AttributeError: + # extension is None + return 'data' def is_multi_byte( self ): """Data consists of multi-byte characters""" return self.dataset.is_multi_byte() @@ -751,22 +928,18 @@ class DatasetInstance( object ): # See if we can convert the dataset if target_ext not in self.get_converter_types(): raise NoConverterException("Conversion from '%s' to '%s' not possible" % (self.extension, target_ext) ) - deps = {} # List of string of dependencies try: depends_list = trans.app.datatypes_registry.converter_deps[self.extension][target_ext] except KeyError: depends_list = [] - # See if converted dataset already exists converted_dataset = self.get_converted_files_by_type( target_ext ) if converted_dataset: return converted_dataset - # Conversion is possible but hasn't been done yet, run converter. # Check if we have dependencies - try: for dependency in depends_list: dep_dataset = self.get_converted_dataset(trans, dependency) @@ -777,14 +950,12 @@ class DatasetInstance( object ): raise ConverterDependencyException("A dependency (%s) was in an error state." % dependency) elif dep_dataset.state != trans.app.model.Job.states.OK: # Pending - return None - + return None deps[dependency] = dep_dataset except NoConverterException: raise NoConverterException("A dependency (%s) is missing a converter." % dependency) except KeyError: pass # No deps - assoc = ImplicitlyConvertedDatasetAssociation( parent=self, file_type=target_ext, metadata_safe=False ) new_dataset = self.datatype.convert_dataset( trans, self, target_ext, return_output=True, visible=False, deps=deps, set_output_history=False ).values()[0] new_dataset.name = self.name @@ -873,20 +1044,20 @@ class HistoryDatasetAssociation( DatasetInstance ): self.history = history self.copied_from_history_dataset_association = copied_from_history_dataset_association self.copied_from_library_dataset_dataset_association = copied_from_library_dataset_dataset_association - def copy( self, copy_children = False, parent_id = None, target_history = None ): + def copy( self, copy_children = False, parent_id = None ): hda = HistoryDatasetAssociation( hid=self.hid, name=self.name, info=self.info, blurb=self.blurb, peek=self.peek, + tool_version=self.tool_version, extension=self.extension, dbkey=self.dbkey, dataset = self.dataset, visible=self.visible, deleted=self.deleted, parent_id=parent_id, - copied_from_history_dataset_association=self, - history = target_history ) + copied_from_history_dataset_association=self ) object_session( self ).add( hda ) object_session( self ).flush() hda.set_size() @@ -917,6 +1088,7 @@ class HistoryDatasetAssociation( DatasetInstance ): info=self.info, blurb=self.blurb, peek=self.peek, + tool_version=self.tool_version, extension=self.extension, dbkey=self.dbkey, dataset=self.dataset, @@ -968,6 +1140,50 @@ class HistoryDatasetAssociation( DatasetInstance ): return hda_name def get_access_roles( self, trans ): return self.dataset.get_access_roles( trans ) + def quota_amount( self, user ): + """ + If the user has multiple instances of this dataset, it will not affect their disk usage statistic. + """ + rval = 0 + # Anon users are handled just by their single history size. + if not user: + return rval + # Gets an HDA and its children's disk usage, if the user does not already have an association of the same dataset + if not self.dataset.library_associations and not self.purged and not self.dataset.purged: + for hda in self.dataset.history_associations: + if hda.id == self.id: + continue + if not hda.purged and hda.history and hda.history.user and hda.history.user == user: + break + else: + rval += self.get_total_size() + for child in self.children: + rval += child.get_disk_usage( user ) + return rval + def get_api_value( self, view='collection' ): + # Since this class is a proxy to rather complex attributes we want to + # display in other objects, we can't use the simpler method used by + # other model classes. + hda = self + rval = dict( id = hda.id, + model_class = self.__class__.__name__, + name = hda.name, + deleted = hda.deleted, + visible = hda.visible, + state = hda.state, + file_size = int( hda.get_size() ), + data_type = hda.ext, + genome_build = hda.dbkey, + misc_info = hda.info, + misc_blurb = hda.blurb ) + for name, spec in hda.metadata.spec.items(): + val = hda.metadata.get( name ) + if isinstance( val, MetadataFile ): + val = val.file_name + elif isinstance( val, list ): + val = ', '.join( [str(v) for v in val] ) + rval['metadata_' + name] = val + return rval class HistoryDatasetAssociationDisplayAtAuthorization( object ): def __init__( self, hda=None, user=None, site=None ): @@ -1045,7 +1261,7 @@ class Library( object, APIItem ): return name class LibraryFolder( object, APIItem ): - api_element_visible_keys = ( 'name', 'description', 'item_count', 'genome_build' ) + api_element_visible_keys = ( 'id', 'name', 'description', 'item_count', 'genome_build' ) def __init__( self, name=None, description=None, item_count=0, order_id=None ): self.name = name or "Unnamed folder" self.description = description @@ -1117,7 +1333,7 @@ class LibraryFolder( object, APIItem ): name = unicode( name, 'utf-8' ) return name def get_api_value( self, view='collection' ): - rval = super( APIItem, self ).get_api_value( vew=view ) + rval = super( LibraryFolder, self ).get_api_value( view=view ) info_association, inherited = self.get_info_association() if info_association: if inherited: @@ -1185,8 +1401,11 @@ class LibraryDataset( object ): for field in template.fields: tmp_dict[field['label']] = content[field['name']] template_data[template.name] = tmp_dict - - rval = dict( name = ldda.name, + + rval = dict( id = self.id, + ldda_id = ldda.id, + model_class = self.__class__.__name__, + name = ldda.name, file_name = ldda.file_name, uploaded_by = ldda.user.email, message = ldda.message, @@ -1228,6 +1447,7 @@ class LibraryDatasetDatasetAssociation( DatasetInstance ): info=self.info, blurb=self.blurb, peek=self.peek, + tool_version=self.tool_version, extension=self.extension, dbkey=self.dbkey, dataset=self.dataset, @@ -1252,6 +1472,7 @@ class LibraryDatasetDatasetAssociation( DatasetInstance ): info=self.info, blurb=self.blurb, peek=self.peek, + tool_version=self.tool_version, extension=self.extension, dbkey=self.dbkey, dataset=self.dataset, @@ -1276,6 +1497,10 @@ class LibraryDatasetDatasetAssociation( DatasetInstance ): return def get_access_roles( self, trans ): return self.dataset.get_access_roles( trans ) + def get_manage_permissions_roles( self, trans ): + return self.dataset.get_manage_permissions_roles( trans ) + def has_manage_permissions_roles( self, trans ): + return self.dataset.has_manage_permissions_roles( trans ) def get_info_association( self, restrict=False, inherited=False ): # If restrict is True, we will return this ldda's info_association whether it # exists or not ( in which case None will be returned ). If restrict is False, @@ -1313,6 +1538,22 @@ class LibraryDatasetDatasetAssociation( DatasetInstance ): else: return template.get_widgets( trans.user ) return [] + def templates_dict( self ): + """ + Returns a dict of template info + """ + template_data = {} + for temp_info in self.info_association: + template = temp_info.template + content = temp_info.info.content + tmp_dict = {} + for field in template.fields: + tmp_dict[field['label']] = content[field['name']] + template_data[template.name] = tmp_dict + return template_data + def templates_json( self ): + return simplejson.dumps( self.templates_dict() ) + def get_display_name( self ): """ LibraryDatasetDatasetAssociation name can be either a string or a unicode object. @@ -1364,7 +1605,7 @@ class ImplicitlyConvertedDatasetAssociation( object ): elif isinstance(parent, LibraryDatasetDatasetAssociation): self.parent_ldda = parent else: - raise AttributeError + raise AttributeError, 'Unknown dataset type provided for parent: %s' % type( parent ) self.type = file_type self.deleted = deleted self.purged = purged @@ -1414,7 +1655,14 @@ class GalaxySession( object ): self.histories.append( GalaxySessionToHistoryAssociation( self, history ) ) else: self.histories.append( association ) - + def get_disk_usage( self ): + if self.disk_usage is None: + return 0 + return self.disk_usage + def set_disk_usage( self, bytes ): + self.disk_usage = bytes + total_disk_usage = property( get_disk_usage, set_disk_usage ) + class GalaxySessionToHistoryAssociation( object ): def __init__( self, galaxy_session, history ): self.galaxy_session = galaxy_session @@ -1425,7 +1673,7 @@ class CloudImage( object ): self.id = None self.instance_id = None self.state = None - + class UCI( object ): def __init__( self ): self.id = None @@ -1441,7 +1689,7 @@ class CloudInstance( object ): self.state = None self.public_dns = None self.availability_zone = None - + class CloudStore( object ): def __init__( self ): self.id = None @@ -1449,14 +1697,14 @@ class CloudStore( object ): self.user = None self.size = None self.availability_zone = None - + class CloudSnapshot( object ): def __init__( self ): self.id = None self.user = None self.store_id = None self.snapshot_id = None - + class CloudProvider( object ): def __init__( self ): self.id = None @@ -1491,7 +1739,7 @@ class Workflow( object ): self.has_cycles = None self.has_errors = None self.steps = [] - + class WorkflowStep( object ): def __init__( self ): self.id = None @@ -1502,7 +1750,7 @@ class WorkflowStep( object ): self.position = None self.input_connections = [] self.config = None - + class WorkflowStepConnection( object ): def __init__( self ): self.output_step_id = None @@ -1514,7 +1762,7 @@ class WorkflowOutput(object): def __init__( self, workflow_step, output_name): self.workflow_step = workflow_step self.output_name = output_name - + class StoredWorkflowUserShareAssociation( object ): def __init__( self ): self.stored_workflow = None @@ -1675,12 +1923,12 @@ class FormDefinition( object, APIItem ): class FormDefinitionCurrent( object ): def __init__(self, form_definition=None): self.latest_form = form_definition - + class FormValues( object ): def __init__(self, form_def=None, content=None): self.form_definition = form_def self.content = content - + class Request( object, APIItem ): states = Bunch( NEW = 'New', SUBMITTED = 'In Progress', @@ -1764,7 +2012,7 @@ class Request( object, APIItem ): for sample in self.samples: if sample.bar_code: samples.append( sample ) - return samples + return samples def send_email_notification( self, trans, common_state, final_state=False ): # Check if an email notification is configured to be sent when the samples # are in this state @@ -1814,12 +2062,8 @@ All samples in state: %(sample_state)s to = self.notification['email'] frm = 'galaxy-no-reply@' + host subject = "Galaxy Sample Tracking notification: '%s' sequencing request" % self.name - message = "From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n%s" % ( frm, ", ".join( to ), subject, body ) try: - s = smtplib.SMTP() - s.connect( trans.app.config.smtp_server ) - s.sendmail( frm, to, message ) - s.quit() + util.send_mail( frm, to, subject, body, trans.app.config ) comments = "Email notification sent to %s." % ", ".join( to ).strip().strip( ',' ) except Exception,e: comments = "Email notification failed. (%s)" % str(e) @@ -1831,13 +2075,13 @@ All samples in state: %(sample_state)s trans.sa_session.add( event ) trans.sa_session.flush() return comments - + class RequestEvent( object ): def __init__(self, request=None, request_state=None, comment=''): self.request = request self.state = request_state self.comment = comment - + class ExternalService( object ): data_transfer_protocol = Bunch( HTTP = 'http', HTTPS = 'https', @@ -1875,7 +2119,7 @@ class ExternalService( object ): self.data_transfer[ self.data_transfer_protocol.HTTP ] = http_configs def populate_actions( self, trans, item, param_dict=None ): return self.get_external_service_type( trans ).actions.populate( self, item, param_dict=param_dict ) - + class RequestType( object, APIItem ): api_collection_visible_keys = ( 'id', 'name', 'desc' ) api_element_visible_keys = ( 'id', 'name', 'desc', 'request_form_id', 'sample_form_id' ) @@ -1950,18 +2194,18 @@ class RequestType( object, APIItem ): return template.get_widgets( trans.user, contents=info.content ) return template.get_widgets( trans.user ) return [] - + class RequestTypeExternalServiceAssociation( object ): def __init__( self, request_type, external_service ): self.request_type = request_type self.external_service = external_service - + class RequestTypePermissions( object ): def __init__( self, action, request_type, role ): self.action = action self.request_type = request_type self.role = role - + class Sample( object, APIItem ): # The following form_builder classes are supported by the Sample class. supported_field_types = [ CheckboxField, SelectField, TextField, WorkflowField, WorkflowMappingField, HistoryField ] @@ -2042,14 +2286,14 @@ class Sample( object, APIItem ): def print_ticks( d ): pass error_msg = 'Error encountered in determining the file size of %s on the external_service.' % filepath - if not scp_configs[ 'host' ] or not scp_configs[ 'user_name' ] or not scp_configs[ 'password' ]: + if not scp_configs['host'] or not scp_configs['user_name'] or not scp_configs['password']: return error_msg login_str = '%s@%s' % ( scp_configs['user_name'], scp_configs['host'] ) cmd = 'ssh %s "du -sh \'%s\'"' % ( login_str, filepath ) try: output = pexpect.run( cmd, - events={ '.ssword:*' : scp_configs['password'] + '\r\n', - pexpect.TIMEOUT : print_ticks }, + events={ '.ssword:*': scp_configs['password']+'\r\n', + pexpect.TIMEOUT:print_ticks}, timeout=10 ) except Exception, e: return error_msg @@ -2107,7 +2351,7 @@ class SampleEvent( object ): self.sample = sample self.state = sample_state self.comment = comment - + class SampleDataset( object ): transfer_status = Bunch( NOT_STARTED = 'Not started', IN_QUEUE = 'In queue', @@ -2196,33 +2440,72 @@ class PageRevision( object ): self.user = None self.title = None self.content = None - + class PageUserShareAssociation( object ): def __init__( self ): self.page = None self.user = None class Visualization( object ): - def __init__( self ): + def __init__( self, user=None, type=None, title=None, dbkey=None, latest_revision=None ): self.id = None - self.user = None - self.type = None - self.title = None - self.latest_revision = None + self.user = user + self.type = type + self.title = title + self.dbkey = dbkey + self.latest_revision = latest_revision self.revisions = [] + if self.latest_revision: + self.revisions.append( latest_revision ) + + def copy( self, user=None, title=None ): + """ + Provide copy of visualization with only its latest revision. + """ + # NOTE: a shallow copy is done: the config is copied as is but datasets + # are not copied nor are the dataset ids changed. This means that the + # user does not have a copy of the data in his/her history and the + # user who owns the datasets may delete them, making them inaccessible + # for the current user. + # TODO: a deep copy option is needed. + + if not user: + user = self.user + if not title: + title = self.title + + copy_viz = Visualization( user=user, type=self.type, title=title, dbkey=self.dbkey ) + copy_revision = self.latest_revision.copy( visualization=copy_viz ) + copy_viz.latest_revision = copy_revision + return copy_viz class VisualizationRevision( object ): - def __init__( self ): + def __init__( self, visualization=None, title=None, dbkey=None, config=None ): self.id = None - self.visualization = None - self.title = None - self.config = None - + self.visualization = visualization + self.title = title + self.dbkey = dbkey + self.config = config + + def copy( self, visualization=None ): + """ + Returns a copy of this object. + """ + if not visualization: + visualization = self.visualization + + return VisualizationRevision( + visualization=visualization, + title=self.title, + dbkey=self.dbkey, + config=self.config + ) + class VisualizationUserShareAssociation( object ): def __init__( self ): self.visualization = None self.user = None - + class TransferJob( object ): # These states are used both by the transfer manager's IPC and the object # state in the database. Not all states are used by both. @@ -2248,10 +2531,10 @@ class Tag ( object ): self.type = type self.parent_id = parent_id self.name = name - + def __str__ ( self ): return "Tag(id=%s, type=%i, parent_id=%s, name=%s)" % ( self.id, self.type, self.parent_id, self.name ) - + class ItemTagAssociation ( object ): def __init__( self, id=None, user=None, item_id=None, tag_id=None, user_tname=None, value=None ): self.id = id @@ -2261,13 +2544,13 @@ class ItemTagAssociation ( object ): self.user_tname = user_tname self.value = None self.user_value = None - + class HistoryTagAssociation ( ItemTagAssociation ): pass - + class DatasetTagAssociation ( ItemTagAssociation ): pass - + class HistoryDatasetAssociationTagAssociation ( ItemTagAssociation ): pass @@ -2276,13 +2559,13 @@ class PageTagAssociation ( ItemTagAssociation ): class WorkflowStepTagAssociation ( ItemTagAssociation ): pass - + class StoredWorkflowTagAssociation ( ItemTagAssociation ): pass - + class VisualizationTagAssociation ( ItemTagAssociation ): pass - + class ToolTagAssociation( ItemTagAssociation ): def __init__( self, id=None, user=None, tool_id=None, tag_id=None, user_tname=None, value=None ): self.id = id @@ -2294,50 +2577,50 @@ class ToolTagAssociation( ItemTagAssociation ): self.user_value = None # Item annotation classes. - + class HistoryAnnotationAssociation( object ): pass - + class HistoryDatasetAssociationAnnotationAssociation( object ): pass - + class StoredWorkflowAnnotationAssociation( object ): pass - + class WorkflowStepAnnotationAssociation( object ): pass - + class PageAnnotationAssociation( object ): pass - + class VisualizationAnnotationAssociation( object ): pass - + # Item rating classes. - + class ItemRatingAssociation( object ): def __init__( self, id=None, user=None, item=None, rating=0 ): self.id = id self.user = user self.item = item self.rating = rating - + def set_item( self, item ): """ Set association's item. """ pass - + class HistoryRatingAssociation( ItemRatingAssociation ): def set_item( self, history ): self.history = history - + class HistoryDatasetAssociationRatingAssociation( ItemRatingAssociation ): def set_item( self, history_dataset_association ): self.history_dataset_association = history_dataset_association - + class StoredWorkflowRatingAssociation( ItemRatingAssociation ): def set_item( self, stored_workflow ): self.stored_workflow = stored_workflow - + class PageRatingAssociation( ItemRatingAssociation ): def set_item( self, page ): self.page = page @@ -2345,12 +2628,12 @@ class PageRatingAssociation( ItemRatingAssociation ): class VisualizationRatingAssociation( ItemRatingAssociation ): def set_item( self, visualization ): self.visualization = visualization - + class UserPreference ( object ): def __init__( self, name=None, value=None ): self.name = name self.value = value - + class UserAction( object ): def __init__( self, id=None, create_time=None, user_id=None, session_id=None, action=None, params=None, context=None): self.id = id @@ -2364,6 +2647,17 @@ class UserAction( object ): class APIKeys( object ): pass +class ToolShedRepository( object ): + def __init__( self, id=None, create_time=None, tool_shed=None, name=None, description=None, owner=None, changeset_revision=None, deleted=False ): + self.id = id + self.create_time = create_time + self.tool_shed = tool_shed + self.name = name + self.description = description + self.owner = owner + self.changeset_revision = changeset_revision + self.deleted = deleted + ## ---- Utility methods ------------------------------------------------------- def directory_hash_id( id ): diff --git a/lib/galaxy/model/custom_types.py b/lib/galaxy/model/custom_types.py index c87c99f7a7e..3ff071e77af 100644 --- a/lib/galaxy/model/custom_types.py +++ b/lib/galaxy/model/custom_types.py @@ -8,6 +8,11 @@ import binascii from galaxy.util.bunch import Bunch from galaxy.util.aliaspickler import AliasPickleModule +# For monkeypatching BIGINT +import sqlalchemy.databases.sqlite +import sqlalchemy.databases.postgres +import sqlalchemy.databases.mysql + import logging log = logging.getLogger( __name__ ) @@ -87,3 +92,25 @@ class TrimmedString( TypeDecorator ): value = value[0:self.impl.length] return value + +class BigInteger( Integer ): + """ + A type for bigger ``int`` integers. + + Typically generates a ``BIGINT`` in DDL, and otherwise acts like + a normal :class:`Integer` on the Python side. + + """ + +class BIGINT( BigInteger ): + """The SQL BIGINT type.""" + +class SLBigInteger( BigInteger ): + def get_col_spec( self ): + return "BIGINT" + +sqlalchemy.databases.sqlite.SLBigInteger = SLBigInteger +sqlalchemy.databases.sqlite.colspecs[BigInteger] = SLBigInteger +sqlalchemy.databases.sqlite.ischema_names['BIGINT'] = SLBigInteger +sqlalchemy.databases.postgres.colspecs[BigInteger] = sqlalchemy.databases.postgres.PGBigInteger +sqlalchemy.databases.mysql.colspecs[BigInteger] = sqlalchemy.databases.mysql.MSBigInteger diff --git a/lib/galaxy/model/item_attrs.py b/lib/galaxy/model/item_attrs.py index 8bd4211354c..5bd33bd3640 100644 --- a/lib/galaxy/model/item_attrs.py +++ b/lib/galaxy/model/item_attrs.py @@ -1,4 +1,5 @@ from sqlalchemy.sql.expression import func +from sqlalchemy.orm.collections import InstrumentedList # Cannot import galaxy.model b/c it creates a circular import graph. import galaxy import logging @@ -156,6 +157,13 @@ class APIItem: #api_collection_visible_keys = ( 'id' ) #api_element_visible_keys = ( 'id' ) def get_api_value( self, view='collection', value_mapper = None ): + def get_value( key, item ): + try: + return item.get_api_value( view=view, value_mapper=value_mapper ) + except: + if key in value_mapper: + return value_mapper.get( key )( item ) + return item if value_mapper is None: value_mapper = {} rval = {} @@ -165,9 +173,13 @@ class APIItem: raise Exception( 'Unknown API view: %s' % view ) for key in visible_keys: try: - rval[key] = self.__getattribute__( key ) - if key in value_mapper: - rval[key] = value_mapper.get( key )( rval[key] ) + item = self.__getattribute__( key ) + if type( item ) == InstrumentedList: + rval[key] = [] + for i in item: + rval[key].append( get_value( key, i ) ) + else: + rval[key] = get_value( key, item ) except AttributeError: rval[key] = None return rval diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 53adf5164ab..f013186a978 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -49,7 +49,8 @@ User.table = Table( "galaxy_user", metadata, Column( "external", Boolean, default=False ), Column( "form_values_id", Integer, ForeignKey( "form_values.id" ), index=True ), Column( "deleted", Boolean, index=True, default=False ), - Column( "purged", Boolean, index=True, default=False ) ) + Column( "purged", Boolean, index=True, default=False ), + Column( "disk_usage", Numeric( 15, 0 ), index=True ) ) UserAddress.table = Table( "user_address", metadata, Column( "id", Integer, primary_key=True), @@ -112,11 +113,13 @@ HistoryDatasetAssociation.table = Table( "history_dataset_association", metadata Column( "info", TrimmedString( 255 ) ), Column( "blurb", TrimmedString( 255 ) ), Column( "peek" , TEXT ), + Column( "tool_version" , TEXT ), Column( "extension", TrimmedString( 64 ) ), Column( "metadata", MetadataType(), key="_metadata" ), Column( "parent_id", Integer, ForeignKey( "history_dataset_association.id" ), nullable=True ), Column( "designation", TrimmedString( 255 ) ), Column( "deleted", Boolean, index=True, default=False ), + Column( "purged", Boolean, index=True, default=False ), Column( "visible", Boolean ) ) Dataset.table = Table( "dataset", metadata, @@ -129,7 +132,8 @@ Dataset.table = Table( "dataset", metadata, Column( "purgable", Boolean, default=True ), Column( "external_filename" , TEXT ), Column( "_extra_files_path", TEXT ), - Column( 'file_size', Numeric( 15, 0 ) ) ) + Column( 'file_size', Numeric( 15, 0 ) ), + Column( 'total_size', Numeric( 15, 0 ) ) ) HistoryDatasetAssociationDisplayAtAuthorization.table = Table( "history_dataset_association_display_at_authorization", metadata, Column( "id", Integer, primary_key=True ), @@ -194,6 +198,37 @@ Role.table = Table( "role", metadata, Column( "type", String( 40 ), index=True ), Column( "deleted", Boolean, index=True, default=False ) ) +UserQuotaAssociation.table = Table( "user_quota_association", metadata, + Column( "id", Integer, primary_key=True ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), + Column( "quota_id", Integer, ForeignKey( "quota.id" ), index=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ) ) + +GroupQuotaAssociation.table = Table( "group_quota_association", metadata, + Column( "id", Integer, primary_key=True ), + Column( "group_id", Integer, ForeignKey( "galaxy_group.id" ), index=True ), + Column( "quota_id", Integer, ForeignKey( "quota.id" ), index=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ) ) + +Quota.table = Table( "quota", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "name", String( 255 ), index=True, unique=True ), + Column( "description", TEXT ), + Column( "bytes", BigInteger ), + Column( "operation", String( 8 ) ), + Column( "deleted", Boolean, index=True, default=False ) ) + +DefaultQuotaAssociation.table = Table( "default_quota_association", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "type", String( 32 ), index=True, unique=True ), + Column( "quota_id", Integer, ForeignKey( "quota.id" ), index=True ) ) + DatasetPermissions.table = Table( "dataset_permissions", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -271,6 +306,7 @@ LibraryDatasetDatasetAssociation.table = Table( "library_dataset_dataset_associa Column( "info", TrimmedString( 255 ) ), Column( "blurb", TrimmedString( 255 ) ), Column( "peek" , TEXT ), + Column( "tool_version" , TEXT ), Column( "extension", TrimmedString( 64 ) ), Column( "metadata", MetadataType(), key="_metadata" ), Column( "parent_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), nullable=True ), @@ -327,6 +363,17 @@ LibraryDatasetDatasetInfoAssociation.table = Table( 'library_dataset_dataset_inf Column( "form_values_id", Integer, ForeignKey( "form_values.id" ), index=True ), Column( "deleted", Boolean, index=True, default=False ) ) +ToolShedRepository.table = Table( "tool_shed_repository", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "tool_shed", TrimmedString( 255 ), index=True ), + Column( "name", TrimmedString( 255 ), index=True ), + Column( "description" , TEXT ), + Column( "owner", TrimmedString( 255 ), index=True ), + Column( "changeset_revision", TrimmedString( 255 ), index=True ), + Column( "deleted", Boolean, index=True, default=False ) ) + Job.table = Table( "job", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -367,6 +414,12 @@ JobToOutputDatasetAssociation.table = Table( "job_to_output_dataset", metadata, Column( "dataset_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ), Column( "name", String(255) ) ) +JobToInputLibraryDatasetAssociation.table = Table( "job_to_input_library_dataset", metadata, + Column( "id", Integer, primary_key=True ), + Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ), + Column( "ldda_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), index=True ), + Column( "name", String(255) ) ) + JobToOutputLibraryDatasetAssociation.table = Table( "job_to_output_library_dataset", metadata, Column( "id", Integer, primary_key=True ), Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ), @@ -472,8 +525,8 @@ GalaxySession.table = Table( "galaxy_session", metadata, Column( "current_history_id", Integer, ForeignKey( "history.id" ), nullable=True ), Column( "session_key", TrimmedString( 255 ), index=True, unique=True ), # unique 128 bit random number coerced to a string Column( "is_valid", Boolean, default=False ), - Column( "prev_session_id", Integer ) # saves a reference to the previous session so we have a way to chain them together - ) + Column( "prev_session_id", Integer ), # saves a reference to the previous session so we have a way to chain them together + Column( "disk_usage", Numeric( 15, 0 ), index=True ) ) GalaxySessionToHistoryAssociation.table = Table( "galaxy_session_to_history", metadata, Column( "id", Integer, primary_key=True ), @@ -1132,7 +1185,10 @@ assign_mapper( context, Dataset, Dataset.table, primaryjoin=( Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id ) ), active_history_associations=relation( HistoryDatasetAssociation, - primaryjoin=( ( Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id ) & ( HistoryDatasetAssociation.table.c.deleted == False ) ) ), + primaryjoin=( ( Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id ) & ( HistoryDatasetAssociation.table.c.deleted == False ) & ( HistoryDatasetAssociation.table.c.purged == False ) ) ), + purged_history_associations=relation( + HistoryDatasetAssociation, + primaryjoin=( ( Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id ) & ( HistoryDatasetAssociation.table.c.purged == True ) ) ), library_associations=relation( LibraryDatasetDatasetAssociation, primaryjoin=( Dataset.table.c.id == LibraryDatasetDatasetAssociation.table.c.dataset_id ) ), @@ -1239,6 +1295,21 @@ assign_mapper( context, GroupRoleAssociation, GroupRoleAssociation.table, ) ) +assign_mapper( context, Quota, Quota.table, + properties=dict( users=relation( UserQuotaAssociation ), + groups=relation( GroupQuotaAssociation ) ) ) + +assign_mapper( context, UserQuotaAssociation, UserQuotaAssociation.table, + properties=dict( user=relation( User, backref="quotas" ), + quota=relation( Quota ) ) ) + +assign_mapper( context, GroupQuotaAssociation, GroupQuotaAssociation.table, + properties=dict( group=relation( Group, backref="quotas" ), + quota=relation( Quota ) ) ) + +assign_mapper( context, DefaultQuotaAssociation, DefaultQuotaAssociation.table, + properties=dict( quota=relation( Quota, backref="default" ) ) ) + assign_mapper( context, DatasetPermissions, DatasetPermissions.table, properties=dict( dataset=relation( Dataset, backref="actions" ), @@ -1370,6 +1441,9 @@ assign_mapper( context, JobToInputDatasetAssociation, JobToInputDatasetAssociati assign_mapper( context, JobToOutputDatasetAssociation, JobToOutputDatasetAssociation.table, properties=dict( job=relation( Job ), dataset=relation( HistoryDatasetAssociation, lazy=False ) ) ) +assign_mapper( context, JobToInputLibraryDatasetAssociation, JobToInputLibraryDatasetAssociation.table, + properties=dict( job=relation( Job ), dataset=relation( LibraryDatasetDatasetAssociation, lazy=False ) ) ) + assign_mapper( context, JobToOutputLibraryDatasetAssociation, JobToOutputLibraryDatasetAssociation.table, properties=dict( job=relation( Job ), dataset=relation( LibraryDatasetDatasetAssociation, lazy=False ) ) ) @@ -1404,6 +1478,7 @@ assign_mapper( context, Job, Job.table, input_datasets=relation( JobToInputDatasetAssociation ), output_datasets=relation( JobToOutputDatasetAssociation ), post_job_actions=relation( PostJobActionAssociation, lazy=False ), + input_library_datasets=relation( JobToInputLibraryDatasetAssociation ), output_library_datasets=relation( JobToOutputLibraryDatasetAssociation ), external_output_metadata = relation( JobExternalOutputMetadata, lazy = False ) ) ) @@ -1519,7 +1594,9 @@ assign_mapper( context, Page, Page.table, annotations=relation( PageAnnotationAssociation, order_by=PageAnnotationAssociation.table.c.id, backref="pages" ), ratings=relation( PageRatingAssociation, order_by=PageRatingAssociation.table.c.id, backref="pages" ) ) ) - + +assign_mapper( context, ToolShedRepository, ToolShedRepository.table ) + # Set up proxy so that # Page.users_shared_with # returns a list of users that page is shared with. diff --git a/lib/galaxy/model/migrate/check.py b/lib/galaxy/model/migrate/check.py index 3ca81a319a9..e699e6ddf9d 100644 --- a/lib/galaxy/model/migrate/check.py +++ b/lib/galaxy/model/migrate/check.py @@ -20,7 +20,7 @@ dialect_to_egg = { "mysql" : "MySQL_python" } -def create_or_verify_database( url, engine_options={} ): +def create_or_verify_database( url, galaxy_config_file, engine_options={} ): """ Check that the database is use-able, possibly creating it if empty (this is the only time we automatically create tables, otherwise we force the @@ -98,8 +98,11 @@ def create_or_verify_database( url, engine_options={} ): # Verify that the code and the DB are in sync db_schema = schema.ControlledSchema( engine, migrate_repository ) if migrate_repository.versions.latest != db_schema.version: - raise Exception( "Your database has version '%d' but this code expects version '%d'. Please backup your database and then migrate the schema by running 'sh manage_db.sh upgrade'." - % ( db_schema.version, migrate_repository.versions.latest ) ) + config_arg = '' + if os.path.abspath( os.path.join( os.getcwd(), 'universe_wsgi.ini' ) ) != galaxy_config_file: + config_arg = ' -c %s' % galaxy_config_file.replace( os.path.abspath( os.getcwd() ), '.' ) + raise Exception( "Your database has version '%d' but this code expects version '%d'. Please backup your database and then migrate the schema by running 'sh manage_db.sh%s upgrade'." + % ( db_schema.version, migrate_repository.versions.latest, config_arg ) ) else: log.info( "At database version %d" % db_schema.version ) @@ -123,4 +126,4 @@ def migrate_to_current_version( engine, schema ): finally: for message in "".join( sys.stdout.buffer ).split( "\n" ): log.info( message ) - sys.stdout = old_stdout \ No newline at end of file + sys.stdout = old_stdout diff --git a/lib/galaxy/model/migrate/versions/0065_add_name_to_form_fields_and_values.py b/lib/galaxy/model/migrate/versions/0065_add_name_to_form_fields_and_values.py index d684f5a698a..8083fb1d7a3 100644 --- a/lib/galaxy/model/migrate/versions/0065_add_name_to_form_fields_and_values.py +++ b/lib/galaxy/model/migrate/versions/0065_add_name_to_form_fields_and_values.py @@ -10,12 +10,19 @@ from migrate import * from migrate.changeset import * from sqlalchemy.exc import * from galaxy.util.json import from_json_string, to_json_string +from galaxy.model.custom_types import _sniffnfix_pg9_hex import datetime now = datetime.datetime.utcnow -import logging +import sys, logging log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) metadata = MetaData( migrate_engine ) db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) @@ -39,21 +46,24 @@ def upgrade(): return '' # Go through the entire table and add a 'name' attribute for each field # in the list of fields for each form definition - cmd = "SELECT id, fields FROM form_definition" + cmd = "SELECT f.id, f.fields FROM form_definition AS f" result = db_session.execute( cmd ) for row in result: form_definition_id = row[0] fields = str( row[1] ) if not fields.strip(): continue - fields_list = from_json_string( fields ) + fields_list = from_json_string( _sniffnfix_pg9_hex( fields ) ) if len( fields_list ): for index, field in enumerate( fields_list ): field[ 'name' ] = 'field_%i' % index field[ 'helptext' ] = field[ 'helptext' ].replace("'", "''").replace('"', "") field[ 'label' ] = field[ 'label' ].replace("'", "''") fields_json = to_json_string( fields_list ) - cmd = "UPDATE form_definition SET fields='%s' WHERE id=%i" %( fields_json, form_definition_id ) + if migrate_engine.name == 'mysql': + cmd = "UPDATE form_definition AS f SET f.fields='%s' WHERE f.id=%i" %( fields_json, form_definition_id ) + else: + cmd = "UPDATE form_definition SET fields='%s' WHERE id=%i" %( fields_json, form_definition_id ) db_session.execute( cmd ) # replace the values list in the content field of the form_values table with a name:value dict cmd = "SELECT form_values.id, form_values.content, form_definition.fields" \ @@ -112,17 +122,20 @@ def downgrade(): cmd = "UPDATE form_values SET content='%s' WHERE id=%i" %( to_json_string( values_list ), form_values_id ) db_session.execute( cmd ) # remove name attribute from the field column of the form_definition table - cmd = "SELECT id, fields FROM form_definition" + cmd = "SELECT f.id, f.fields FROM form_definition AS f" result = db_session.execute( cmd ) for row in result: form_definition_id = row[0] fields = str( row[1] ) if not fields.strip(): continue - fields_list = from_json_string( fields ) + fields_list = from_json_string( _sniffnfix_pg9_hex( fields ) ) if len( fields_list ): for index, field in enumerate( fields_list ): if field.has_key( 'name' ): del field[ 'name' ] - cmd = "UPDATE form_definition SET fields='%s' WHERE id=%i" %( to_json_string( fields_list ), form_definition_id ) + if migrate_engine.name == 'mysql': + cmd = "UPDATE form_definition AS f SET f.fields='%s' WHERE f.id=%i" %( to_json_string( fields_list ), form_definition_id ) + else: + cmd = "UPDATE form_definition SET fields='%s' WHERE id=%i" %( to_json_string( fields_list ), form_definition_id ) db_session.execute( cmd ) diff --git a/lib/galaxy/model/migrate/versions/0074_add_purged_column_to_library_dataset_table.py b/lib/galaxy/model/migrate/versions/0074_add_purged_column_to_library_dataset_table.py index fbf937b3eba..c5086486a8a 100644 --- a/lib/galaxy/model/migrate/versions/0074_add_purged_column_to_library_dataset_table.py +++ b/lib/galaxy/model/migrate/versions/0074_add_purged_column_to_library_dataset_table.py @@ -7,6 +7,9 @@ from sqlalchemy.orm import * from migrate import * from migrate.changeset import * +import logging +log = logging.getLogger( __name__ ) + metadata = MetaData( migrate_engine ) db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) diff --git a/lib/galaxy/model/migrate/versions/0076_fix_form_values_data_corruption.py b/lib/galaxy/model/migrate/versions/0076_fix_form_values_data_corruption.py index 48b4b206650..dc42a4eb3ff 100644 --- a/lib/galaxy/model/migrate/versions/0076_fix_form_values_data_corruption.py +++ b/lib/galaxy/model/migrate/versions/0076_fix_form_values_data_corruption.py @@ -32,7 +32,7 @@ def _sniffnfix_pg9_hex(value): def upgrade(): print __doc__ metadata.reflect() - cmd = "SELECT form_values.id as id, form_values.content as field_values, form_definition.fields as fields " \ + cmd = "SELECT form_values.id as id, form_values.content as field_values, form_definition.fields as fdfields " \ + " FROM form_definition, form_values " \ + " WHERE form_values.form_definition_id=form_definition.id " \ + " ORDER BY form_values.id" @@ -46,7 +46,7 @@ def upgrade(): except Exception, e: corrupted_rows = corrupted_rows + 1 # content field is corrupted - fields_list = from_json_string( _sniffnfix_pg9_hex( str( row['fields'] ) ) ) + fields_list = from_json_string( _sniffnfix_pg9_hex( str( row['fdfields'] ) ) ) field_values_str = _sniffnfix_pg9_hex( str( row['field_values'] ) ) try: #Encoding errors? Just to be safe. diff --git a/lib/galaxy/model/migrate/versions/0078_add_columns_for_disk_usage_accounting.py b/lib/galaxy/model/migrate/versions/0078_add_columns_for_disk_usage_accounting.py new file mode 100644 index 00000000000..866b4b8c542 --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0078_add_columns_for_disk_usage_accounting.py @@ -0,0 +1,87 @@ +""" +Migration script to add 'total_size' column to the dataset table, 'purged' +column to the HDA table, and 'disk_usage' column to the User and GalaxySession +tables. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import logging +log = logging.getLogger( __name__ ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + + try: + Dataset_table = Table( "dataset", metadata, autoload=True ) + c = Column( 'total_size', Numeric( 15, 0 ) ) + c.create( Dataset_table ) + assert c is Dataset_table.c.total_size + except Exception, e: + print "Adding total_size column to dataset table failed: %s" % str( e ) + log.debug( "Adding total_size column to dataset table failed: %s" % str( e ) ) + + try: + HistoryDatasetAssociation_table = Table( "history_dataset_association", metadata, autoload=True ) + c = Column( "purged", Boolean, index=True, default=False ) + c.create( HistoryDatasetAssociation_table ) + assert c is HistoryDatasetAssociation_table.c.purged + db_session.execute(HistoryDatasetAssociation_table.update().values(purged=False)) + except Exception, e: + print "Adding purged column to history_dataset_association table failed: %s" % str( e ) + log.debug( "Adding purged column to history_dataset_association table failed: %s" % str( e ) ) + + try: + User_table = Table( "galaxy_user", metadata, autoload=True ) + c = Column( 'disk_usage', Numeric( 15, 0 ), index=True ) + c.create( User_table ) + assert c is User_table.c.disk_usage + except Exception, e: + print "Adding disk_usage column to galaxy_user table failed: %s" % str( e ) + log.debug( "Adding disk_usage column to galaxy_user table failed: %s" % str( e ) ) + + try: + GalaxySession_table = Table( "galaxy_session", metadata, autoload=True ) + c = Column( 'disk_usage', Numeric( 15, 0 ), index=True ) + c.create( GalaxySession_table ) + assert c is GalaxySession_table.c.disk_usage + except Exception, e: + print "Adding disk_usage column to galaxy_session table failed: %s" % str( e ) + log.debug( "Adding disk_usage column to galaxy_session table failed: %s" % str( e ) ) + +def downgrade(): + metadata.reflect() + try: + Dataset_table = Table( "dataset", metadata, autoload=True ) + Dataset_table.c.total_size.drop() + except Exception, e: + print "Dropping total_size column from dataset table failed: %s" % str( e ) + log.debug( "Dropping total_size column from dataset table failed: %s" % str( e ) ) + + try: + HistoryDatasetAssociation_table = Table( "history_dataset_association", metadata, autoload=True ) + HistoryDatasetAssociation_table.c.purged.drop() + except Exception, e: + print "Dropping purged column from history_dataset_association table failed: %s" % str( e ) + log.debug( "Dropping purged column from history_dataset_association table failed: %s" % str( e ) ) + + try: + User_table = Table( "galaxy_user", metadata, autoload=True ) + User_table.c.disk_usage.drop() + except Exception, e: + print "Dropping disk_usage column from galaxy_user table failed: %s" % str( e ) + log.debug( "Dropping disk_usage column from galaxy_user table failed: %s" % str( e ) ) + + try: + GalaxySession_table = Table( "galaxy_session", metadata, autoload=True ) + GalaxySession_table.c.disk_usage.drop() + except Exception, e: + print "Dropping disk_usage column from galaxy_session table failed: %s" % str( e ) + log.debug( "Dropping disk_usage column from galaxy_session table failed: %s" % str( e ) ) diff --git a/lib/galaxy/model/migrate/versions/0079_input_library_to_job_table.py b/lib/galaxy/model/migrate/versions/0079_input_library_to_job_table.py new file mode 100644 index 00000000000..9a46ee3a720 --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0079_input_library_to_job_table.py @@ -0,0 +1,41 @@ +""" +Migration script to add the job_to_input_library_dataset table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import logging +log = logging.getLogger( __name__ ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +JobToInputLibraryDatasetAssociation_table = Table( "job_to_input_library_dataset", metadata, + Column( "id", Integer, primary_key=True ), + Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ), + Column( "ldda_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), index=True ), + Column( "name", String(255) ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + + # Create the job_to_input_library_dataset table + try: + JobToInputLibraryDatasetAssociation_table.create() + except Exception, e: + print "Creating job_to_input_library_dataset table failed: %s" % str( e ) + log.debug( "Creating job_to_input_library_dataset table failed: %s" % str( e ) ) + +def downgrade(): + metadata.reflect() + + # Drop the job_to_input_library_dataset table + try: + JobToInputLibraryDatasetAssociation_table.drop() + except Exception, e: + print str(e) + log.debug( "Dropping job_to_input_library_dataset table failed: %s" % str( e ) ) diff --git a/lib/galaxy/model/migrate/versions/0080_quota_tables.py b/lib/galaxy/model/migrate/versions/0080_quota_tables.py new file mode 100644 index 00000000000..ec642dc65d7 --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0080_quota_tables.py @@ -0,0 +1,120 @@ +""" +Migration script to create tables for disk quotas. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * +from galaxy.model.orm.ext.assignmapper import * +from galaxy.model.custom_types import * + +import datetime +now = datetime.datetime.utcnow + +import logging +log = logging.getLogger( __name__ ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +# Tables to add + +Quota_table = Table( "quota", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "name", String( 255 ), index=True, unique=True ), + Column( "description", TEXT ), + Column( "bytes", BigInteger ), + Column( "operation", String( 8 ) ), + Column( "deleted", Boolean, index=True, default=False ) ) + +UserQuotaAssociation_table = Table( "user_quota_association", metadata, + Column( "id", Integer, primary_key=True ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), + Column( "quota_id", Integer, ForeignKey( "quota.id" ), index=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ) ) + +GroupQuotaAssociation_table = Table( "group_quota_association", metadata, + Column( "id", Integer, primary_key=True ), + Column( "group_id", Integer, ForeignKey( "galaxy_group.id" ), index=True ), + Column( "quota_id", Integer, ForeignKey( "quota.id" ), index=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ) ) + +DefaultQuotaAssociation_table = Table( "default_quota_association", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "type", String( 32 ), index=True, unique=True ), + Column( "quota_id", Integer, ForeignKey( "quota.id" ), index=True ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + + # Create quota table + try: + Quota_table.create() + except Exception, e: + log.debug( "Creating quota table failed: %s" % str( e ) ) + + # Create user_quota_association table + try: + UserQuotaAssociation_table.create() + except Exception, e: + log.debug( "Creating user_quota_association table failed: %s" % str( e ) ) + + # Create group_quota_association table + try: + GroupQuotaAssociation_table.create() + except Exception, e: + log.debug( "Creating group_quota_association table failed: %s" % str( e ) ) + + # Create default_quota_association table + try: + DefaultQuotaAssociation_table.create() + except Exception, e: + log.debug( "Creating default_quota_association table failed: %s" % str( e ) ) + + # Create the default quota record + #class Quota( object ): + # def __init__( self, name, description, bytes, operation ): + # self.name = name + # self.description = description + # self.bytes = bytes + # self.operation = operation + #assign_mapper( db_session, Quota, Quota_table ) + #default_quota = Quota( 'Default Quota', 'The base quota applied to all users', -1, '=' ) + #db_session.add( default_quota ) + #db_session.flush() + + +def downgrade(): + metadata.reflect() + + # Drop default_quota_association table + try: + DefaultQuotaAssociation_table.drop() + except Exception, e: + log.debug( "Dropping default_quota_association table failed: %s" % str( e ) ) + + # Drop group_quota_association table + try: + GroupQuotaAssociation_table.drop() + except Exception, e: + log.debug( "Dropping group_quota_association table failed: %s" % str( e ) ) + + # Drop user_quota_association table + try: + UserQuotaAssociation_table.drop() + except Exception, e: + log.debug( "Dropping user_quota_association table failed: %s" % str( e ) ) + + # Drop quota table + try: + Quota_table.drop() + except Exception, e: + log.debug( "Dropping quota table failed: %s" % str( e ) ) diff --git a/lib/galaxy/model/migrate/versions/0081_add_tool_version_to_hda_ldda.py b/lib/galaxy/model/migrate/versions/0081_add_tool_version_to_hda_ldda.py new file mode 100644 index 00000000000..10dfdc81daf --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0081_add_tool_version_to_hda_ldda.py @@ -0,0 +1,41 @@ +""" +Migration script to add a 'tool_version' column to the hda/ldda tables. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +from galaxy.model.custom_types import * + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + try: + hda_table = Table( "history_dataset_association", metadata, autoload=True ) + c = Column( "tool_version", TEXT ) + c.create( hda_table ) + assert c is hda_table.c.tool_version + + ldda_table = Table( "library_dataset_dataset_association", metadata, autoload=True ) + c = Column( "tool_version", TEXT ) + c.create( ldda_table ) + assert c is ldda_table.c.tool_version + + except Exception, e: + print "Adding the tool_version column to the hda/ldda tables failed: ", str( e ) + +def downgrade(): + metadata.reflect() + try: + hda_table = Table( "history_dataset_association", metadata, autoload=True ) + hda_table.c.tool_version.drop() + + ldda_table = Table( "library_dataset_dataset_association", metadata, autoload=True ) + ldda_table.c.tool_version.drop() + except Exception, e: + print "Dropping the tool_version column from hda/ldda table failed: ", str( e ) diff --git a/lib/galaxy/model/migrate/versions/0082_add_tool_shed_repository_table.py b/lib/galaxy/model/migrate/versions/0082_add_tool_shed_repository_table.py new file mode 100644 index 00000000000..d06c2354a29 --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0082_add_tool_shed_repository_table.py @@ -0,0 +1,49 @@ +""" +Migration script to add the tool_shed_repository table. +""" +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * +import sys, logging +from galaxy.model.custom_types import * +from sqlalchemy.exc import * +import datetime +now = datetime.datetime.utcnow + +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData( migrate_engine ) + +# New table to store information about cloned tool shed repositories. +ToolShedRepository_table = Table( "tool_shed_repository", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "tool_shed", TrimmedString( 255 ), index=True ), + Column( "name", TrimmedString( 255 ), index=True ), + Column( "description" , TEXT ), + Column( "owner", TrimmedString( 255 ), index=True ), + Column( "changeset_revision", TrimmedString( 255 ), index=True ), + Column( "deleted", Boolean, index=True, default=False ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + try: + ToolShedRepository_table.create() + except Exception, e: + log.debug( "Creating tool_shed_repository table failed: %s" % str( e ) ) + +def downgrade(): + metadata.reflect() + try: + ToolShedRepository_table.drop() + except Exception, e: + log.debug( "Dropping tool_shed_repository table failed: %s" % str( e ) ) diff --git a/lib/galaxy/quota/__init__.py b/lib/galaxy/quota/__init__.py new file mode 100644 index 00000000000..e5f7e88bbbd --- /dev/null +++ b/lib/galaxy/quota/__init__.py @@ -0,0 +1,172 @@ +""" +Galaxy Quotas + +""" +import logging, socket, operator +from datetime import datetime, timedelta +from galaxy import util +from galaxy.util.bunch import Bunch +from galaxy.model.orm import * + +log = logging.getLogger(__name__) + +class NoQuotaAgent( object ): + """Base quota agent, always returns no quota""" + def __init__( self, model ): + self.model = model + self.sa_session = model.context + def get_quota( self, user, nice_size=False ): + return None + @property + def default_quota( self ): + return None + def get_usage( self, trans=None, user=False, history=False ): + if trans: + user = trans.user + history = trans.history + assert user is not False, "Could not determine user." + if not user: + assert history, "Could not determine anonymous user's history." + usage = history.get_disk_size() + else: + usage = user.total_disk_usage + return usage + def get_percent( self, trans=None, user=False, history=False, usage=False, quota=False ): + return None + def get_user_quotas( self, user ): + return [] + +class QuotaAgent( NoQuotaAgent ): + """Class that handles galaxy quotas""" + def get_quota( self, user, nice_size=False ): + """ + Calculated like so: + 1. Anonymous users get the default quota. + 2. Logged in users start with the highest of their associated '=' + quotas or the default quota, if there are no associated '=' + quotas. If an '=' unlimited (-1 in the database) quota is found + during this process, the user has no quota (aka unlimited). + 3. Quota is increased or decreased by any corresponding '+' or '-' + quotas. + """ + if not user: + return self.default_unregistered_quota + quotas = [] + for group in [ uga.group for uga in user.groups ]: + for quota in [ gqa.quota for gqa in group.quotas ]: + if quota not in quotas: + quotas.append( quota ) + for quota in [ uqa.quota for uqa in user.quotas ]: + if quota not in quotas: + quotas.append( quota ) + use_default = True + max = 0 + adjustment = 0 + rval = 0 + for quota in quotas: + if quota.deleted: + continue + if quota.operation == '=' and quota.bytes == -1: + rval = None + break + elif quota.operation == '=': + use_default = False + if quota.bytes > max: + max = quota.bytes + elif quota.operation == '+': + adjustment += quota.bytes + elif quota.operation == '-': + adjustment -= quota.bytes + if use_default: + max = self.default_registered_quota + if max is None: + rval = None + if rval is not None: + rval = max + adjustment + if rval <= 0: + rval = 0 + if nice_size: + if rval is not None: + rval = util.nice_size( rval ) + else: + rval = 'unlimited' + return rval + @property + def default_unregistered_quota( self ): + return self._default_quota( self.model.DefaultQuotaAssociation.types.UNREGISTERED ) + @property + def default_registered_quota( self ): + return self._default_quota( self.model.DefaultQuotaAssociation.types.REGISTERED ) + def _default_quota( self, default_type ): + dqa = self.sa_session.query( self.model.DefaultQuotaAssociation ).filter( self.model.DefaultQuotaAssociation.table.c.type==default_type ).first() + if not dqa: + return None + if dqa.quota.bytes < 0: + return None + return dqa.quota.bytes + def set_default_quota( self, default_type, quota ): + # Unset the current default(s) associated with this quota, if there are any + for dqa in quota.default: + self.sa_session.delete( dqa ) + # Unset the current users/groups associated with this quota + for uqa in quota.users: + self.sa_session.delete( uqa ) + for gqa in quota.groups: + self.sa_session.delete( gqa ) + # Find the old default, assign the new quota if it exists + dqa = self.sa_session.query( self.model.DefaultQuotaAssociation ).filter( self.model.DefaultQuotaAssociation.table.c.type==default_type ).first() + if dqa: + dqa.quota = quota + # Or create if necessary + else: + dqa = self.model.DefaultQuotaAssociation( default_type, quota ) + self.sa_session.add( dqa ) + self.sa_session.flush() + def get_percent( self, trans=None, user=False, history=False, usage=False, quota=False ): + if trans: + user = trans.user + history = trans.history + if quota is False: + quota = self.get_quota( user ) + if quota is None: + return None + if usage is False: + usage = self.get_usage( trans, user, history ) + percent = int( float( usage ) / quota * 100 ) + if percent > 100: + percent = 100 + return percent + def set_entity_quota_associations( self, quotas=[], users=[], groups=[], delete_existing_assocs=True ): + for quota in quotas: + if delete_existing_assocs: + flush_needed = False + for a in quota.users + quota.groups: + self.sa_session.delete( a ) + flush_neeeded = True + if flush_needed: + self.sa_session.flush() + for user in users: + uqa = self.model.UserQuotaAssociation( user, quota ) + self.sa_session.add( uqa ) + for group in groups: + gqa = self.model.GroupQuotaAssociation( group, quota ) + self.sa_session.add( gqa ) + self.sa_session.flush() + def get_user_quotas( self, user ): + rval = [] + if not user: + dqa = self.sa_session.query( self.model.DefaultQuotaAssociation ) \ + .filter( self.model.DefaultQuotaAssociation.table.c.type==self.model.DefaultQuotaAssociation.types.UNREGISTERED ).first() + if dqa: + rval.append( dqa.quota ) + else: + dqa = self.sa_session.query( self.model.DefaultQuotaAssociation ) \ + .filter( self.model.DefaultQuotaAssociation.table.c.type==self.model.DefaultQuotaAssociation.types.REGISTERED ).first() + if dqa: + rval.append( dqa.quota ) + for uqa in user.quotas: + rval.append( uqa.quota ) + for group in [ uga.group for uga in user.groups ]: + for gqa in group.quotas: + rval.append( gqa.quota ) + return rval diff --git a/lib/galaxy/security/__init__.py b/lib/galaxy/security/__init__.py index fdad35c1c6b..b2289b1940a 100644 --- a/lib/galaxy/security/__init__.py +++ b/lib/galaxy/security/__init__.py @@ -70,7 +70,7 @@ class RBACAgent: raise "Unimplemented Method" def set_dataset_permission( self, dataset, permission ): raise "Unimplemented Method" - def set_all_library_permissions( self, dataset, permissions ): + def set_all_library_permissions( self, trans, dataset, permissions ): raise "Unimplemented Method" def library_is_public( self, library ): raise "Unimplemented Method" @@ -90,6 +90,8 @@ class RBACAgent: raise "Unimplemented Method" def get_permissions( self, library_dataset ): raise "Unimplemented Method" + def get_all_roles( self, trans, cntrller ): + raise "Unimplemented Method" def get_legitimate_roles( self, trans, item, cntrller ): raise "Unimplemented Method" def derive_roles_from_access( self, trans, item_id, cntrller, library=False, **kwd ): @@ -120,6 +122,50 @@ class GalaxyRBACAgent( RBACAgent ): def sa_session( self ): """Returns a SQLAlchemy session""" return self.model.context + def sort_by_attr( self, seq, attr ): + """ + Sort the sequence of objects by object's attribute + Arguments: + seq - the list or any sequence (including immutable one) of objects to sort. + attr - the name of attribute to sort by + """ + # Use the "Schwartzian transform" + # Create the auxiliary list of tuples where every i-th tuple has form + # (seq[i].attr, i, seq[i]) and sort it. The second item of tuple is needed not + # only to provide stable sorting, but mainly to eliminate comparison of objects + # (which can be expensive or prohibited) in case of equal attribute values. + intermed = map( None, map( getattr, seq, ( attr, ) * len( seq ) ), xrange( len( seq ) ), seq ) + intermed.sort() + return map( operator.getitem, intermed, ( -1, ) * len( intermed ) ) + def get_all_roles( self, trans, cntrller ): + admin_controller = cntrller in [ 'library_admin' ] + roles = set() + if not trans.user: + return trans.sa_session.query( trans.app.model.Role ) \ + .filter( and_( self.model.Role.table.c.deleted==False, + self.model.Role.table.c.type != self.model.Role.types.PRIVATE, + self.model.Role.table.c.type != self.model.Role.types.SHARING ) ) \ + .order_by( self.model.Role.table.c.name ) + if admin_controller: + # The library is public and the user is an admin, so all roles are legitimate + for role in trans.sa_session.query( trans.app.model.Role ) \ + .filter( self.model.Role.table.c.deleted==False ) \ + .order_by( self.model.Role.table.c.name ): + roles.add( role ) + else: + # Add the current user's private role + roles.add( self.get_private_user_role( trans.user ) ) + # Add the current user's sharing roles + for role in self.get_sharing_roles( trans.user ): + roles.add( role ) + # Add all remaining non-private, non-sharing roles + for role in trans.sa_session.query( trans.app.model.Role ) \ + .filter( and_( self.model.Role.table.c.deleted==False, + self.model.Role.table.c.type != self.model.Role.types.PRIVATE, + self.model.Role.table.c.type != self.model.Role.types.SHARING ) ) \ + .order_by( self.model.Role.table.c.name ): + roles.add( role ) + return self.sort_by_attr( [ role for role in roles ], 'name' ) def get_legitimate_roles( self, trans, item, cntrller ): """ Return a sorted list of legitimate roles that can be associated with a permission on @@ -140,51 +186,10 @@ class GalaxyRBACAgent( RBACAgent ): for the current user's private role, will be excluded. """ admin_controller = cntrller in [ 'library_admin' ] - def sort_by_attr( seq, attr ): - """ - Sort the sequence of objects by object's attribute - Arguments: - seq - the list or any sequence (including immutable one) of objects to sort. - attr - the name of attribute to sort by - """ - # Use the "Schwartzian transform" - # Create the auxiliary list of tuples where every i-th tuple has form - # (seq[i].attr, i, seq[i]) and sort it. The second item of tuple is needed not - # only to provide stable sorting, but mainly to eliminate comparison of objects - # (which can be expensive or prohibited) in case of equal attribute values. - intermed = map( None, map( getattr, seq, ( attr, ) * len( seq ) ), xrange( len( seq ) ), seq ) - intermed.sort() - return map( operator.getitem, intermed, ( -1, ) * len( intermed ) ) roles = set() if ( isinstance( item, self.model.Library ) and self.library_is_public( item ) ) or \ ( isinstance( item, self.model.Dataset ) and self.dataset_is_public( item ) ): - if not trans.user: - return trans.sa_session.query( trans.app.model.Role ) \ - .filter( and_( self.model.Role.table.c.deleted==False, - self.model.Role.table.c.type != self.model.Role.types.PRIVATE, - self.model.Role.table.c.type != self.model.Role.types.SHARING ) ) \ - .order_by( self.model.Role.table.c.name ) - if admin_controller: - # The library is public and the user is an admin, so all roles are legitimate - for role in trans.sa_session.query( trans.app.model.Role ) \ - .filter( self.model.Role.table.c.deleted==False ) \ - .order_by( self.model.Role.table.c.name ): - roles.add( role ) - return sort_by_attr( [ role for role in roles ], 'name' ) - else: - # Add the current user's private role - roles.add( self.get_private_user_role( trans.user ) ) - # Add the current user's sharing roles - for role in self.get_sharing_roles( trans.user ): - roles.add( role ) - # Add all remaining non-private, non-sharing roles - for role in trans.sa_session.query( trans.app.model.Role ) \ - .filter( and_( self.model.Role.table.c.deleted==False, - self.model.Role.table.c.type != self.model.Role.types.PRIVATE, - self.model.Role.table.c.type != self.model.Role.types.SHARING ) ) \ - .order_by( self.model.Role.table.c.name ): - roles.add( role ) - return sort_by_attr( [ role for role in roles ], 'name' ) + return self.get_all_roles( trans, cntrller ) # If item has roles associated with the access permission, we need to start with them. access_roles = item.get_access_roles( trans ) for role in access_roles: @@ -205,7 +210,7 @@ class GalaxyRBACAgent( RBACAgent ): for ura in user.roles: if admin_controller or self.ok_to_display( trans.user, ura.role ): roles.add( ura.role ) - return sort_by_attr( [ role for role in roles ], 'name' ) + return self.sort_by_attr( [ role for role in roles ], 'name' ) def ok_to_display( self, user, role ): """ Method for checking if: @@ -287,8 +292,12 @@ class GalaxyRBACAgent( RBACAgent ): if self.can_access_library_item( roles, library_dataset, user ): return True if search_downward: - for folder in folder.active_folders: - return self.has_accessible_library_datasets( trans, folder, user, roles, search_downward=search_downward ) + return self.__active_folders_have_accessible_library_datasets( trans, folder, user, roles ) + return False + def __active_folders_have_accessible_library_datasets( self, trans, folder, user, roles ): + for active_folder in folder.active_folders: + if self.has_accessible_library_datasets( trans, active_folder, user, roles ): + return True return False def can_access_library_item( self, roles, item, user ): if type( item ) == self.model.Library: @@ -475,6 +484,18 @@ class GalaxyRBACAgent( RBACAgent ): Set new permissions on a dataset, eliminating all current permissions permissions looks like: { Action : [ Role, Role ] } """ + # Make sure that DATASET_MANAGE_PERMISSIONS is associated with at least 1 role + has_dataset_manage_permissions = False + for action, roles in permissions.items(): + if isinstance( action, Action ): + if action == self.permitted_actions.DATASET_MANAGE_PERMISSIONS and roles: + has_dataset_manage_permissions = True + break + elif action == self.permitted_actions.DATASET_MANAGE_PERMISSIONS.action and roles: + has_dataset_manage_permissions = True + break + if not has_dataset_manage_permissions: + return "At least 1 role must be associated with the manage permissions permission on this dataset." flush_needed = False # Delete all of the current permissions on the dataset for dp in dataset.actions: @@ -489,6 +510,7 @@ class GalaxyRBACAgent( RBACAgent ): flush_needed = True if flush_needed: self.sa_session.flush() + return "" def set_dataset_permission( self, dataset, permission={} ): """ Set a specific permission on a dataset, leaving all other current permissions on the dataset alone @@ -576,7 +598,7 @@ class GalaxyRBACAgent( RBACAgent ): for user in users: self.associate_components( user=user, role=sharing_role ) self.set_dataset_permission( dataset, { self.permitted_actions.DATASET_ACCESS : [ sharing_role ] } ) - def set_all_library_permissions( self, library_item, permissions={} ): + def set_all_library_permissions( self, trans, library_item, permissions={} ): # Set new permissions on library_item, eliminating all current permissions flush_needed = False for role_assoc in library_item.actions: @@ -591,14 +613,21 @@ class GalaxyRBACAgent( RBACAgent ): for role_assoc in [ permission_class( action, library_item, role ) for role in roles ]: self.sa_session.add( role_assoc ) flush_needed = True - if isinstance( library_item, self.model.LibraryDatasetDatasetAssociation ) and \ - action == self.permitted_actions.LIBRARY_MANAGE.action: - # Handle the special case when we are setting the LIBRARY_MANAGE_PERMISSION on a - # library_dataset_dataset_association since the roles need to be applied to the - # DATASET_MANAGE_PERMISSIONS permission on the associated dataset - permissions = {} - permissions[ self.permitted_actions.DATASET_MANAGE_PERMISSIONS ] = roles - self.set_dataset_permission( library_item.dataset, permissions ) + if isinstance( library_item, self.model.LibraryDatasetDatasetAssociation ): + # Permission setting related to DATASET_MANAGE_PERMISSIONS was broken for a period of time, + # so it is possible that some Datasets have no roles associated with the DATASET_MANAGE_PERMISSIONS + # permission. In this case, we'll reset this permission to the library_item user's private role. + if not library_item.dataset.has_manage_permissions_roles( trans ): + permission = {} + permissions[ self.permitted_actions.DATASET_MANAGE_PERMISSIONS ] = [ trans.app.security_agent.get_private_user_role( library_item.user ) ] + self.set_dataset_permission( library_item.dataset, permissions ) + if action == self.permitted_actions.LIBRARY_MANAGE.action and roles: + # Handle the special case when we are setting the LIBRARY_MANAGE_PERMISSION on a + # library_dataset_dataset_association since the roles need to be applied to the + # DATASET_MANAGE_PERMISSIONS permission on the associated dataset. + permissions = {} + permissions[ self.permitted_actions.DATASET_MANAGE_PERMISSIONS ] = roles + self.set_dataset_permission( library_item.dataset, permissions ) if flush_needed: self.sa_session.flush() def library_is_public( self, library, contents=False ): @@ -625,7 +654,8 @@ class GalaxyRBACAgent( RBACAgent ): if not self.folder_is_public( sub_folder ): return False for library_dataset in folder.datasets: - if not self.dataset_is_public( library_dataset.library_dataset_dataset_association.dataset ): + ldda = library_dataset.library_dataset_dataset_association + if ldda and ldda.dataset and not self.dataset_is_public( ldda.dataset ): return False return True def make_folder_public( self, folder ): @@ -748,7 +778,7 @@ class GalaxyRBACAgent( RBACAgent ): else: permissions[ self.get_action( v.action ) ] = in_roles return permissions, in_roles, error, msg - def copy_library_permissions( self, source_library_item, target_library_item, user=None ): + def copy_library_permissions( self, trans, source_library_item, target_library_item, user=None ): # Copy all relevant permissions from source. permissions = {} for role_assoc in source_library_item.actions: @@ -758,7 +788,7 @@ class GalaxyRBACAgent( RBACAgent ): permissions[role_assoc.action].append( role_assoc.role ) else: permissions[role_assoc.action] = [ role_assoc.role ] - self.set_all_library_permissions( target_library_item, permissions ) + self.set_all_library_permissions( trans, target_library_item, permissions ) if user: item_class = None for item_class, permission_class in self.library_item_assocs: diff --git a/lib/galaxy/security/validate_user_input.py b/lib/galaxy/security/validate_user_input.py new file mode 100644 index 00000000000..f933780c916 --- /dev/null +++ b/lib/galaxy/security/validate_user_input.py @@ -0,0 +1,39 @@ +import re + +VALID_USERNAME_RE = re.compile( "^[a-z0-9\-]+$" ) + +def validate_email( trans, email, user=None, check_dup=True ): + message = '' + if user and user.email == email: + return message + if len( email ) == 0 or "@" not in email or "." not in email: + message = "Enter a real email address" + elif len( email ) > 255: + message = "Email address exceeds maximum allowable length" + elif check_dup and trans.sa_session.query( trans.app.model.User ).filter_by( email=email ).first(): + message = "User with that email already exists" + return message + +def validate_username( trans, username, user=None ): + # User names must be at least four characters in length and contain only lower-case + # letters, numbers, and the '-' character. + if username in [ 'None', None, '' ]: + return '' + if user and user.username == username: + return '' + if len( username ) < 4: + return "User name must be at least 4 characters in length" + if len( username ) > 255: + return "User name cannot be more than 255 characters in length" + if not( VALID_USERNAME_RE.match( username ) ): + return "User name must contain only lower-case letters, numbers and '-'" + if trans.sa_session.query( trans.app.model.User ).filter_by( username=username ).first(): + return "This user name is not available" + return '' + +def validate_password( trans, password, confirm ): + if len( password ) < 6: + return "Use a password of at least 6 characters" + elif password != confirm: + return "Passwords do not match" + return '' \ No newline at end of file diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index da6a6fd3b08..dcc20347c0d 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1,11 +1,11 @@ """ Classes encapsulating galaxy tools and tool configuration. """ -import pkg_resources; +import pkg_resources pkg_resources.require( "simplejson" ) -import logging, os, string, sys, tempfile, glob, shutil, types, urllib +import logging, os, string, sys, tempfile, glob, shutil, types, urllib, subprocess import simplejson import binascii from UserDict import DictMixin @@ -28,6 +28,7 @@ from galaxy.util.none_like import NoneDataset from galaxy.datatypes import sniff from cgi import FieldStorage from galaxy.util.hash_util import * +from galaxy.util import listify log = logging.getLogger( __name__ ) @@ -39,23 +40,28 @@ class ToolBox( object ): Container for a collection of tools """ - def __init__( self, config_filename, tool_root_dir, app ): + def __init__( self, config_filenames, tool_root_dir, app ): """ Create a toolbox from the config file names by `config_filename`, using `tool_root_directory` as the base directory for finding individual tool config files. """ + # The shed_tool_confs dictionary contains shed_conf_filename : tool_path pairs. + self.shed_tool_confs = {} self.tools_by_id = {} self.workflows_by_id = {} self.tool_panel = odict() + # The following refers to the tool_path config setting for backward compatibility. + # Additional newer (e.g., shed_tool_conf.xml) files include the tool_path attribute + # within the tag. self.tool_root_dir = tool_root_dir self.app = app self.init_dependency_manager() - try: - self.init_tools( config_filename ) - except: - log.exception( "ToolBox error reading %s", config_filename ) - + for config_filename in listify( config_filenames ): + try: + self.init_tools( config_filename ) + except: + log.exception( "ToolBox error reading %s", config_filename ) def init_tools( self, config_filename ): """ Read the configuration file and load each tool. @@ -71,83 +77,99 @@ class ToolBox( object ): """ - def load_tool( elem, panel_dict ): - try: - path = elem.get( "file" ) - tool = self.load_tool( os.path.join( self.tool_root_dir, path ) ) - if self.app.config.get_bool( 'enable_tool_tags', False ): - tag_names = elem.get( "tags", "" ).split( "," ) - for tag_name in tag_names: - if tag_name == '': - continue - tag = self.sa_session.query( self.app.model.Tag ).filter_by( name=tag_name ).first() - if not tag: - tag = self.app.model.Tag( name=tag_name ) - self.sa_session.add( tag ) - self.sa_session.flush() - tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id ) - self.sa_session.add( tta ) - self.sa_session.flush() - else: - for tagged_tool in tag.tagged_tools: - if tagged_tool.tool_id == tool.id: - break - else: - tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id ) - self.sa_session.add( tta ) - self.sa_session.flush() - self.tools_by_id[ tool.id ] = tool - key = 'tool_' + tool.id - panel_dict[ key ] = tool - log.debug( "Loaded tool: %s %s" % ( tool.id, tool.version ) ) - except: - log.exception( "error reading tool from path: %s" % path ) - def load_workflow( elem, panel_dict ): - try: - # TODO: should id be encoded? - workflow_id = elem.get( 'id' ) - workflow = self.load_workflow( workflow_id ) - self.workflows_by_id[ workflow_id ] = workflow - key = 'workflow_' + workflow_id - panel_dict[ key ] = workflow - log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) - except: - log.exception( "error loading workflow: %s" % workflow_id ) - def load_label( elem, panel_dict ): - label = ToolSectionLabel( elem ) - key = 'label_' + label.id - panel_dict[ key ] = label - def load_section( elem, panel_dict ): - section = ToolSection( elem ) - log.debug( "Loading section: %s" % section.name ) - for section_elem in elem: - if section_elem.tag == 'tool': - load_tool( section_elem, section.elems ) - elif section_elem.tag == 'workflow': - load_workflow( section_elem, section.elems ) - elif section_elem.tag == 'label': - load_label( section_elem, section.elems ) - key = 'section_' + section.id - panel_dict[ key ] = section - if self.app.config.get_bool( 'enable_tool_tags', False ): log.info("removing all tool tag associations (" + str( self.sa_session.query( self.app.model.ToolTagAssociation ).count() ) + ")") self.sa_session.query( self.app.model.ToolTagAssociation ).delete() self.sa_session.flush() - log.info("parsing the tool configuration") + log.info( "parsing the tool configuration %s" % config_filename ) tree = util.parse_xml( config_filename ) root = tree.getroot() + tool_path = root.get( 'tool_path' ) + if tool_path: + # We're parsing a shed_tool_conf file since we have a tool_path attribute. + self.shed_tool_confs[ config_filename ] = tool_path + else: + # Default to backward compatible config setting. + tool_path = self.tool_root_dir for elem in root: if elem.tag == 'tool': - load_tool( elem, self.tool_panel ) + self.load_tool_tag_set( elem, self.tool_panel, tool_path, guid=elem.get( 'guid' ) ) elif elem.tag == 'workflow': - load_workflow( elem, self.tool_panel ) + self.load_workflow_tag_set( elem, self.tool_panel ) elif elem.tag == 'section' : - load_section( elem, self.tool_panel ) + self.load_section_tag_set( elem, self.tool_panel, tool_path ) elif elem.tag == 'label': - load_label( elem, self.tool_panel ) - - def load_tool( self, config_file ): + self.load_label_tag_set( elem, self.tool_panel ) + def load_tool_tag_set( self, elem, panel_dict, tool_path, guid=None ): + try: + path = elem.get( "file" ) + tool = self.load_tool( os.path.join( tool_path, path ), guid=guid ) + if self.app.config.get_bool( 'enable_tool_tags', False ): + tag_names = elem.get( "tags", "" ).split( "," ) + for tag_name in tag_names: + if tag_name == '': + continue + tag = self.sa_session.query( self.app.model.Tag ).filter_by( name=tag_name ).first() + if not tag: + tag = self.app.model.Tag( name=tag_name ) + self.sa_session.add( tag ) + self.sa_session.flush() + tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id ) + self.sa_session.add( tta ) + self.sa_session.flush() + else: + for tagged_tool in tag.tagged_tools: + if tagged_tool.tool_id == tool.id: + break + else: + tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id ) + self.sa_session.add( tta ) + self.sa_session.flush() + if tool.id not in self.tools_by_id: + # Allow for the same tool to be loaded into multiple places in the + # tool panel. + self.tools_by_id[ tool.id ] = tool + key = 'tool_' + tool.id + panel_dict[ key ] = tool + log.debug( "Loaded tool: %s %s" % ( tool.id, tool.version ) ) + except: + log.exception( "error reading tool from path: %s" % path ) + def load_workflow_tag_set( self, elem, panel_dict ): + try: + # TODO: should id be encoded? + workflow_id = elem.get( 'id' ) + workflow = self.load_workflow( workflow_id ) + self.workflows_by_id[ workflow_id ] = workflow + key = 'workflow_' + workflow_id + panel_dict[ key ] = workflow + log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) + except: + log.exception( "error loading workflow: %s" % workflow_id ) + def load_label_tag_set( self, elem, panel_dict ): + label = ToolSectionLabel( elem ) + key = 'label_' + label.id + panel_dict[ key ] = label + def load_section_tag_set( self, elem, panel_dict, tool_path ): + key = 'section_' + elem.get( "id" ) + if key in panel_dict: + # Appending a tool to an existing section in self.tool_panel + elems = panel_dict[ key ].elems + log.debug( "Appending to section: %s" % elem.get( "name" ) ) + else: + # Appending a new section to self.tool_panel + section = ToolSection( elem ) + elems = section.elems + log.debug( "Loading section: %s" % section.name ) + for section_elem in elem: + if section_elem.tag == 'tool': + self.load_tool_tag_set( section_elem, elems, tool_path, guid=section_elem.get( 'guid' ) ) + elif section_elem.tag == 'workflow': + self.load_workflow_tag_set( section_elem, elems ) + elif section_elem.tag == 'label': + self.load_label_tag_set( section_elem, elems ) + if key not in panel_dict: + panel_dict[ key ] = section + def load_tool( self, config_file, guid=None ): """ Load a single tool from the file named by `config_file` and return an instance of `Tool`. @@ -160,38 +182,43 @@ class ToolBox( object ): type_elem = root.find( "type" ) module = type_elem.get( 'module', 'galaxy.tools' ) cls = type_elem.get( 'class' ) - mod = __import__( module, globals(), locals(), [cls]) + mod = __import__( module, globals(), locals(), [cls] ) ToolClass = getattr( mod, cls ) elif root.get( 'tool_type', None ) is not None: ToolClass = tool_types.get( root.get( 'tool_type' ) ) else: ToolClass = Tool - return ToolClass( config_file, root, self.app ) - - def reload( self, tool_id ): + return ToolClass( config_file, root, self.app, guid=guid ) + def reload_tool_by_id( self, tool_id ): """ Attempt to reload the tool identified by 'tool_id', if successful replace the old tool. """ if tool_id not in self.tools_by_id: - raise ToolNotFoundException( "No tool with id %s" % tool_id ) - old_tool = self.tools_by_id[ tool_id ] - new_tool = self.load_tool( old_tool.config_file ) - # Replace old_tool with new_tool in self.tool_panel - tool_key = 'tool_' + tool_id - for key, val in self.tool_panel.items(): - if key == tool_key: - self.tool_panel[ key ] = new_tool - break - elif key.startswith( 'section' ): - section = val - for section_key, section_val in section.elems.items(): - if section_key == tool_key: - self.tool_panel[ key ].elems[ section_key ] = new_tool - break - self.tools_by_id[ tool_id ] = new_tool - log.debug( "Reloaded tool %s %s" %( old_tool.id, old_tool.version ) ) - + message = "No tool with id %s" % tool_id + status = 'error' + else: + old_tool = self.tools_by_id[ tool_id ] + new_tool = self.load_tool( old_tool.config_file ) + # Replace old_tool with new_tool in self.tool_panel + tool_key = 'tool_' + tool_id + for key, val in self.tool_panel.items(): + if key == tool_key: + self.tool_panel[ key ] = new_tool + break + elif key.startswith( 'section' ): + section = val + for section_key, section_val in section.elems.items(): + if section_key == tool_key: + self.tool_panel[ key ].elems[ section_key ] = new_tool + break + self.tools_by_id[ tool_id ] = new_tool + message = "Reloaded the tool:
    " + message += "name: %s
    " % old_tool.name + message += "id: %s
    " % old_tool.id + message += "version: %s" % old_tool.version + status = 'done' + return message, status def load_workflow( self, workflow_id ): """ Return an instance of 'Workflow' identified by `id`, @@ -200,12 +227,11 @@ class ToolBox( object ): id = self.app.security.decode_id( workflow_id ) stored = self.app.model.context.query( self.app.model.StoredWorkflow ).get( id ) return stored.latest_workflow - def init_dependency_manager( self ): - self.dependency_manager = None if self.app.config.use_tool_dependencies: self.dependency_manager = DependencyManager( [ self.app.config.tool_dependency_dir ] ) - + else: + self.dependency_manager = None @property def sa_session( self ): """ @@ -314,12 +340,17 @@ class ToolRequirement( object ): """ Represents an external requirement that must be available for the tool to run (for example, a program, package, or library). Requirements can - optionally assert a specific version + optionally assert a specific version, or reference a command to execute a + fabric script. If fabric is used, the type is 'fabfile' and the version + attribute is not used since the fabric script includes all necessary + information for automatic dependency installation. """ - def __init__( self ): - self.name = None - self.type = None - self.version = None + def __init__( self, name=None, type=None, version=None, fabfile=None, method=None ): + self.name = name + self.type = type + self.version = version + self.fabfile = fabfile + self.method = method class ToolParallelismInfo(object): """ @@ -343,16 +374,21 @@ class Tool: tool_type = 'default' - def __init__( self, config_file, root, app ): - """ - Load a tool from the config named by `config_file` - """ + def __init__( self, config_file, root, app, guid=None ): + """Load a tool from the config named by `config_file`""" # Determine the full path of the directory where the tool config is self.config_file = config_file self.tool_dir = os.path.dirname( config_file ) self.app = app + # Define a place to keep track of all input parameters. These + # differ from the inputs dictionary in that inputs can be page + # elements like conditionals, but input_params are basic form + # parameters like SelectField objects. This enables us to more + # easily ensure that parameter dependencies like index files or + # tool_data_table_conf.xml entries exist. + self.input_params = [] # Parse XML element containing configuration - self.parse( root ) + self.parse( root, guid=guid ) @property def sa_session( self ): @@ -361,7 +397,7 @@ class Tool: """ return self.app.model.context - def parse( self, root ): + def parse( self, root, guid=None ): """ Read tool configuration from the element `root` and fill in `self`. """ @@ -371,7 +407,10 @@ class Tool: raise Exception, "Missing tool 'name'" # Get the UNIQUE id for the tool # TODO: can this be generated automatically? - self.id = root.get( "id" ) + if guid is not None: + self.id = guid + else: + self.id = root.get( "id" ) if not self.id: raise Exception, "Missing tool 'id'" self.version = root.get( "version" ) @@ -389,20 +428,15 @@ class Tool: self.input_translator = root.find( "request_param_translation" ) if self.input_translator: self.input_translator = ToolInputTranslator.from_element( self.input_translator ) - # Command line (template). Optional for tools that do not invoke a - # local program + # Command line (template). Optional for tools that do not invoke a local program command = root.find("command") if command is not None and command.text is not None: self.command = command.text.lstrip() # get rid of leading whitespace - interpreter = command.get("interpreter") - if interpreter: - # TODO: path munging for cluster/dataset server relocatability - executable = self.command.split()[0] - abs_executable = os.path.abspath(os.path.join(self.tool_dir, executable)) - self.command = self.command.replace(executable, abs_executable, 1) - self.command = interpreter + " " + self.command + # Must pre-pend this AFTER processing the cheetah command template + self.interpreter = command.get( "interpreter", None ) else: self.command = '' + self.interpreter = None # Parameters used to build URL for redirection to external app redirect_url_params = root.find( "redirect_url_params" ) if redirect_url_params is not None and redirect_url_params.text is not None: @@ -415,6 +449,11 @@ class Tool: self.redirect_url_params = '' # Short description of the tool self.description = util.xml_text(root, "description") + # Versioning for tools + self.version_string_cmd = None + version_cmd = root.find("version_command") + if version_cmd is not None: + self.version_string_cmd = version_cmd.text # Parallelism for tasks, read from tool config. parallelism = root.find("parallelism") if parallelism is not None and parallelism.get("method"): @@ -503,6 +542,8 @@ class Tool: self.parse_requirements( requirements_elem ) # Determine if this tool can be used in workflows self.is_workflow_compatible = self.check_workflow_compatible() + # Trackster configuration. + self.trackster_conf = ( root.find( "trackster_conf" ) is not None ) def parse_inputs( self, root ): """ @@ -677,8 +718,27 @@ class Tool: name = attrib.pop( 'name', None ) if name is None: raise Exception( "Test output does not have a 'name'" ) + assert_elem = output_elem.find("assert_contents") + assert_list = None + # Trying to keep testing patch as localized as + # possible, this function should be relocated + # somewhere more conventional. + def convert_elem(elem): + """ Converts and XML element to a dictionary format, used by assertion checking code. """ + tag = elem.tag + attributes = dict( elem.attrib ) + child_elems = list( elem.getchildren() ) + converted_children = [] + for child_elem in child_elems: + converted_children.append( convert_elem(child_elem) ) + return {"tag" : tag, "attributes" : attributes, "children" : converted_children} + if assert_elem is not None: + assert_list = [] + for assert_child in list(assert_elem): + assert_list.append(convert_elem(assert_child)) file = attrib.pop( 'file', None ) - if file is None: + # File no longer required if an list of assertions was present. + if assert_list is None and file is None: raise Exception( "Test output does not have a 'file'") attributes = {} # Method of comparison @@ -689,6 +749,7 @@ class Tool: attributes['delta'] = int( attrib.pop( 'delta', '10000' ) ) attributes['sort'] = util.string_as_bool( attrib.pop( 'sort', False ) ) attributes['extra_files'] = [] + attributes['assert_list'] = assert_list if 'ftype' in attrib: attributes['ftype'] = attrib['ftype'] for extra in output_elem.findall( 'extra_files' ): @@ -771,13 +832,14 @@ class Tool: case.inputs = self.parse_input_elem( ElementTree.XML( "%s" % case_inputs ), enctypes, context ) else: - case.inputs = {} + case.inputs = odict() group.cases.append( case ) else: # Should have one child "input" which determines the case input_elem = elem.find( "param" ) assert input_elem is not None, " must have a child " group.test_param = self.parse_param_elem( input_elem, enctypes, context ) + possible_cases = list( group.test_param.legal_values ) #store possible cases, undefined whens will have no inputs # Must refresh when test_param changes group.test_param.refresh_on_change = True # And a set of possible cases @@ -786,6 +848,16 @@ class Tool: case.value = case_elem.get( "value" ) case.inputs = self.parse_input_elem( case_elem, enctypes, context ) group.cases.append( case ) + try: + possible_cases.remove( case.value ) + except: + log.warning( "A when tag has been defined for '%s (%s) --> %s', but does not appear to be selectable." % ( group.name, group.test_param.name, case.value ) ) + for unspecified_case in possible_cases: + log.warning( "A when tag has not been defined for '%s (%s) --> %s', assuming empty inputs." % ( group.name, group.test_param.name, unspecified_case ) ) + case = ConditionalWhen() + case.value = unspecified_case + case.inputs = odict() + group.cases.append( case ) rval[group.name] = group elif elem.tag == "upload_dataset": group = UploadDataset() @@ -802,6 +874,9 @@ class Tool: elif elem.tag == "param": param = self.parse_param_elem( elem, enctypes, context ) rval[param.name] = param + if hasattr( param, 'data_ref' ): + param.ref_input = context[ param.data_ref ] + self.input_params.append( param ) return rval def parse_param_elem( self, input_elem, enctypes, context ): @@ -826,10 +901,21 @@ class Tool: self.requirements """ for requirement_elem in requirements_elem.findall( 'requirement' ): - requirement = ToolRequirement() - requirement.name = util.xml_text( requirement_elem ) - requirement.type = requirement_elem.get( "type", "package" ) - requirement.version = requirement_elem.get( "version" ) + name = util.xml_text( requirement_elem ) + type = requirement_elem.get( "type", "package" ) + if type == 'fabfile': + # The fabric script will include all necessary information for + # automatically installing the tool dependencies. + fabfile = requirement_elem.get( "fabfile" ) + method = requirement_elem.get( "method" ) + version = None + else: + # For backward compatibility, requirements tag sets should not require the + # use of a fabric script. + version = requirement_elem.get( "version" ) + fabfile = None + method = None + requirement = ToolRequirement( name=name, type=type, version=version, fabfile=fabfile, method=method ) self.requirements.append( requirement ) def check_workflow_compatible( self ): @@ -1249,7 +1335,32 @@ class Tool: errors[ input.name ] = error state[ input.name ] = value return errors - + @property + def params_with_missing_data_table_entry( self ): + """ + Return all parameters that are dynamically generated select lists whose + options require an entry not currently in the tool_data_table_conf.xml file. + """ + params = [] + for input_param in self.input_params: + if isinstance( input_param, basic.SelectToolParameter ) and input_param.is_dynamic: + options = input_param.options + if options and options.missing_tool_data_table_name and input_param not in params: + params.append( input_param ) + return params + @property + def params_with_missing_index_file( self ): + """ + Return all parameters that are dynamically generated + select lists whose options refer to a missing .loc file. + """ + params = [] + for input_param in self.input_params: + if isinstance( input_param, basic.SelectToolParameter ) and input_param.is_dynamic: + options = input_param.options + if options and options.missing_index_file and input_param not in params: + params.append( input_param ) + return params def get_static_param_values( self, trans ): """ Returns a map of parameter names and values if the tool does not @@ -1463,6 +1574,11 @@ class Tool: elif isinstance( input, SelectToolParameter ): input_values[ input.name ] = SelectToolParameterWrapper( input, input_values[ input.name ], self.app, other_values = param_dict ) + + elif isinstance( input, LibraryDatasetToolParameter ): + input_values[ input.name ] = LibraryDatasetValueWrapper( + input, input_values[ input.name ], param_dict ) + else: input_values[ input.name ] = InputValueWrapper( input, input_values[ input.name ], param_dict ) @@ -1588,12 +1704,18 @@ class Tool: try: # Substituting parameters into the command command_line = fill_template( self.command, context=param_dict ) - # Remove newlines from command line - command_line = command_line.replace( "\n", " " ).replace( "\r", " " ) + # Remove newlines from command line, and any leading/trailing white space + command_line = command_line.replace( "\n", " " ).replace( "\r", " " ).strip() except Exception, e: # Modify exception message to be more clear #e.args = ( 'Error substituting into command line. Params: %r, Command: %s' % ( param_dict, self.command ) ) raise + if self.interpreter: + # TODO: path munging for cluster/dataset server relocatability + executable = command_line.split()[0] + abs_executable = os.path.abspath(os.path.join(self.tool_dir, executable)) + command_line = command_line.replace(executable, abs_executable, 1) + command_line = self.interpreter + " " + command_line return command_line def build_dependency_shell_commands( self ): @@ -1921,6 +2043,7 @@ class DataSourceTool( Tool ): data_dict = dict( out_data_name = out_name, ext = data.ext, dataset_id = data.dataset.id, + hda_id = data.id, file_name = file_name, extra_files_path = extra_files_path ) @@ -2007,6 +2130,27 @@ class RawObjectWrapper( object ): def __getattr__( self, key ): return getattr( self.obj, key ) +class LibraryDatasetValueWrapper( object ): + """ + Wraps an input so that __str__ gives the "param_dict" representation. + """ + def __init__( self, input, value, other_values={} ): + self.input = input + self.value = value + self._other_values = other_values + self.counter = 0 + def __str__( self ): + return self.value + def __iter__( self ): + return self + def next( self ): + if self.counter >= len(self.value): + raise StopIteration + self.counter += 1 + return self.value[self.counter-1] + def __getattr__( self, key ): + return getattr( self.value, key ) + class InputValueWrapper( object ): """ Wraps an input so that __str__ gives the "param_dict" representation. diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index 3dbe8b6821f..000b6980716 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -37,7 +37,7 @@ class DefaultToolAction( object ): if data and not isinstance( data.datatype, formats ): # Need to refresh in case this conversion just took place, i.e. input above in tool performed the same conversion trans.sa_session.refresh( data ) - target_ext, converted_dataset = data.find_conversion_destination( formats, converter_safe = input.converter_safe( param_values, trans ) ) + target_ext, converted_dataset = data.find_conversion_destination( formats ) if target_ext: if converted_dataset: data = converted_dataset diff --git a/lib/galaxy/tools/actions/metadata.py b/lib/galaxy/tools/actions/metadata.py index cb2b0ddbf36..18849849d25 100644 --- a/lib/galaxy/tools/actions/metadata.py +++ b/lib/galaxy/tools/actions/metadata.py @@ -13,6 +13,12 @@ class SetMetadataToolAction( ToolAction ): if isinstance( value, trans.app.model.HistoryDatasetAssociation ): dataset = value dataset_name = name + type = 'hda' + break + elif isinstance( value, trans.app.model.LibraryDatasetDatasetAssociation ): + dataset = value + dataset_name = name + type = 'ldda' break else: raise Exception( 'The dataset to set metadata on could not be determined.' ) @@ -22,6 +28,8 @@ class SetMetadataToolAction( ToolAction ): job.session_id = trans.get_galaxy_session().id job.history_id = trans.history.id job.tool_id = tool.id + if trans.user: + job.user_id = trans.user.id start_job_state = job.state #should be job.states.NEW try: # For backward compatibility, some tools may not have versions yet. @@ -50,7 +58,10 @@ class SetMetadataToolAction( ToolAction ): for name, value in tool.params_to_strings( incoming, trans.app ).iteritems(): job.add_parameter( name, value ) #add the dataset to job_to_input_dataset table - job.add_input_dataset( dataset_name, dataset ) + if type == 'hda': + job.add_input_dataset( dataset_name, dataset ) + elif type == 'ldda': + job.add_input_library_dataset( dataset_name, dataset ) #Need a special state here to show that metadata is being set and also allow the job to run # i.e. if state was set to 'running' the set metadata job would never run, as it would wait for input (the dataset to set metadata on) to be in a ready state dataset._state = dataset.states.SETTING_METADATA diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index 4a842c29666..c6b7bb3f53e 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -143,7 +143,7 @@ def new_library_upload( trans, cntrller, uploaded_dataset, library_bunch, state= folder.add_folder( new_folder ) trans.sa_session.add( new_folder ) trans.sa_session.flush() - trans.app.security_agent.copy_library_permissions( folder, new_folder ) + trans.app.security_agent.copy_library_permissions( trans, folder, new_folder ) folder = new_folder if library_bunch.replace_dataset: ld = library_bunch.replace_dataset @@ -151,7 +151,7 @@ def new_library_upload( trans, cntrller, uploaded_dataset, library_bunch, state= ld = trans.app.model.LibraryDataset( folder=folder, name=uploaded_dataset.name ) trans.sa_session.add( ld ) trans.sa_session.flush() - trans.app.security_agent.copy_library_permissions( folder, ld ) + trans.app.security_agent.copy_library_permissions( trans, folder, ld ) ldda = trans.app.model.LibraryDatasetDatasetAssociation( name = uploaded_dataset.name, extension = uploaded_dataset.file_type, dbkey = uploaded_dataset.dbkey, @@ -167,7 +167,7 @@ def new_library_upload( trans, cntrller, uploaded_dataset, library_bunch, state= ldda.message = library_bunch.message trans.sa_session.flush() # Permissions must be the same on the LibraryDatasetDatasetAssociation and the associated LibraryDataset - trans.app.security_agent.copy_library_permissions( ld, ldda ) + trans.app.security_agent.copy_library_permissions( trans, ld, ldda ) if library_bunch.replace_dataset: # Copy the Dataset level permissions from replace_dataset to the new LibraryDatasetDatasetAssociation.dataset trans.app.security_agent.copy_dataset_permissions( library_bunch.replace_dataset.library_dataset_dataset_association.dataset, ldda.dataset ) @@ -322,9 +322,15 @@ def create_job( trans, params, tool, json_file_path, data_list, folder=None, ret if folder: for i, dataset in enumerate( data_list ): job.add_output_library_dataset( 'output%i' % i, dataset ) + # Create an empty file immediately + if not dataset.dataset.external_filename: + open( dataset.file_name, "w" ).close() else: for i, dataset in enumerate( data_list ): job.add_output_dataset( 'output%i' % i, dataset ) + # Create an empty file immediately + if not dataset.dataset.external_filename: + open( dataset.file_name, "w" ).close() job.state = job.states.NEW trans.sa_session.add( job ) trans.sa_session.flush() diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py index 45e9a634a2b..9c60ecb375a 100644 --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -12,34 +12,64 @@ from galaxy import util log = logging.getLogger( __name__ ) class ToolDataTableManager( object ): - """ - Manages a collection of tool data tables - """ - + """Manages a collection of tool data tables""" def __init__( self, config_filename=None ): self.data_tables = {} if config_filename: - self.add_from_config_file( config_filename ) - + self.load_from_config_file( config_filename ) def __getitem__( self, key ): return self.data_tables.__getitem__( key ) - def __contains__( self, key ): return self.data_tables.__contains__( key ) - - def add_from_config_file( self, config_filename ): + def load_from_config_file( self, config_filename ): tree = util.parse_xml( config_filename ) root = tree.getroot() + table_elems = [] for table_elem in root.findall( 'table' ): type = table_elem.get( 'type', 'tabular' ) assert type in tool_data_table_types, "Unknown data table type '%s'" % type + table_elems.append( table_elem ) table = tool_data_table_types[ type ]( table_elem ) - self.data_tables[ table.name ] = table - log.debug( "Loaded tool data table '%s", table.name ) + if table.name not in self.data_tables: + self.data_tables[ table.name ] = table + log.debug( "Loaded tool data table '%s", table.name ) + return table_elems + def add_new_entries_from_config_file( self, config_filename ): + """ + We have 2 cases to handle, files whose root tag is , for example: + + + + value, dbkey, name, path + +
    +     + and files whose root tag is , for example: + +
    + value, dbkey, name, path + +
    + """ + tree = util.parse_xml( config_filename ) + root = tree.getroot() + if root.tag == 'tables': + table_elems = self.load_from_config_file( config_filename ) + else: + table_elems = [] + type = root.get( 'type', 'tabular' ) + assert type in tool_data_table_types, "Unknown data table type '%s'" % type + table_elems.append( root ) + table = tool_data_table_types[ type ]( root ) + if table.name not in self.data_tables: + self.data_tables[ table.name ] = table + log.debug( "Loaded tool data table '%s", table.name ) + return table_elems class ToolDataTable( object ): def __init__( self, config_element ): self.name = config_element.get( 'name' ) + self.missing_index_file = None class TabularToolDataTable( ToolDataTable ): """ @@ -58,7 +88,6 @@ class TabularToolDataTable( ToolDataTable ): def __init__( self, config_element ): super( TabularToolDataTable, self ).__init__( config_element ) self.configure_and_load( config_element ) - def configure_and_load( self, config_element ): """ Configure and load table from an XML element. @@ -71,15 +100,17 @@ class TabularToolDataTable( ToolDataTable ): all_rows = [] for file_element in config_element.findall( 'file' ): filename = file_element.get( 'path' ) - if not os.path.exists( filename ): - log.warn( "Cannot find index file '%s' for tool data table '%s'" % ( filename, self.name ) ) - else: + if os.path.exists( filename ): all_rows.extend( self.parse_file_fields( open( filename ) ) ) + else: + self.missing_index_file = filename + log.warn( "Cannot find index file '%s' for tool data table '%s'" % ( filename, self.name ) ) self.data = all_rows - + def handle_found_index_file( self, filename ): + self.missing_index_file = None + self.data.extend( self.parse_file_fields( open( filename ) ) ) def get_fields( self ): return self.data - def parse_column_spec( self, config_element ): """ Parse column definitions, which can either be a set of 'column' elements @@ -109,7 +140,6 @@ class TabularToolDataTable( ToolDataTable ): assert 'value' in self.columns, "Required 'value' column missing from column def" if 'name' not in self.columns: self.columns['name'] = self.columns['value'] - def parse_file_fields( self, reader ): """ Parse separated lines from file and return a list of tuples. diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index 7449f5563d7..4f26d2ad3b2 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -18,11 +18,10 @@ class DependencyManager( object ): and should each contain a file 'env.sh' which can be sourced to make the dependency available in the current shell environment. """ - def __init__( self, base_paths=[] ): """ - Create a new dependency manager looking for packages under the - paths listed in `base_paths`. + Create a new dependency manager looking for packages under the paths listed + in `base_paths`. The default base path is app.config.tool_dependency_dir. """ self.base_paths = [] for base_path in base_paths: @@ -31,19 +30,17 @@ class DependencyManager( object ): if not os.path.isdir( base_path ): log.warn( "Path '%s' is not directory, ignoring", base_path ) self.base_paths.append( os.path.abspath( base_path ) ) - def find_dep( self, name, version=None ): """ Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a symbolic link (if found). Returns a triple of: - env_script, base_path, real_version + env_script, base_path, real_version """ if version is None: return self._find_dep_default( name ) else: return self._find_dep_versioned( name, version ) - def _find_dep_versioned( self, name, version ): for base_path in self.base_paths: path = os.path.join( base_path, name, version ) @@ -52,7 +49,6 @@ class DependencyManager( object ): return script, path, version else: return None, None, None - def _find_dep_default( self, name ): version = None for base_path in self.base_paths: @@ -65,5 +61,3 @@ class DependencyManager( object ): return script, real_path, real_version else: return None, None, None - - diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index a762a66f145..8d298a68ea0 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -36,6 +36,8 @@ class ToolParameter( object ): self.html = "no html set" self.repeat = param.get("repeat", None) self.condition = param.get( "condition", None ) + # Optional DataToolParameters are used in tools like GMAJ and LAJ + self.optional = string_as_bool( param.get( 'optional', False ) ) self.validators = [] for elem in param.findall("validator"): self.validators.append( validation.Validator.from_element( self, elem ) ) @@ -131,7 +133,11 @@ class ToolParameter( object ): return value def to_param_dict_string( self, value, other_values={} ): - value = str( value ) + """Called via __str__ when used in the Cheetah template""" + if value is None: + value = "" + else: + value = str( value ) if self.tool is None or self.tool.options.sanitize: if self.sanitizer: value = self.sanitizer.sanitize_param( value ) @@ -140,6 +146,8 @@ class ToolParameter( object ): return value def validate( self, value, history=None ): + if value=="" and self.optional: + return for validator in self.validators: validator.validate( value, history ) @@ -226,9 +234,22 @@ class IntegerToolParameter( TextToolParameter ): try: return int( value ) except: + if not value and self.optional: + return "" raise ValueError( "An integer is required" ) + def to_string( self, value, app ): + """Convert a value to a string representation suitable for persisting""" + if value is None: + return "" + else: + return str( value ) def to_python( self, value, app ): - return int( value ) + try: + return int( value ) + except Exception, err: + if not value and self.optional: + return None + raise err def get_initial_value( self, trans, context ): if self.value: return int( self.value ) @@ -281,10 +302,23 @@ class FloatToolParameter( TextToolParameter ): def from_html( self, value, trans=None, other_values={} ): try: return float( value ) - except: + except: + if not value and self.optional: + return "" raise ValueError( "A real number is required" ) + def to_string( self, value, app ): + """Convert a value to a string representation suitable for persisting""" + if value is None: + return "" + else: + return str( value ) def to_python( self, value, app ): - return float( value ) + try: + return float( value ) + except Exception, err: + if not value and self.optional: + return None + raise err def get_initial_value( self, trans, context ): try: return float( self.value ) @@ -318,7 +352,7 @@ class BooleanToolParameter( ToolParameter ): checked = self.checked if value is not None: checked = form_builder.CheckboxField.is_checked( value ) - return form_builder.CheckboxField( self.name, checked ) + return form_builder.CheckboxField( self.name, checked, refresh_on_change = self.refresh_on_change ) def from_html( self, value, trans=None, other_values={} ): return form_builder.CheckboxField.is_checked( value ) def to_python( self, value, app ): @@ -330,6 +364,9 @@ class BooleanToolParameter( ToolParameter ): return self.truevalue else: return self.falsevalue + @property + def legal_values( self ): + return [ self.truevalue, self.falsevalue ] class FileToolParameter( ToolParameter ): """ @@ -417,7 +454,11 @@ class FTPFileToolParameter( ToolParameter ): user_ftp_dir = os.path.join( trans.app.config.ftp_upload_dir, trans.user.email ) return form_builder.FTPFileField( self.name, user_ftp_dir, trans.app.config.ftp_upload_site, value = value ) def from_html( self, value, trans=None, other_values={} ): - return util.listify( value ) + try: + assert type( value ) is list + except: + value = [ value ] + return value def to_string( self, value, app ): if value in [ None, '' ]: return None @@ -600,8 +641,9 @@ class SelectToolParameter( ToolParameter ): # Dynamic options are not yet supported in workflow, allow # specifying the value as text for now. if self.need_late_validation( trans, context ): - assert isinstance( value, UnvalidatedValue ) - value = value.value + if value is not None: + assert isinstance( value, UnvalidatedValue ), "Late validation needed for '%s', but provided value (%s) is not of type UnvalidatedValue (%s)." % ( self.name, value, type( value ) ) + value = value.value if self.multiple: if value is None: value = "" @@ -641,13 +683,11 @@ class SelectToolParameter( ToolParameter ): assert self.multiple, "Multiple values provided but parameter is not expecting multiple values" rval = [] for v in value: - v = util.restore_text( v ) if v not in legal_values: raise ValueError( "An invalid option was selected, please verify" ) rval.append( v ) return rval else: - value = util.restore_text( value ) if value not in legal_values: raise ValueError( "An invalid option was selected, please verify" ) return value @@ -689,7 +729,7 @@ class SelectToolParameter( ToolParameter ): dynamic options, we need to check whether the other parameters which determine what options are valid have been set. For the old style dynamic options which do not specify dependencies, this is always true - (must valiate at runtime). + (must validate at runtime). """ # Option list is statically defined, never need late validation if not self.is_dynamic: @@ -697,8 +737,8 @@ class SelectToolParameter( ToolParameter ): # Old style dynamic options, no dependency information so there isn't # a lot we can do: if we're dealing with workflows, have to assume # late validation no matter what. - if self.dynamic_options is not None: - return ( trans is None or trans.workflow_building_mode ) + if self.dynamic_options is not None and ( trans is None or trans.workflow_building_mode ): + return True # If we got this far, we can actually look at the dependencies # to see if their values will not be available until runtime. for dep_name in self.get_dependencies(): @@ -712,6 +752,9 @@ class SelectToolParameter( ToolParameter ): # Dependency on a value that does not yet exist if isinstance( dep_value, RuntimeValue ): return True + #dataset not ready yet + if hasattr( self, 'ref_input' ) and isinstance( dep_value, self.tool.app.model.HistoryDatasetAssociation ) and ( dep_value.is_pending or not isinstance( dep_value.datatype, self.ref_input.formats ) ): + return True # Dynamic, but all dependenceis are known and have values return False def get_initial_value( self, trans, context ): @@ -765,7 +808,7 @@ class GenomeBuildParameter( SelectToolParameter ): Select list that sets the last used genome build for the current history as "selected". - >>> # Create a mock transcation with 'hg17' as the current build + >>> # Create a mock transaction with 'hg17' as the current build >>> from galaxy.util.bunch import Bunch >>> trans = Bunch( history=Bunch( genome_build='hg17' ), db_builds=util.dbnames ) @@ -801,10 +844,15 @@ class GenomeBuildParameter( SelectToolParameter ): hg17 """ def get_options( self, trans, other_values ): - last_used_build = trans.history.genome_build - for dbkey, build_name in trans.db_builds: - yield build_name, dbkey, ( dbkey == last_used_build ) + if not trans.history: + yield 'unspecified', '?', False + else: + last_used_build = trans.history.genome_build + for dbkey, build_name in trans.db_builds: + yield build_name, dbkey, ( dbkey == last_used_build ) def get_legal_values( self, trans, other_values ): + if not trans.history: + return set( '?' ) return set( dbkey for dbkey, _ in trans.db_builds ) class ColumnListParameter( SelectToolParameter ): @@ -837,6 +885,7 @@ class ColumnListParameter( SelectToolParameter ): self.force_select = string_as_bool( elem.get( "force_select", True )) self.accept_default = string_as_bool( elem.get( "accept_default", False )) self.data_ref = elem.get( "data_ref", None ) + self.ref_input = None self.default_value = elem.get( "default_value", None ) self.is_dynamic = True def from_html( self, value, trans=None, context={} ): @@ -932,7 +981,7 @@ class ColumnListParameter( SelectToolParameter ): if not dataset.metadata.columns: # Only allow late validation if the dataset is not yet ready # (since we have reason to expect the metadata to be ready eventually) - if dataset.is_pending: + if dataset.is_pending or not isinstance( dataset.datatype, self.ref_input.formats ): return True # No late validation return False @@ -1091,7 +1140,7 @@ class DrillDownSelectToolParameter( SelectToolParameter ): # specifying the value as text for now. if self.need_late_validation( trans, other_values ): if value is not None: - assert isinstance( value, UnvalidatedValue ) + assert isinstance( value, UnvalidatedValue ), "Late validation needed for '%s', but provided value (%s) is not of type UnvalidatedValue (%s)." % ( self.name, value, type( value ) ) value = value.value if self.multiple: if value is None: @@ -1119,7 +1168,7 @@ class DrillDownSelectToolParameter( SelectToolParameter ): rval = [] for val in value: if val not in self.get_legal_values( trans, other_values ): raise ValueError( "An invalid option was selected, please verify" ) - rval.append( util.restore_text( val ) ) + rval.append( val ) return rval def to_param_dict_string( self, value, other_values={} ): @@ -1227,7 +1276,7 @@ class DataToolParameter( ToolParameter ): displayed as radio buttons and multiple selects as a set of checkboxes TODO: The following must be fixed to test correctly for the new security_check tag in the DataToolParameter ( the last test below is broken ) - Nate's next passs at the dataset security stuff will dramatically alter this anyway. + Nate's next pass at the dataset security stuff will dramatically alter this anyway. """ def __init__( self, tool, elem ): @@ -1248,8 +1297,6 @@ class DataToolParameter( ToolParameter ): formats.append( tool.app.datatypes_registry.get_datatype_by_extension( extension.lower() ).__class__ ) self.formats = tuple( formats ) self.multiple = string_as_bool( elem.get( 'multiple', False ) ) - # Optional DataToolParameters are used in tools like GMAJ and LAJ - self.optional = string_as_bool( elem.get( 'optional', False ) ) # TODO: Enhance dynamic options for DataToolParameters. Currently, # only the special case key='build' of type='data_meta' is # a valid filter @@ -1314,7 +1361,7 @@ class DataToolParameter( ToolParameter ): selected = ( value and ( hda in value ) ) field.add_option( "%s: %s" % ( hid, hda_name ), hda.id, selected ) else: - target_ext, converted_dataset = hda.find_conversion_destination( self.formats, converter_safe = self.converter_safe( other_values, trans ) ) + target_ext, converted_dataset = hda.find_conversion_destination( self.formats ) if target_ext: if converted_dataset: hda = converted_dataset @@ -1347,7 +1394,7 @@ class DataToolParameter( ToolParameter ): happens twice (here and when generating HTML). """ # Can't look at history in workflow mode - if trans.workflow_building_mode: + if trans is None or trans.workflow_building_mode: return DummyDataset() assert trans is not None, "DataToolParameter requires a trans" history = trans.get_history() @@ -1363,13 +1410,22 @@ class DataToolParameter( ToolParameter ): pass #no valid options def dataset_collector( datasets ): def is_convertable( dataset ): - target_ext, converted_dataset = dataset.find_conversion_destination( self.formats, converter_safe = self.converter_safe( None, trans ) ) + target_ext, converted_dataset = dataset.find_conversion_destination( self.formats ) if target_ext is not None: return True return False for i, data in enumerate( datasets ): - if data.visible and not data.deleted and data.state not in [data.states.ERROR, data.states.DISCARDED] and ( isinstance( data.datatype, self.formats) or is_convertable( data ) ): - if self.options and self._options_filter_attribute( data ) != filter_value: + if data.visible and not data.deleted and data.state not in [data.states.ERROR, data.states.DISCARDED]: + is_valid = False + if isinstance( data.datatype, self.formats ): + is_valid = True + else: + target_ext, converted_dataset = data.find_conversion_destination( self.formats ) + if target_ext: + is_valid = True + if converted_dataset: + data = converted_dataset + if not is_valid or ( self.options and self._options_filter_attribute( data ) != filter_value ): continue most_recent_dataset[0] = data # Also collect children via association object @@ -1470,6 +1526,44 @@ class DataToolParameter( ToolParameter ): ref = ref() return ref +class LibraryDatasetToolParameter( ToolParameter ): + """ + Parameter that lets users select a LDDA from a modal window, then use it within the wrapper. + """ + + def __init__( self, tool, elem ): + ToolParameter.__init__( self, tool, elem ) + + def get_html_field( self, trans=None, value=None, other_values={} ): + return form_builder.LibraryField( self.name, value=value, trans=trans ) + + def get_initial_value( self, trans, context ): + return None + + def from_html( self, value, trans, other_values={} ): + if not value: + return None + elif isinstance( value, list ): + return value + else: + decoded_lst = [] + for encoded_id in value.split("||"): + decoded_lst.append( trans.sa_session.query( trans.app.model.LibraryDatasetDatasetAssociation ).get( trans.security.decode_id( encoded_id ) ) ) + return decoded_lst + + def to_string( self, value, app ): + if not value: + return value + return [ldda.id for ldda in value] + + def to_python( self, value, app ): + if not value: + return value + lddas = [] + for ldda_id in value: + lddas.append( app.model.context.query( app.model.LibraryDatasetDatasetAssociation ).get( ldda_id ) ) + return lddas + # class RawToolParameter( ToolParameter ): # """ # Completely nondescript parameter, HTML representation is provided as text @@ -1518,19 +1612,20 @@ class DataToolParameter( ToolParameter ): # self.html = form_builder.HiddenField( self.name, trans.history.id ).get_html() # return self.html -parameter_types = dict( text = TextToolParameter, - integer = IntegerToolParameter, - float = FloatToolParameter, - boolean = BooleanToolParameter, - genomebuild = GenomeBuildParameter, - select = SelectToolParameter, - data_column = ColumnListParameter, - hidden = HiddenToolParameter, - baseurl = BaseURLToolParameter, - file = FileToolParameter, - ftpfile = FTPFileToolParameter, - data = DataToolParameter, - drill_down = DrillDownSelectToolParameter ) +parameter_types = dict( text = TextToolParameter, + integer = IntegerToolParameter, + float = FloatToolParameter, + boolean = BooleanToolParameter, + genomebuild = GenomeBuildParameter, + select = SelectToolParameter, + data_column = ColumnListParameter, + hidden = HiddenToolParameter, + baseurl = BaseURLToolParameter, + file = FileToolParameter, + ftpfile = FTPFileToolParameter, + data = DataToolParameter, + library_data = LibraryDatasetToolParameter, + drill_down = DrillDownSelectToolParameter ) class UnvalidatedValue( object ): """ diff --git a/lib/galaxy/tools/parameters/dynamic_options.py b/lib/galaxy/tools/parameters/dynamic_options.py index d0ff3d7df83..f4b86fa772d 100644 --- a/lib/galaxy/tools/parameters/dynamic_options.py +++ b/lib/galaxy/tools/parameters/dynamic_options.py @@ -399,32 +399,44 @@ class DynamicOptions( object ): self.separator = elem.get( 'separator', '\t' ) self.line_startswith = elem.get( 'startswith', None ) data_file = elem.get( 'from_file', None ) + self.index_file = None + self.missing_index_file = None dataset_file = elem.get( 'from_dataset', None ) from_parameter = elem.get( 'from_parameter', None ) tool_data_table_name = elem.get( 'from_data_table', None ) - # Options are defined from a data table loaded by the app self.tool_data_table = None + self.missing_tool_data_table_name = None if tool_data_table_name: app = tool_param.tool.app - assert tool_data_table_name in app.tool_data_tables, \ - "Data table named '%s' is required by tool but not configured" % tool_data_table_name - self.tool_data_table = app.tool_data_tables[ tool_data_table_name ] - # Column definitions are optional, but if provided override those from the table - if elem.find( "column" ) is not None: - self.parse_column_definitions( elem ) + if tool_data_table_name in app.tool_data_tables: + self.tool_data_table = app.tool_data_tables[ tool_data_table_name ] + # Set self.missing_index_file if the index file to + # which the tool_data_table refers does not exist. + if self.tool_data_table.missing_index_file: + self.missing_index_file = self.tool_data_table.missing_index_file + else: + # Column definitions are optional, but if provided override those from the table + if elem.find( "column" ) is not None: + self.parse_column_definitions( elem ) + else: + self.columns = self.tool_data_table.columns else: - self.columns = self.tool_data_table.columns - - # Options are defined by parsing tabular text data from an data file + self.missing_tool_data_table_name = tool_data_table_name + log.warn( "Data table named '%s' is required by tool but not configured" % tool_data_table_name ) + # Options are defined by parsing tabular text data from a data file # on disk, a dataset, or the value of another parameter elif data_file is not None or dataset_file is not None or from_parameter is not None: self.parse_column_definitions( elem ) if data_file is not None: data_file = data_file.strip() if not os.path.isabs( data_file ): - data_file = os.path.join( self.tool_param.tool.app.config.tool_data_path, data_file ) - self.file_fields = self.parse_file_fields( open( data_file ) ) + full_path = os.path.join( self.tool_param.tool.app.config.tool_data_path, data_file ) + if os.path.exists( full_path ): + self.index_file = data_file + self.file_fields = self.parse_file_fields( open( full_path ) ) + else: + self.missing_index_file = data_file elif dataset_file is not None: self.dataset_ref_name = dataset_file self.has_dataset_dependencies = True @@ -440,6 +452,9 @@ class DynamicOptions( object ): # Load Validators for validator in elem.findall( 'validator' ): self.validators.append( validation.Validator.from_element( self.tool_param, validator ) ) + + if self.dataset_ref_name: + tool_param.data_ref = self.dataset_ref_name def parse_column_definitions( self, elem ): for column_elem in elem.findall( 'column' ): diff --git a/lib/galaxy/tools/parameters/grouping.py b/lib/galaxy/tools/parameters/grouping.py index 878086fd509..13a2e4b5091 100644 --- a/lib/galaxy/tools/parameters/grouping.py +++ b/lib/galaxy/tools/parameters/grouping.py @@ -414,7 +414,10 @@ class Conditional( Group ): return "Conditional (%s)" % self.name def get_current_case( self, value, trans ): # Convert value to user representation - str_value = self.test_param.filter_value( value, trans ) + if isinstance( value, bool ): + str_value = self.test_param.to_param_dict_string( value ) + else: + str_value = self.test_param.filter_value( value, trans ) # Find the matching case for index, case in enumerate( self.cases ): if str_value == case.value: diff --git a/lib/galaxy/tools/parameters/output.py b/lib/galaxy/tools/parameters/output.py index 9d406a164b8..43f9c2c7c2f 100644 --- a/lib/galaxy/tools/parameters/output.py +++ b/lib/galaxy/tools/parameters/output.py @@ -206,13 +206,16 @@ class FromDataTableOutputActionOption( ToolOutputActionOption ): super( FromDataTableOutputActionOption, self ).__init__( parent, elem ) self.name = elem.get( 'name', None ) assert self.name is not None, "Required 'name' attribute missing from FromDataTableOutputActionOption" - assert self.name in self.tool.app.tool_data_tables, "Data table named '%s' is required by tool but not configured" % self.name - self.options = self.tool.app.tool_data_tables[ self.name ].get_fields() - self.column = elem.get( 'column', None ) - assert self.column is not None, "Required 'column' attribute missing from FromDataTableOutputActionOption" - self.column = int( self.column ) - self.offset = elem.get( 'offset', -1 ) - self.offset = int( self.offset ) + self.missing_tool_data_table_name = None + if self.name in self.tool.app.tool_data_tables: + self.options = self.tool.app.tool_data_tables[ self.name ].get_fields() + self.column = elem.get( 'column', None ) + assert self.column is not None, "Required 'column' attribute missing from FromDataTableOutputActionOption" + self.column = int( self.column ) + self.offset = elem.get( 'offset', -1 ) + self.offset = int( self.offset ) + else: + self.missing_tool_data_table_name = self.name def get_value( self, other_values ): options = self.options for filter in self.filters: diff --git a/lib/galaxy/tools/parameters/sanitize.py b/lib/galaxy/tools/parameters/sanitize.py index c830ceb58e1..50420f93faa 100644 --- a/lib/galaxy/tools/parameters/sanitize.py +++ b/lib/galaxy/tools/parameters/sanitize.py @@ -19,7 +19,7 @@ class ToolParameterSanitizer( object ): ... ... ... ''' ) ) - >>> sanitizer.sanitize_param( string.printable ) == string.letters + >>> sanitizer.sanitize_param( ''.join( sorted( [ c for c in string.printable ] ) ) ) == ''.join( sorted( [ c for c in string.letters ] ) ) True >>> slash = chr( 92 ) >>> sanitizer = ToolParameterSanitizer.from_element( XML( diff --git a/lib/galaxy/tools/search/__init__.py b/lib/galaxy/tools/search/__init__.py index 3ae71ad4efb..e4fc1ad5a35 100644 --- a/lib/galaxy/tools/search/__init__.py +++ b/lib/galaxy/tools/search/__init__.py @@ -1,4 +1,5 @@ from galaxy.eggs import require +from galaxy.web.framework.helpers import to_unicode # Whoosh is compatible with Python 2.5+ Try to import Whoosh and set flag to indicate whether tool search is enabled. try: require( "Whoosh" ) @@ -35,12 +36,6 @@ class ToolBoxSearch( object ): writer = self.index.writer() ## TODO: would also be nice to search section headers. for id, tool in self.toolbox.tools_by_id.iteritems(): - def to_unicode( a_basestr ): - if type( a_basestr ) is str: - return unicode( a_basestr, 'utf-8' ) - else: - return a_basestr - writer.add_document( id=id, title=to_unicode(tool.name), description=to_unicode(tool.description), help=to_unicode(tool.help) ) writer.commit() diff --git a/lib/galaxy/tools/tool_shed_registry.py b/lib/galaxy/tools/tool_shed_registry.py new file mode 100644 index 00000000000..ddd60ddd770 --- /dev/null +++ b/lib/galaxy/tools/tool_shed_registry.py @@ -0,0 +1,31 @@ +import sys, logging +from galaxy.util import parse_xml +from galaxy.util.odict import odict + +log = logging.getLogger( __name__ ) + +if sys.version_info[:2] == ( 2, 4 ): + from galaxy import eggs + eggs.require( 'ElementTree' ) + from elementtree import ElementTree +else: + from xml.etree import ElementTree + +class Registry( object ): + def __init__( self, root_dir=None, config=None ): + self.tool_sheds = odict() + if root_dir and config: + # Parse datatypes_conf.xml + tree = parse_xml( config ) + root = tree.getroot() + # Load datatypes and converters from config + log.debug( 'Loading references to tool sheds from %s' % config ) + for elem in root.findall( 'tool_shed' ): + try: + name = elem.get( 'name', None ) + url = elem.get( 'url', None ) + if name and url: + self.tool_sheds[ name ] = url + log.debug( 'Loaded reference to tool shed: %s' % name ) + except Exception, e: + log.warning( 'Error loading reference to tool shed "%s", problem: %s' % ( name, str( e ) ) ) diff --git a/lib/galaxy/tools/util/maf_utilities.py b/lib/galaxy/tools/util/maf_utilities.py index cdcae1f54ab..dd1e8c2fe67 100644 --- a/lib/galaxy/tools/util/maf_utilities.py +++ b/lib/galaxy/tools/util/maf_utilities.py @@ -227,7 +227,7 @@ def build_maf_index_species_chromosomes( filename, index_species = None ): except Exception, e: #most likely a bad MAF log.debug( 'Building MAF index on %s failed: %s' % ( filename, e ) ) - return ( None, [], {} ) + return ( None, [], {}, 0 ) return ( indexes, species, species_chromosomes, blocks ) #builds and returns ( index, index_filename ) for specified maf_file diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 3261ac7d37f..cad925ec778 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -2,7 +2,8 @@ Utility functions used systemwide. """ -import logging, threading, random, string, re, binascii, pickle, time, datetime, math, re, os, sys, tempfile, stat, grp +import logging, threading, random, string, re, binascii, pickle, time, datetime, math, re, os, sys, tempfile, stat, grp, smtplib +from email.MIMEText import MIMEText # Older py compatibility try: @@ -126,7 +127,8 @@ mapped_chars = { '>' :'__gt__', '@' : '__at__', '\n' : '__cn__', '\r' : '__cr__', - '\t' : '__tc__' + '\t' : '__tc__', + '#' : '__pd__' } def restore_text(text): @@ -177,7 +179,7 @@ def sanitize_for_filename( text, default=None ): return default return out -class Params: +class Params( object ): """ Stores and 'sanitizes' parameters. Alphanumeric characters and the non-alphanumeric ones that are deemed safe are let to pass through (see L{valid_chars}). @@ -195,9 +197,9 @@ class Params: >>> par.get('price', 0) 0 >>> par.symbols # replaces unknown symbols with X - ['alpha', '__lt____gt__', 'XrmXX!'] + ['alpha', '__lt____gt__', 'XrmX__pd__!'] >>> par.flatten() # flattening to a list - [('status', 'on'), ('symbols', 'alpha'), ('symbols', '__lt____gt__'), ('symbols', 'XrmXX!')] + [('status', 'on'), ('symbols', 'alpha'), ('symbols', '__lt____gt__'), ('symbols', 'XrmX__pd__!')] """ # is NEVER_SANITIZE required now that sanitizing for tool parameters can be controlled on a per parameter basis and occurs via InputValueWrappers? @@ -526,7 +528,7 @@ def nice_size(size): >>> nice_size(100000000) '95.4 Mb' """ - words = [ 'bytes', 'Kb', 'Mb', 'Gb' ] + words = [ 'bytes', 'Kb', 'Mb', 'Gb', 'Tb' ] try: size = float( size ) except: @@ -540,6 +542,80 @@ def nice_size(size): return "%.1f %s" % (size, word) return '??? bytes' +def size_to_bytes( size ): + """ + Returns a number of bytes if given a reasonably formatted string with the size + """ + # Assume input in bytes if we can convert directly to an int + try: + return int( size ) + except: + pass + # Otherwise it must have non-numeric characters + size_re = re.compile( '([\d\.]+)\s*([tgmk]b?|b|bytes?)$' ) + size_match = re.match( size_re, size.lower() ) + assert size_match is not None + size = float( size_match.group(1) ) + multiple = size_match.group(2) + if multiple.startswith( 't' ): + return int( size * 1024**4 ) + elif multiple.startswith( 'g' ): + return int( size * 1024**3 ) + elif multiple.startswith( 'm' ): + return int( size * 1024**2 ) + elif multiple.startswith( 'k' ): + return int( size * 1024 ) + elif multiple.startswith( 'b' ): + return int( size ) + +def send_mail( frm, to, subject, body, config ): + """ + Sends an email. + """ + header_to = to + if isinstance( to, list ): + header_to = ', '.join( to ) + msg = MIMEText( body ) + msg[ 'To' ] = header_to + msg[ 'From' ] = frm + msg[ 'Subject' ] = subject + if config.smtp_server is None: + log.error( "Mail is not configured for this Galaxy instance." ) + log.info( msg ) + return + s = smtplib.SMTP() + s.connect( config.smtp_server ) + try: + s.starttls() + log.debug( 'Initiated SSL/TLS connection to SMTP server: %s' % config.smtp_server ) + except RuntimeError, e: + log.warning( 'SSL/TLS support is not available to your Python interpreter: %s' % e ) + except smtplib.SMTPHeloError, e: + log.error( "The server didn't reply properly to the HELO greeting: %s" % e ) + s.close() + raise + except smtplib.SMTPException, e: + log.warning( 'The server does not support the STARTTLS extension: %s' % e ) + if config.smtp_username and config.smtp_password: + try: + s.login( config.smtp_username, config.smtp_password ) + except smtplib.SMTPHeloError, e: + log.error( "The server didn't reply properly to the HELO greeting: %s" % e ) + s.close() + raise + except smtplib.SMTPAuthenticationError, e: + log.error( "The server didn't accept the username/password combination: %s" % e ) + s.close() + raise + except smtplib.SMTPError, e: + log.error( "No suitable authentication method was found: %s" % e ) + s.close() + raise + if isinstance( to, basestring ): + to = [ to ] + s.sendmail( frm, to, msg.as_string() ) + s.quit() + galaxy_root_path = os.path.join(__path__[0], "..","..","..") # The dbnames list is used in edit attributes and the upload tool dbnames = read_dbnames( os.path.join( galaxy_root_path, "tool-data", "shared", "ucsc", "builds.txt" ) ) diff --git a/lib/galaxy/util/expressions.py b/lib/galaxy/util/expressions.py index fe839cb77ee..88452602b16 100644 --- a/lib/galaxy/util/expressions.py +++ b/lib/galaxy/util/expressions.py @@ -21,6 +21,8 @@ class ExpressionContext( object, DictMixin ): if self.parent is not None and key in self.parent: return self.parent[key] raise KeyError( key ) + def __setitem__( self, key, value ): + self.dict[key] = value def __contains__( self, key ): if key in self.dict: return True @@ -29,4 +31,7 @@ class ExpressionContext( object, DictMixin ): return False def __str__( self ): return str( self.dict ) - + def __nonzero__( self ): + if not self.dict and not self.parent: + return False + return True diff --git a/lib/galaxy/util/sanitize_html.py b/lib/galaxy/util/sanitize_html.py index 10ec4d3f392..74e090cc81a 100644 --- a/lib/galaxy/util/sanitize_html.py +++ b/lib/galaxy/util/sanitize_html.py @@ -431,7 +431,7 @@ class _HTMLSanitizer(_BaseHTMLProcessor): return ' '.join(clean) -def sanitize_html(htmlSource, encoding, type): +def sanitize_html(htmlSource, encoding="utf-8", type="text/html"): p = _HTMLSanitizer(encoding, type) p.feed(htmlSource) data = p.output() diff --git a/lib/galaxy/visualization/tracks/data_providers.py b/lib/galaxy/visualization/tracks/data_providers.py index 2df14d9cfe4..787b4ad32e8 100644 --- a/lib/galaxy/visualization/tracks/data_providers.py +++ b/lib/galaxy/visualization/tracks/data_providers.py @@ -3,7 +3,7 @@ Data providers for tracks visualizations. """ import sys -from math import floor, ceil, log, pow +from math import ceil, log import pkg_resources pkg_resources.require( "bx-python" ) if sys.version_info[:2] == (2, 4): @@ -13,19 +13,16 @@ pkg_resources.require( "numpy" ) from galaxy.datatypes.util.gff_util import * from galaxy.util.json import from_json_string from bx.interval_index_file import Indexes -from bx.arrays.array_tree import FileArrayTreeDict from bx.bbi.bigwig_file import BigWigFile from galaxy.util.lrucache import LRUCache from galaxy.visualization.tracks.summary import * import galaxy_utils.sequence.vcf from galaxy.datatypes.tabular import Vcf from galaxy.datatypes.interval import Bed, Gff, Gtf -from galaxy.datatypes.util.gff_util import parse_gff_attributes from pysam import csamtools, ctabix -MAX_VALS = 5000 # only display first MAX_VALS features -ERROR_MAX_VALS = "Only the first " + str(MAX_VALS) + " %s in the region denoted by the red line are displayed." +ERROR_MAX_VALS = "Only the first %i %s in this region are displayed." # Return None instead of NaN to pass jQuery 1.4's strict JSON def float_nan(n): @@ -33,7 +30,20 @@ def float_nan(n): return None else: return float(n) - + +def get_bounds( reads, start_pos_index, end_pos_index ): + """ + Returns the minimum and maximum position for a set of reads. + """ + max_low = sys.maxint + max_high = -sys.maxint + for read in reads: + if read[ start_pos_index ] < max_low: + max_low = read[ start_pos_index ] + if read[ end_pos_index ] > max_high: + max_high = read[ end_pos_index ] + return max_low, max_high + class TracksDataProvider( object ): """ Base class for tracks data providers. """ @@ -73,8 +83,15 @@ class TracksDataProvider( object ): # Override. pass - def get_data( self, chrom, start, end, **kwargs ): - """ Returns data in region defined by chrom, start, and end. """ + def get_data( self, chrom, start, end, start_val=0, max_vals=None, **kwargs ): + """ + Returns data in region defined by chrom, start, and end. start_val and + max_vals are used to denote the data to return: start_val is the first element to + return and max_vals indicates the number of values to return. + + Return value must be a dictionary with the following attributes: + dataset_type, data + """ # Override. pass @@ -118,6 +135,88 @@ class TracksDataProvider( object ): { 'name' : attrs[ 'name' ], 'type' : column_types[viz_col_index], \ 'index' : attrs[ 'index' ] } ) return filters + +class BedDataProvider( TracksDataProvider ): + """ + Abstract class that processes BED data from text format to payload format. + + Payload format: [ uid (offset), start, end, name, strand, thick_start, thick_end, blocks ] + """ + + def get_iterator( self, chrom, start, end ): + raise "Unimplemented Method" + + def get_data( self, chrom, start, end, start_val=0, max_vals=None, **kwargs ): + iterator = self.get_iterator( chrom, start, end ) + return self.process_data( iterator, start_val, max_vals, **kwargs ) + + def process_data( self, iterator, start_val=0, max_vals=None, **kwargs ): + """ + Provides + """ + # Build data to return. Payload format is: + # [ , , , , , , , + # , ] + # + # First three entries are mandatory, others are optional. + # + filter_cols = from_json_string( kwargs.get( "filter_cols", "[]" ) ) + no_detail = ( "no_detail" in kwargs ) + rval = [] + message = None + for count, line in enumerate( iterator ): + if count < start_val: + continue + if max_vals and count-start_val >= max_vals: + message = ERROR_MAX_VALS % ( max_vals, "features" ) + break + # TODO: can we use column metadata to fill out payload? + # TODO: use function to set payload data + + feature = line.split() + length = len(feature) + # Unique id is just a hash of the line + payload = [ hash(line), int(feature[1]), int(feature[2]) ] + + if no_detail: + rval.append( payload ) + continue + + # Simpler way to add stuff, but type casting is not done. + # Name, score, strand, thick start, thick end. + #end = min( len( feature ), 8 ) + #payload.extend( feature[ 3:end ] ) + + # Name, strand, thick start, thick end. + if length >= 4: + payload.append(feature[3]) + if length >= 6: + payload.append(feature[5]) + if length >= 8: + payload.append(int(feature[6])) + payload.append(int(feature[7])) + + # Blocks. + if length >= 12: + block_sizes = [ int(n) for n in feature[10].split(',') if n != ''] + block_starts = [ int(n) for n in feature[11].split(',') if n != '' ] + blocks = zip( block_sizes, block_starts ) + payload.append( [ ( int(feature[1]) + block[1], int(feature[1]) + block[1] + block[0] ) for block in blocks ] ) + + # Score (filter data) + if length >= 5 and filter_cols and filter_cols[0] == "Score": + payload.append( float(feature[4]) ) + + rval.append( payload ) + + return { 'data': rval, 'message': message } + + def write_data_to_file( self, chrom, start, end, filename ): + iterator = self.get_iterator( chrom, start, end ) + out = open( filename, "w" ) + for line in iterator: + out.write( "%s\n" % line ) + out.close() class SummaryTreeDataProvider( TracksDataProvider ): """ @@ -213,14 +312,31 @@ class BamDataProvider( TracksDataProvider ): # Cleanup. bamfile.close() - def get_data( self, chrom, start, end, **kwargs ): + def get_data( self, chrom, start, end, start_val=0, max_vals=sys.maxint, **kwargs ): """ - Fetch intervals in the region + Fetch reads in the region and additional metadata. + + Returns a dict with the following attributes: + data - a list of reads with the format + [, , , , , ] + where has the format + [, , , ??] + and has the format + [, , , ??] + For single-end reads, read has format: + [, , , , cigar, seq] + NOTE: read end and sequence data are not valid for reads outside of + requested region and should not be used. + + max_low - lowest coordinate for the returned reads + max_high - highest coordinate for the returned reads + message - error/informative message """ start, end = int(start), int(end) orig_data_filename = self.original_dataset.file_name index_filename = self.converted_dataset.file_name no_detail = "no_detail" in kwargs + # Attempt to open the BAM file with index bamfile = csamtools.Samfile( filename=orig_data_filename, mode='rb', index_filename=index_filename ) message = None @@ -235,21 +351,15 @@ class BamDataProvider( TracksDataProvider ): return None else: return None - # Encode reads as list of lists; each read is a list with the format - # [, , , , , ] - # where has the format - # [, , , ??] - # and has the format - # [, , , ??] - # For single-end reads, read has format: - # [, , , , cigar, seq] - # NOTE: read end and sequence data are not valid for reads outside of - # requested region and should not be used. + + # Encode reads as list of lists. results = [] paired_pending = {} - for read in data: - if len(results) > MAX_VALS: - message = ERROR_MAX_VALS % "reads" + for count, read in enumerate( data ): + if count < start_val: + continue + if count-start_val >= max_vals: + message = ERROR_MAX_VALS % ( max_vals, "reads" ) break qname = read.qname seq = read.seq @@ -273,7 +383,9 @@ class BamDataProvider( TracksDataProvider ): paired_pending[qname] = { 'start': read.pos, 'end': read.pos + read_len, 'seq': seq, 'mate_start': read.mpos, 'rlen': read_len, 'cigar': read.cigar } else: results.append( [ "%i_%s" % ( read.pos, qname ), read.pos, read.pos + read_len, qname, read.cigar, read.seq] ) + # Take care of reads whose mates are out of range. + # TODO: count paired reads when adhering to max_vals? for qname, read in paired_pending.iteritems(): if read['mate_start'] < read['start']: # Mate is before read. @@ -293,9 +405,13 @@ class BamDataProvider( TracksDataProvider ): r2 = [ read['mate_start'], read['mate_start'] ] results.append( [ "%i_%s" % ( read_start, qname ), read_start, read_end, qname, r1, r2 ] ) - + + # Clean up. bamfile.close() - return { 'data': results, 'message': message } + + max_low, max_high = get_bounds( results, 1, 2 ) + + return { 'data': results, 'message': message, 'max_low': max_low, 'max_high': max_high } class BBIDataProvider( TracksDataProvider ): """ @@ -311,7 +427,7 @@ class BBIDataProvider( TracksDataProvider ): f.close() return all_dat is not None - def get_data( self, chrom, start, end, **kwargs ): + def get_data( self, chrom, start, end, start_val=0, max_vals=None, **kwargs ): # Bigwig has the possibility of it being a standalone bigwig file, in which case we use # original_dataset, or coming from wig->bigwig conversion in which we use converted_dataset f, bbi = self._get_dataset() @@ -323,9 +439,10 @@ class BBIDataProvider( TracksDataProvider ): return None all_dat = all_dat[0] # only 1 summary - return { 'max': float( all_dat['max'] ), \ - 'min': float( all_dat['min'] ), \ - 'total_frequency': float( all_dat['coverage'] ) } + return { 'data' : { 'max': float( all_dat['max'] ), \ + 'min': float( all_dat['min'] ), \ + 'total_frequency': float( all_dat['coverage'] ) } \ + } start = int(start) end = int(end) @@ -350,7 +467,7 @@ class BBIDataProvider( TracksDataProvider ): result.append( (pos, float_nan(dat_dict['mean']) ) ) pos += step_size - return result + return { 'data': result } class BigBedDataProvider( BBIDataProvider ): def _get_dataset( self ): @@ -401,7 +518,7 @@ class FilterableMixin: 'type': 'int', 'index': filter_col, 'tool_id': 'Filter1', - 'tool_exp_name': 'c5' } ] + 'tool_exp_name': 'c6' } ] filter_col += 1 if isinstance( self.original_dataset.datatype, Gtf ): # Create filters based on dataset metadata. @@ -473,11 +590,14 @@ class TabixDataProvider( FilterableMixin, TracksDataProvider ): return tabix.fetch(reference=chrom, start=start, end=end) - def get_data( self, chrom, start, end, **kwargs ): + def get_data( self, chrom, start, end, start_val=0, max_vals=None, **kwargs ): iterator = self.get_iterator( chrom, start, end ) - return self.process_data(iterator, **kwargs) - + return self.process_data( iterator, start_val, max_vals, **kwargs ) + class IntervalIndexDataProvider( FilterableMixin, TracksDataProvider ): + """ + Interval index files used only for GFF files. + """ col_name_data_attr_mapping = { 4 : { 'index': 4 , 'name' : 'Score' } } def write_data_to_file( self, chrom, start, end, filename ): @@ -493,12 +613,11 @@ class IntervalIndexDataProvider( FilterableMixin, TracksDataProvider ): out.write(interval.raw_line + '\n') out.close() - def get_data( self, chrom, start, end, **kwargs ): + def get_data( self, chrom, start, end, start_val=0, max_vals=sys.maxint, **kwargs ): start, end = int(start), int(end) source = open( self.original_dataset.file_name ) index = Indexes( self.converted_dataset.file_name ) results = [] - count = 0 message = None # If chrom is not found in indexes, try removing the first three @@ -517,14 +636,15 @@ class IntervalIndexDataProvider( FilterableMixin, TracksDataProvider ): # filter_cols = from_json_string( kwargs.get( "filter_cols", "[]" ) ) no_detail = ( "no_detail" in kwargs ) - for start, end, offset in index.find(chrom, start, end): - if count >= MAX_VALS: - message = ERROR_MAX_VALS % "features" + for count, val in enumerate( index.find(chrom, start, end) ): + start, end, offset = val[0], val[1], val[2] + if count < start_val: + continue + if count-start_val >= max_vals: + message = ERROR_MAX_VALS % ( max_vals, "features" ) break - count += 1 source.seek( offset ) # TODO: can we use column metadata to fill out payload? - # TODO: use function to set payload data # GFF dataset. reader = GFFReaderWrapper( source, fix_strand=True ) @@ -535,143 +655,96 @@ class IntervalIndexDataProvider( FilterableMixin, TracksDataProvider ): results.append( payload ) return { 'data': results, 'message': message } - -class BedDataProvider( TabixDataProvider ): - """ - Payload format: [ uid (offset), start, end, name, strand, thick_start, thick_end, blocks ] - """ - - def process_data( self, iterator, **kwargs ): - # - # Build data to return. Payload format is: - # [ , , , , , , , - # , ] - # - # First three entries are mandatory, others are optional. - # - filter_cols = from_json_string( kwargs.get( "filter_cols", "[]" ) ) - no_detail = ( "no_detail" in kwargs ) - count = 0 - rval = [] - message = None - for line in iterator: - if count >= MAX_VALS: - message = ERROR_MAX_VALS % "features" - break - count += 1 - # TODO: can we use column metadata to fill out payload? - # TODO: use function to set payload data - - feature = line.split() - length = len(feature) - # Unique id is just a hash of the line - payload = [ hash(line), int(feature[1]), int(feature[2]) ] - - if no_detail: - rval.append( payload ) - continue - - # Simpler way to add stuff, but type casting is not done. - # Name, score, strand, thick start, thick end. - #end = min( len( feature ), 8 ) - #payload.extend( feature[ 3:end ] ) - - # Name, strand, thick start, thick end. - if length >= 4: - payload.append(feature[3]) - if length >= 6: - payload.append(feature[5]) - if length >= 8: - payload.append(int(feature[6])) - payload.append(int(feature[7])) - - # Blocks. - if length >= 12: - block_sizes = [ int(n) for n in feature[10].split(',') if n != ''] - block_starts = [ int(n) for n in feature[11].split(',') if n != '' ] - blocks = zip( block_sizes, block_starts ) - payload.append( [ ( int(feature[1]) + block[1], int(feature[1]) + block[1] + block[0] ) for block in blocks ] ) - - # Score (filter data) - if length >= 5 and filter_cols and filter_cols[0] == "Score": - payload.append( float(feature[4]) ) - rval.append( payload ) - - return { 'data': rval, 'message': message } - - def write_data_to_file( self, chrom, start, end, filename ): - iterator = self.get_iterator( chrom, start, end ) - out = open( filename, "w" ) - for line in iterator: - out.write( line ) - out.close() - -class VcfDataProvider( TracksDataProvider ): +class VcfDataProvider( TabixDataProvider ): """ VCF data provider for the Galaxy track browser. Payload format: - [ uid (offset), start, end, ID, reference base(s), alternate base(s), quality score] + [ uid (offset), start, end, ID, reference base(s), alternate base(s), quality score ] """ col_name_data_attr_mapping = { 'Qual' : { 'index': 6 , 'name' : 'Qual' } } - def process_data( self, iterator, **kwargs ): + def process_data( self, iterator, start_val=0, max_vals=sys.maxint, **kwargs ): rval = [] - count = 0 message = None - reader = galaxy_utils.sequence.vcf.Reader( iterator ) - for line in reader: - if count >= MAX_VALS: - message = ERROR_MAX_VALS % "features" + for count, line in enumerate( iterator ): + if count < start_val: + continue + if count-start_val >= max_vals: + message = ERROR_MAX_VALS % ( "max_vals", "features" ) break - count += 1 feature = line.split() - payload = [ hash(line), vcf_line.pos-1, vcf_line.pos, \ + payload = [ hash(line), int(feature[1])-1, int(feature[1]), # ID: - feature[2], \ + feature[2], # reference base(s): - feature[3], \ + feature[3], # alternative base(s) - feature[4], \ + feature[4], # phred quality score - int( feature[5] )] + float( feature[5] )] rval.append(payload) - return { 'data_type' : 'vcf', 'data': rval, 'message': message } + return { 'data': rval, 'message': message } class GFFDataProvider( TracksDataProvider ): """ Provide data from GFF file. NOTE: this data provider does not use indices, and hence will be very slow - for large datasets. + for large datasets. """ - def get_data( self, chrom, start, end, **kwargs ): + def get_data( self, chrom, start, end, start_val=0, max_vals=sys.maxint, **kwargs ): start, end = int( start ), int( end ) source = open( self.original_dataset.file_name ) results = [] - count = 0 message = None offset = 0 - for feature in GFFReaderWrapper( source, fix_strand=True ): + for count, feature in enumerate( GFFReaderWrapper( source, fix_strand=True ) ): + if count < start_val: + continue + if count-start_val >= max_vals: + message = ERROR_MAX_VALS % ( max_vals, "reads" ) + break + feature_start, feature_end = convert_gff_coords_to_bed( [ feature.start, feature.end ] ) if feature.chrom != chrom or feature_start < start or feature_end > end: continue - if count >= MAX_VALS: - message = ERROR_MAX_VALS % "features" - break - count += 1 payload = package_gff_feature( feature ) payload.insert( 0, offset ) results.append( payload ) offset += feature.raw_size return { 'data': results, 'message': message } + +class BedTabixDataProvider( TabixDataProvider, BedDataProvider ): + """ + Provides data from a BED file indexed via tabix. + """ + pass + +class RawBedDataProvider( BedDataProvider ): + """ + Provide data from BED file. + + NOTE: this data provider does not use indices, and hence will be very slow + for large datasets. + """ + + def get_iterator( self, chrom, start, end ): + def line_filter_iter(): + for line in open( self.original_dataset.file_name ): + feature = line.split() + feature_chrom, feature_start, feature_end = feature[ 0:3 ] + if feature_chrom != chrom or feature_start > end or feature_end < start: + continue + yield line + return line_filter_iter() # # Helper methods. @@ -681,7 +754,7 @@ class GFFDataProvider( TracksDataProvider ): # type. First key is converted dataset type; if result is another dict, second key # is original dataset type. TODO: This needs to be more flexible. dataset_type_name_to_data_provider = { - "tabix": { Vcf: VcfDataProvider, Bed: BedDataProvider, "default" : TabixDataProvider }, + "tabix": { Vcf: VcfDataProvider, Bed: BedTabixDataProvider, "default" : TabixDataProvider }, "interval_index": IntervalIndexDataProvider, "bai": BamDataProvider, "summary_tree": SummaryTreeDataProvider, @@ -693,6 +766,7 @@ def get_data_provider( name=None, original_dataset=None ): """ Returns data provider class by name and/or original dataset. """ + data_provider = None if name: value = dataset_type_name_to_data_provider[ name ] if isinstance( value, dict ): diff --git a/lib/galaxy/visualization/tracks/summary.py b/lib/galaxy/visualization/tracks/summary.py index 9b5180c9aa3..0b1996b8844 100644 --- a/lib/galaxy/visualization/tracks/summary.py +++ b/lib/galaxy/visualization/tracks/summary.py @@ -89,5 +89,5 @@ class SummaryTree: cPickle.dump(self, open(filename, 'wb'), 2) def summary_tree_from_file(filename): - return cPickle.load(open(filename, "r")) + return cPickle.load(open(filename, "rb")) diff --git a/lib/galaxy/visualization/tracks/visual_analytics.py b/lib/galaxy/visualization/tracks/visual_analytics.py index 65082fd0787..63849e4ae5d 100644 --- a/lib/galaxy/visualization/tracks/visual_analytics.py +++ b/lib/galaxy/visualization/tracks/visual_analytics.py @@ -24,6 +24,10 @@ def get_tool_def( trans, hda ): # assert tool is not None, 'Requested tool has not been loaded.' if not tool: return {} + + # Tool must have a Trackster configuration. + if not tool.trackster_conf: + return {} # Get list of tool parameters that can be interactively modified. tool_params = [] @@ -46,7 +50,7 @@ def get_tool_def( trans, hda ): 'html' : urllib.quote( input.get_html() ) } ) # If tool has parameters that can be interactively modified, return tool. - # Return empty set otherwise. + tool_def = {} if len( tool_params ) != 0: - return { 'name' : tool.name, 'params' : tool_params } - return {} \ No newline at end of file + tool_def = { 'name' : tool.name, 'params' : tool_params } + return tool_def \ No newline at end of file diff --git a/lib/galaxy/web/api/forms.py b/lib/galaxy/web/api/forms.py index b0d905d4a1b..358af4b80af 100644 --- a/lib/galaxy/web/api/forms.py +++ b/lib/galaxy/web/api/forms.py @@ -2,14 +2,14 @@ API operations on FormDefinition objects. """ import logging -from galaxy.web.base.controller import BaseController, url_for +from galaxy.web.base.controller import BaseAPIController, url_for from galaxy import web from galaxy.forms.forms import form_factory from elementtree.ElementTree import XML log = logging.getLogger( __name__ ) -class FormDefinitionAPIController( BaseController ): +class FormDefinitionAPIController( BaseAPIController ): @web.expose_api def index( self, trans, **kwd ): diff --git a/lib/galaxy/web/api/histories.py b/lib/galaxy/web/api/histories.py new file mode 100644 index 00000000000..cf028201392 --- /dev/null +++ b/lib/galaxy/web/api/histories.py @@ -0,0 +1,163 @@ +""" +API operations on a history. +""" +import logging, os, string, shutil, urllib, re, socket +from cgi import escape, FieldStorage +from galaxy import util, datatypes, jobs, web, util +from galaxy.web.base.controller import * +from galaxy.util.sanitize_html import sanitize_html +from galaxy.model.orm import * +import galaxy.datatypes +from galaxy.util.bunch import Bunch + +log = logging.getLogger( __name__ ) + +class HistoriesController( BaseAPIController, UsesHistory ): + + @web.expose_api + def index( self, trans, deleted='False', **kwd ): + """ + GET /api/histories + GET /api/histories/deleted + Displays a collection (list) of histories. + """ + rval = [] + deleted = util.string_as_bool( deleted ) + + try: + query = trans.sa_session.query( trans.app.model.History ).filter_by( user=trans.user, deleted=deleted ).order_by( + desc(trans.app.model.History.table.c.update_time)).all() + except Exception, e: + rval = "Error in history API" + log.error( rval + ": %s" % str(e) ) + trans.response.status = 500 + + if not rval: + try: + for history in query: + item = history.get_api_value(value_mapper={'id':trans.security.encode_id}) + item['url'] = url_for( 'history', id=trans.security.encode_id( history.id ) ) + rval.append( item ) + except Exception, e: + rval = "Error in history API at constructing return list" + log.error( rval + ": %s" % str(e) ) + trans.response.status = 500 + return rval + + @web.expose_api + def show( self, trans, id, deleted='False', **kwd ): + """ + GET /api/histories/{encoded_history_id} + GET /api/histories/deleted/{encoded_history_id} + Displays information about a history. + """ + history_id = id + params = util.Params( kwd ) + deleted = util.string_as_bool( deleted ) + + def traverse( datasets ): + rval = {} + states = trans.app.model.Dataset.states + for key, state in states.items(): + rval[state] = 0 + for dataset in datasets: + item = dataset.get_api_value( view='element' ) + if not item['deleted']: + rval[item['state']] = rval[item['state']] + 1 + return rval + + try: + history = self.get_history( trans, history_id, check_ownership=True, check_accessible=True, deleted=deleted ) + except Exception, e: + return str( e ) + + try: + item = history.get_api_value(view='element', value_mapper={'id':trans.security.encode_id}) + num_sets = len( [hda.id for hda in history.datasets if not hda.deleted] ) + states = trans.app.model.Dataset.states + state = states.ERROR + if num_sets == 0: + state = states.NEW + else: + summary = traverse(history.datasets) + if summary[states.ERROR] > 0 or summary[states.FAILED_METADATA] > 0: + state = states.ERROR + elif summary[states.RUNNING] > 0 or summary[states.SETTING_METADATA] > 0: + state = states.RUNNING + elif summary[states.QUEUED] > 0: + state = states.QUEUED + elif summary[states.OK] == num_sets: + state = states.OK + item['contents_url'] = url_for( 'history_contents', history_id=history_id ) + item['state'] = state + item['state_details'] = summary + except Exception, e: + item = "Error in history API at showing history detail" + log.error(item + ": %s" % str(e)) + trans.response.status = 500 + return item + + @web.expose_api + def create( self, trans, payload, **kwd ): + """ + POST /api/histories + Creates a new history. + """ + params = util.Params( payload ) + hist_name = None + if payload.get( 'name', None ): + hist_name = util.restore_text( payload['name'] ) + new_history = trans.app.model.History( user=trans.user, name=hist_name ) + + trans.sa_session.add( new_history ) + trans.sa_session.flush() + item = new_history.get_api_value(view='element', value_mapper={'id':trans.security.encode_id}) + return item + + @web.expose_api + def delete( self, trans, id, **kwd ): + """ + DELETE /api/histories/{encoded_history_id} + Deletes a history + """ + history_id = id + # a request body is optional here + purge = False + if kwd.get( 'payload', None ): + purge = util.string_as_bool( kwd['payload'].get( 'purge', False ) ) + + try: + history = self.get_history( trans, history_id, check_ownership=True, check_accessible=False, deleted=True ) + except Exception, e: + return str( e ) + + history.deleted = True + if purge and trans.app.config.allow_user_dataset_purge: + for hda in history.datasets: + if hda.purged: + continue + hda.purged = True + trans.sa_session.add( hda ) + trans.sa_session.flush() + if hda.dataset.user_can_purge: + try: + hda.dataset.full_delete() + trans.sa_session.add( hda.dataset ) + except: + pass + trans.sa_session.flush() + + trans.sa_session.flush() + return 'OK' + + @web.expose_api + def undelete( self, trans, id, **kwd ): + """ + POST /api/histories/deleted/{encoded_quota_id}/undelete + Undeletes a quota + """ + history = self.get_history( trans, history_id, check_ownership=True, check_accessible=False, deleted=True ) + history.deleted = False + trans.sa_session.add( history ) + trans.sa_session.flush() + return 'OK' diff --git a/lib/galaxy/web/api/history_contents.py b/lib/galaxy/web/api/history_contents.py new file mode 100644 index 00000000000..0115079afc6 --- /dev/null +++ b/lib/galaxy/web/api/history_contents.py @@ -0,0 +1,102 @@ +""" +API operations on the contents of a history. +""" +import logging, os, string, shutil, urllib, re, socket +from cgi import escape, FieldStorage +from galaxy import util, datatypes, jobs, web, util +from galaxy.web.base.controller import * +from galaxy.util.sanitize_html import sanitize_html +from galaxy.model.orm import * + +import pkg_resources +pkg_resources.require( "Routes" ) +import routes + +log = logging.getLogger( __name__ ) + +class HistoryContentsController( BaseAPIController, UsesHistoryDatasetAssociation, UsesHistory ): + + @web.expose_api + def index( self, trans, history_id, **kwd ): + """ + GET /api/histories/{encoded_history_id}/contents + Displays a collection (list) of history contents + """ + try: + history = self.get_history( trans, history_id, check_ownership=True, check_accessible=True ) + except Exception, e: + return str( e ) + + rval = [] + try: + for dataset in history.datasets: + api_type = "file" + encoded_id = trans.security.encode_id( dataset.id ) + rval.append( dict( id = encoded_id, + type = api_type, + name = dataset.name, + url = url_for( 'history_content', history_id=history_id, id=encoded_id, ) ) ) + except Exception, e: + rval = "Error in history API at listing contents" + log.error( rval + ": %s" % str(e) ) + trans.response.status = 500 + return rval + + @web.expose_api + def show( self, trans, id, history_id, **kwd ): + """ + GET /api/histories/{encoded_history_id}/contents/{encoded_content_id} + Displays information about a history content (dataset). + """ + content_id = id + try: + content = self.get_history_dataset_association( trans, content_id, check_ownership=True, check_accessible=True ) + except Exception, e: + return str( e ) + try: + item = content.get_api_value( view='element' ) + if not item['deleted']: + # Problem: Method url_for cannot use the dataset controller + # Get the environment from DefaultWebTransaction and use default webapp mapper instead of webapp API mapper + url = routes.URLGenerator(trans.webapp.mapper, trans.environ) + # http://routes.groovie.org/generating.html + # url_for is being phased out, so new applications should use url + item['download_url'] = url(controller='dataset', action='display', dataset_id=trans.security.encode_id(content.id), to_ext=content.ext) + item = self.encode_all_ids( trans, item ) + except Exception, e: + item = "Error in history API at listing dataset" + log.error( item + ": %s" % str(e) ) + trans.response.status = 500 + return item + + @web.expose_api + def create( self, trans, history_id, payload, **kwd ): + """ + POST /api/libraries/{encoded_history_id}/contents + Creates a new history content item (file, aka HistoryDatasetAssociation). + """ + params = util.Params( payload ) + from_ld_id = payload.get( 'from_ld_id', None ) + + try: + history = self.get_history( trans, history_id, check_ownership=True, check_accessible=False ) + except Exception, e: + return str( e ) + + if from_ld_id: + try: + ld = get_library_content_for_access( trans, from_ld_id ) + assert type( ld ) is trans.app.model.LibraryDataset, "Library content id ( %s ) is not a dataset" % from_ld_id + except AssertionError, e: + trans.response.status = 400 + return str( e ) + except Exception, e: + return str( e ) + hda = ld.library_dataset_dataset_association.to_history_dataset_association( history, add_to_history=True ) + history.add_dataset( hda ) + trans.sa_session.flush() + return hda.get_api_value() + else: + # TODO: implement other "upload" methods here. + trans.response.status = 403 + return "Not implemented." diff --git a/lib/galaxy/web/api/libraries.py b/lib/galaxy/web/api/libraries.py index 45476dfbcc6..ce85114cd7d 100644 --- a/lib/galaxy/web/api/libraries.py +++ b/lib/galaxy/web/api/libraries.py @@ -10,7 +10,7 @@ from galaxy.model.orm import * log = logging.getLogger( __name__ ) -class LibrariesController( BaseController ): +class LibrariesController( BaseAPIController ): @web.expose_api def index( self, trans, **kwd ): @@ -58,7 +58,8 @@ class LibrariesController( BaseController ): trans.response.status = 400 return "Invalid library id ( %s ) specified." % str( library_id ) item = library.get_api_value( view='element' ) - item['contents_url'] = url_for( 'contents', library_id=library_id ) + #item['contents_url'] = url_for( 'contents', library_id=library_id ) + item['contents_url'] = url_for( 'library_contents', library_id=library_id ) return item @web.expose_api @@ -86,7 +87,7 @@ class LibrariesController( BaseController ): trans.sa_session.flush() encoded_id = trans.security.encode_id( library.id ) rval = {} - rval['url'] = url_for( 'libraries', id=encoded_id ) + rval['url'] = url_for( 'library', id=encoded_id ) rval['name'] = name rval['id'] = encoded_id return [ rval ] diff --git a/lib/galaxy/web/api/contents.py b/lib/galaxy/web/api/library_contents.py similarity index 54% rename from lib/galaxy/web/api/contents.py rename to lib/galaxy/web/api/library_contents.py index 8eb3ef6e5ca..4d493a066ff 100644 --- a/lib/galaxy/web/api/contents.py +++ b/lib/galaxy/web/api/library_contents.py @@ -10,8 +10,8 @@ from galaxy.model.orm import * log = logging.getLogger( __name__ ) -class ContentsController( BaseController ): - +class LibraryContentsController( BaseAPIController, UsesLibrary, UsesLibraryItems ): + @web.expose_api def index( self, trans, library_id, **kwd ): """ @@ -51,48 +51,34 @@ class ContentsController( BaseController ): if not library or not ( trans.user_is_admin() or trans.app.security_agent.can_access_library( current_user_roles, library ) ): trans.response.status = 400 return "Invalid library id ( %s ) specified." % str( library_id ) - encoded_id = trans.security.encode_id( 'folder.%s' % library.root_folder.id ) + encoded_id = 'F' + trans.security.encode_id( library.root_folder.id ) rval.append( dict( id = encoded_id, type = 'folder', name = '/', - url = url_for( 'content', library_id=library_id, id=encoded_id ) ) ) + url = url_for( 'library_content', library_id=library_id, id=encoded_id ) ) ) library.root_folder.api_path = '' for content in traverse( library.root_folder ): - encoded_id = trans.security.encode_id( '%s.%s' % ( content.api_type, content.id ) ) + encoded_id = trans.security.encode_id( content.id ) + if content.api_type == 'folder': + encoded_id = 'F' + encoded_id rval.append( dict( id = encoded_id, type = content.api_type, name = content.api_path, - url = url_for( 'content', library_id=library_id, id=encoded_id, ) ) ) + url = url_for( 'library_content', library_id=library_id, id=encoded_id, ) ) ) return rval @web.expose_api def show( self, trans, id, library_id, **kwd ): """ - GET /api/libraries/{encoded_library_id}/contents/{encoded_content_type_and_id} + GET /api/libraries/{encoded_library_id}/contents/{encoded_content_id} Displays information about a library content (file or folder). """ - content_id = id - try: - decoded_type_and_id = trans.security.decode_string_id( content_id ) - content_type, decoded_content_id = decoded_type_and_id.split( '.' ) - except: - trans.response.status = 400 - return "Malformed content id ( %s ) specified, unable to decode." % str( content_id ) - if content_type == 'folder': - model_class = trans.app.model.LibraryFolder - elif content_type == 'file': - model_class = trans.app.model.LibraryDataset + class_name, content_id = self.__decode_library_content_id( trans, id ) + if class_name == 'LibraryFolder': + content = self.get_library_folder( trans, content_id, check_ownership=False, check_accessibility=True ) else: - trans.response.status = 400 - return "Invalid type ( %s ) specified." % str( content_type ) - try: - content = trans.sa_session.query( model_class ).get( decoded_content_id ) - except: - content = None - if not content or ( not trans.user_is_admin() and not trans.app.security_agent.can_access_library_item( trans.get_current_user_roles(), content, trans.user ) ): - trans.response.status = 400 - return "Invalid %s id ( %s ) specified." % ( content_type, str( content_id ) ) - return content.get_api_value( view='element' ) + content = self.get_library_dataset( trans, content_id, check_ownership=False, check_accessibility=True ) + return self.encode_all_ids( trans, content.get_api_value( view='element' ) ) @web.expose_api def create( self, trans, library_id, payload, **kwd ): @@ -103,52 +89,49 @@ class ContentsController( BaseController ): create_type = None if 'create_type' not in payload: trans.response.status = 400 - return "Missing required 'create_type' parameter. Please consult the API documentation for help." + return "Missing required 'create_type' parameter." else: create_type = payload.pop( 'create_type' ) if create_type not in ( 'file', 'folder' ): trans.response.status = 400 - return "Invalid value for 'create_type' parameter ( %s ) specified. Please consult the API documentation for help." % create_type + return "Invalid value for 'create_type' parameter ( %s ) specified." % create_type + if 'folder_id' not in payload: + trans.response.status = 400 + return "Missing requred 'folder_id' parameter." + else: + folder_id = payload.pop( 'folder_id' ) try: - content_id = str( payload.pop( 'folder_id' ) ) - decoded_type_and_id = trans.security.decode_string_id( content_id ) - parent_type, decoded_parent_id = decoded_type_and_id.split( '.' ) - assert parent_type in ( 'folder', 'file' ) - except: - trans.response.status = 400 - return "Malformed parent id ( %s ) specified, unable to decode." % content_id - # "content" can be either a folder or a file, but the parent of new contents can only be folders. - if parent_type == 'file': - trans.response.status = 400 - try: - # With admins or people who can access the dataset provided as the parent, be descriptive. - dataset = trans.sa_session.query( trans.app.model.LibraryDataset ).get( decoded_parent_id ).library_dataset_dataset_association.dataset - assert trans.user_is_admin() or trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), dataset ) - return "The parent id ( %s ) points to a file, not a folder." % content_id - except: - # If you can't access the parent we don't want to reveal its existence. - return "Invalid parent folder id ( %s ) specified." % content_id + # security is checked in the downstream controller + parent = self.get_library_folder( trans, folder_id, check_ownership=False, check_accessibility=False ) + except Exception, e: + return str( e ) # The rest of the security happens in the library_common controller. - folder_id = trans.security.encode_id( decoded_parent_id ) + real_folder_id = trans.security.encode_id( parent.id ) # Now create the desired content object, either file or folder. if create_type == 'file': - status, output = trans.webapp.controllers['library_common'].upload_library_dataset( trans, 'api', library_id, folder_id, **payload ) + status, output = trans.webapp.controllers['library_common'].upload_library_dataset( trans, 'api', library_id, real_folder_id, **payload ) elif create_type == 'folder': - status, output = trans.webapp.controllers['library_common'].create_folder( trans, 'api', folder_id, library_id, **payload ) + status, output = trans.webapp.controllers['library_common'].create_folder( trans, 'api', real_folder_id, library_id, **payload ) if status != 200: trans.response.status = status - # We don't want to reveal the encoded folder_id since it's invalid - # in the API context. Instead, return the content_id originally - # supplied by the client. - output = output.replace( folder_id, content_id ) return output else: rval = [] for k, v in output.items(): if type( v ) == trans.app.model.LibraryDatasetDatasetAssociation: v = v.library_dataset - encoded_id = trans.security.encode_id( create_type + '.' + str( v.id ) ) + encoded_id = trans.security.encode_id( v.id ) + if create_type == 'folder': + encoded_id = 'F' + encoded_id rval.append( dict( id = encoded_id, name = v.name, - url = url_for( 'content', library_id=library_id, id=encoded_id ) ) ) + url = url_for( 'library_content', library_id=library_id, id=encoded_id ) ) ) return rval + + def __decode_library_content_id( self, trans, content_id ): + if ( len( content_id ) % 16 == 0 ): + return 'LibraryDataset', content_id + elif ( content_id.startswith( 'F' ) ): + return 'LibraryFolder', content_id[1:] + else: + raise HTTPBadRequest( 'Malformed library content id ( %s ) specified, unable to decode.' % str( content_id ) ) diff --git a/lib/galaxy/web/api/permissions.py b/lib/galaxy/web/api/permissions.py new file mode 100644 index 00000000000..b9ab5213fd9 --- /dev/null +++ b/lib/galaxy/web/api/permissions.py @@ -0,0 +1,51 @@ +""" +API operations on the permissions of a library. +""" +import logging, os, string, shutil, urllib, re, socket +from cgi import escape, FieldStorage +from galaxy import util, datatypes, jobs, web, util +from galaxy.web.base.controller import * +from galaxy.util.sanitize_html import sanitize_html +from galaxy.model.orm import * + +log = logging.getLogger( __name__ ) + +class PermissionsController( BaseAPIController ): + + # Method not ideally named + @web.expose_api + def create( self, trans, library_id, payload, **kwd ): + """ + POST /api/libraries/{encoded_library_id}/permissions + Updates the library permissions. + """ + if not trans.user_is_admin(): + trans.response.status = 403 + return "You are not authorized to update library permissions." + + params = util.Params( payload ) + try: + decoded_library_id = trans.security.decode_id( library_id ) + except TypeError: + trans.response.status = 400 + return "Malformed library id ( %s ) specified, unable to decode." % str( library_id ) + + try: + library = trans.sa_session.query( trans.app.model.Library ).get( decoded_library_id ) + except: + library = None + + permissions = {} + for k, v in trans.app.model.Library.permitted_actions.items(): + role_params = params.get( k + '_in', [] ) + in_roles = [ trans.sa_session.query( trans.app.model.Role ).get( trans.security.decode_id( x ) ) for x in util.listify( role_params ) ] + permissions[ trans.app.security_agent.get_action( v.action ) ] = in_roles + trans.app.security_agent.set_all_library_permissions( library, permissions ) + trans.sa_session.refresh( library ) + # Copy the permissions to the root folder + trans.app.security_agent.copy_library_permissions( library, library.root_folder ) + message = "Permissions updated for library '%s'." % library.name + + item = library.get_api_value( view='element' ) + return item + diff --git a/lib/galaxy/web/api/quotas.py b/lib/galaxy/web/api/quotas.py new file mode 100644 index 00000000000..db8ffe1fe6d --- /dev/null +++ b/lib/galaxy/web/api/quotas.py @@ -0,0 +1,146 @@ +""" +API operations on Quota objects. +""" +import logging +from galaxy.web.base.controller import BaseAPIController, Admin, UsesQuota, url_for +from galaxy import web, util +from elementtree.ElementTree import XML + +from galaxy.web.params import QuotaParamParser +from galaxy.actions.admin import AdminActions + +from paste.httpexceptions import HTTPBadRequest +from galaxy.exceptions import * + +log = logging.getLogger( __name__ ) + +class QuotaAPIController( BaseAPIController, Admin, AdminActions, UsesQuota, QuotaParamParser ): + @web.expose_api + @web.require_admin + def index( self, trans, deleted='False', **kwd ): + """ + GET /api/quotas + GET /api/quotas/deleted + Displays a collection (list) of quotas. + """ + rval = [] + deleted = util.string_as_bool( deleted ) + query = trans.sa_session.query( trans.app.model.Quota ) + if deleted: + route = 'deleted_quota' + query = query.filter( trans.app.model.Quota.table.c.deleted == True ) + else: + route = 'quota' + query = query.filter( trans.app.model.Quota.table.c.deleted == False ) + for quota in query: + item = quota.get_api_value( value_mapper={ 'id': trans.security.encode_id } ) + encoded_id = trans.security.encode_id( quota.id ) + item['url'] = url_for( route, id=encoded_id ) + rval.append( item ) + return rval + + @web.expose_api + @web.require_admin + def show( self, trans, id, deleted='False', **kwd ): + """ + GET /api/quotas/{encoded_quota_id} + GET /api/quotas/deleted/{encoded_quota_id} + Displays information about a quota. + """ + quota = self.get_quota( trans, id, deleted=util.string_as_bool( deleted ) ) + return quota.get_api_value( view='element', value_mapper={ 'id': trans.security.encode_id } ) + + @web.expose_api + @web.require_admin + def create( self, trans, payload, **kwd ): + """ + POST /api/quotas + Creates a new quota. + """ + try: + self.validate_in_users_and_groups( trans, payload ) + except Exception, e: + raise HTTPBadRequest( detail=str( e ) ) + params = self.get_quota_params( payload ) + try: + quota, message = self._create_quota( params ) + except ActionInputError, e: + raise HTTPBadRequest( detail=str( e ) ) + item = quota.get_api_value( value_mapper={ 'id': trans.security.encode_id } ) + item['url'] = url_for( 'quota', id=trans.security.encode_id( quota.id ) ) + item['message'] = message + return item + + @web.expose_api + @web.require_admin + def update( self, trans, id, payload, **kwd ): + """ + PUT /api/quotas/{encoded_quota_id} + Modifies a quota. + """ + try: + self.validate_in_users_and_groups( trans, payload ) + except Exception, e: + raise HTTPBadRequest( detail=str( e ) ) + + quota = self.get_quota( trans, id, deleted=False ) + + # FIXME: Doing it this way makes the update non-atomic if a method fails after an earlier one has succeeded. + payload['id'] = id + params = self.get_quota_params( payload ) + methods = [] + if payload.get( 'name', None ) or payload.get( 'description', None ): + methods.append( self._rename_quota ) + if payload.get( 'amount', None ): + methods.append( self._edit_quota ) + if payload.get( 'default', None ) == 'no': + methods.append( self._unset_quota_default ) + elif payload.get( 'default', None ): + methods.append( self._set_quota_default ) + if payload.get( 'in_users', None ) or payload.get( 'in_groups', None ): + methods.append( self._manage_users_and_groups_for_quota ) + + messages = [] + for method in methods: + try: + message = method( quota, params ) + except ActionInputError, e: + raise HTTPBadRequest( detail=str( e ) ) + messages.append( message ) + return '; '.join( messages ) + + @web.expose_api + @web.require_admin + def delete( self, trans, id, **kwd ): + """ + DELETE /api/quotas/{encoded_quota_id} + Deletes a quota + """ + quota = self.get_quota( trans, id, deleted=False ) # deleted quotas are not technically members of this collection + + # a request body is optional here + payload = kwd.get( 'payload', {} ) + payload['id'] = id + params = self.get_quota_params( payload ) + + try: + message = self._mark_quota_deleted( quota, params ) + if util.string_as_bool( payload.get( 'purge', False ) ): + message += self._purge_quota( quota, params ) + except ActionInputError, e: + raise HTTPBadRequest( detail=str( e ) ) + return message + + @web.expose_api + @web.require_admin + def undelete( self, trans, id, **kwd ): + """ + POST /api/quotas/deleted/{encoded_quota_id}/undelete + Undeletes a quota + """ + quota = self.get_quota( trans, id, deleted=True ) + params = self.get_quota_params( payload ) + try: + return self._undelete_quota( quota, params ) + except ActionInputError, e: + raise HTTPBadRequest( detail=str( e ) ) diff --git a/lib/galaxy/web/api/request_types.py b/lib/galaxy/web/api/request_types.py index 9a3c0d67066..aca7fc7def6 100644 --- a/lib/galaxy/web/api/request_types.py +++ b/lib/galaxy/web/api/request_types.py @@ -2,7 +2,7 @@ API operations on RequestType objects. """ import logging -from galaxy.web.base.controller import BaseController, url_for +from galaxy.web.base.controller import BaseAPIController, url_for from galaxy import web from galaxy.sample_tracking.request_types import request_type_factory from elementtree.ElementTree import XML @@ -10,7 +10,7 @@ from elementtree.ElementTree import XML log = logging.getLogger( __name__ ) -class RequestTypeAPIController( BaseController ): +class RequestTypeAPIController( BaseAPIController ): @web.expose_api def index( self, trans, **kwd ): """ diff --git a/lib/galaxy/web/api/requests.py b/lib/galaxy/web/api/requests.py index d1a509e8d1a..ce05aa59df8 100644 --- a/lib/galaxy/web/api/requests.py +++ b/lib/galaxy/web/api/requests.py @@ -11,7 +11,7 @@ from galaxy.util.bunch import Bunch log = logging.getLogger( __name__ ) -class RequestsAPIController( BaseController ): +class RequestsAPIController( BaseAPIController ): update_types = Bunch( REQUEST = 'request_state' ) update_type_values = [v[1] for v in update_types.items()] @web.expose_api diff --git a/lib/galaxy/web/api/roles.py b/lib/galaxy/web/api/roles.py index 074b0a2175a..fdd3066e114 100644 --- a/lib/galaxy/web/api/roles.py +++ b/lib/galaxy/web/api/roles.py @@ -2,13 +2,13 @@ API operations on Role objects. """ import logging -from galaxy.web.base.controller import BaseController, url_for +from galaxy.web.base.controller import BaseAPIController, url_for from galaxy import web from elementtree.ElementTree import XML log = logging.getLogger( __name__ ) -class RoleAPIController( BaseController ): +class RoleAPIController( BaseAPIController ): @web.expose_api def index( self, trans, **kwd ): """ diff --git a/lib/galaxy/web/api/samples.py b/lib/galaxy/web/api/samples.py index 4f5374d91a7..a6fbcb9ecbc 100644 --- a/lib/galaxy/web/api/samples.py +++ b/lib/galaxy/web/api/samples.py @@ -9,7 +9,7 @@ from galaxy.util.bunch import Bunch log = logging.getLogger( __name__ ) -class SamplesAPIController( BaseController ): +class SamplesAPIController( BaseAPIController ): update_types = Bunch( SAMPLE = [ 'sample_state', 'run_details' ], SAMPLE_DATASET = [ 'sample_dataset_transfer_status' ] ) update_type_values = [] diff --git a/lib/galaxy/web/api/users.py b/lib/galaxy/web/api/users.py index 5180786df6a..edd14d87de3 100644 --- a/lib/galaxy/web/api/users.py +++ b/lib/galaxy/web/api/users.py @@ -2,59 +2,79 @@ API operations on User objects. """ import logging -from galaxy.web.base.controller import BaseController, url_for -from galaxy import web +from galaxy.web.base.controller import BaseAPIController, url_for +from galaxy import web, util from elementtree.ElementTree import XML +from paste.httpexceptions import * log = logging.getLogger( __name__ ) -class UserAPIController( BaseController ): +class UserAPIController( BaseAPIController ): @web.expose_api - def index( self, trans, **kwd ): + def index( self, trans, deleted='False', **kwd ): """ GET /api/users + GET /api/users/deleted Displays a collection (list) of users. """ - if not trans.user_is_admin(): - trans.response.status = 403 - return "You are not authorized to view the list of users." rval = [] - for user in trans.sa_session.query( trans.app.model.User ).filter( trans.app.model.User.table.c.deleted == False ): + query = trans.sa_session.query( trans.app.model.User ) + deleted = util.string_as_bool( deleted ) + if deleted: + route = 'deleted_user' + query = query.filter( trans.app.model.User.table.c.deleted == True ) + # only admins can see deleted users + if not trans.user_is_admin(): + return [] + else: + route = 'user' + query = query.filter( trans.app.model.User.table.c.deleted == False ) + # special case: user can see only their own user + if not trans.user_is_admin(): + item = trans.user.get_api_value( value_mapper={ 'id': trans.security.encode_id } ) + item['url'] = url_for( route, id=encoded_id ) + return item + for user in query: item = user.get_api_value( value_mapper={ 'id': trans.security.encode_id } ) encoded_id = trans.security.encode_id( user.id ) - item['url'] = url_for( 'user', id=encoded_id ) + item['url'] = url_for( route, id=encoded_id ) rval.append( item ) return rval @web.expose_api - def show( self, trans, id, **kwd ): + def show( self, trans, id, deleted='False', **kwd ): """ GET /api/users/{encoded_user_id} + GET /api/users/deleted/{encoded_user_id} Displays information about a user. """ - if not trans.user_is_admin(): - trans.response.status = 403 - return "You are not authorized to view user info." - user_id = id + deleted = util.string_as_bool( deleted ) try: - decoded_user_id = trans.security.decode_id( user_id ) - except TypeError: - trans.response.status = 400 - return "Malformed user id ( %s ) specified, unable to decode." % str( user_id ) - try: - user = trans.sa_session.query( trans.app.model.User ).get( decoded_user_id ) + user = self.get_user( trans, id, deleted=deleted ) + if not trans.user_is_admin(): + assert trans.user == user + assert not user.deleted except: - trans.response.status = 400 - return "That user does not exist." - item = user.get_api_value( view='element', value_mapper={ 'id': trans.security.encode_id } ) - item['url'] = url_for( 'user', id=user_id ) + if trans.user_is_admin(): + raise + else: + raise HTTPBadRequest( detail='Invalid user id ( %s ) specified' % id ) + item = user.get_api_value( view='element', value_mapper={ 'id': trans.security.encode_id, + 'total_disk_usage': float } ) return item - @web.expose_api - def create( self, trans, payload, **kwd ): - """ - POST /api/users - Creates a new user. - """ - trans.response.status = 403 - return "Not implemented." + @web.expose + def create( self, trans, **kwd ): + raise HTTPNotImplemented() + + @web.expose + def update( self, trans, **kwd ): + raise HTTPNotImplemented() + + @web.expose + def delete( self, trans, **kwd ): + raise HTTPNotImplemented() + + @web.expose + def undelete( self, trans, **kwd ): + raise HTTPNotImplemented() diff --git a/lib/galaxy/web/api/workflows.py b/lib/galaxy/web/api/workflows.py index 3623e735924..2eaa10a59ca 100644 --- a/lib/galaxy/web/api/workflows.py +++ b/lib/galaxy/web/api/workflows.py @@ -7,13 +7,13 @@ from sqlalchemy import desc from galaxy import util from galaxy import web from galaxy.tools.parameters import visit_input_values, DataToolParameter -from galaxy.web.base.controller import BaseController, url_for +from galaxy.web.base.controller import BaseAPIController, url_for from galaxy.workflow.modules import module_factory from galaxy.jobs.actions.post import ActionBox log = logging.getLogger(__name__) -class WorkflowsAPIController(BaseController): +class WorkflowsAPIController(BaseAPIController): @web.expose_api def index(self, trans, **kwd): """ @@ -29,6 +29,14 @@ class WorkflowsAPIController(BaseController): encoded_id = trans.security.encode_id(wf.id) item['url'] = url_for('workflow', id=encoded_id) rval.append(item) + for wf_sa in trans.sa_session.query( trans.app.model.StoredWorkflowUserShareAssociation ).filter_by( + user=trans.user ).join( 'stored_workflow' ).filter( + trans.app.model.StoredWorkflow.deleted == False ).order_by( + desc( trans.app.model.StoredWorkflow.update_time ) ).all(): + item = wf_sa.stored_workflow.get_api_value(value_mapper={'id':trans.security.encode_id}) + encoded_id = trans.security.encode_id(wf_sa.stored_workflow.id) + item['url'] = url_for('workflow', id=encoded_id) + rval.append(item) return rval @web.expose_api def show(self, trans, id, **kwd): @@ -98,9 +106,8 @@ class WorkflowsAPIController(BaseController): assert trans.user_is_admin() or trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), ldda.dataset ) hda = ldda.to_history_dataset_association(history, add_to_history=add_to_history) elif ds_map[k]['src'] == 'ld': - ld_t, ld_id = trans.security.decode_string_id(ds_map[k]['id']).split('.') ldda = trans.sa_session.query(self.app.model.LibraryDataset).get( - ld_id).library_dataset_dataset_association + trans.security.decode_id(ds_map[k]['id'])).library_dataset_dataset_association assert trans.user_is_admin() or trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), ldda.dataset ) hda = ldda.to_history_dataset_association(history, add_to_history=add_to_history) elif ds_map[k]['src'] == 'hda': diff --git a/lib/galaxy/web/base/controller.py b/lib/galaxy/web/base/controller.py index 1c9773dd0a4..b51157fe197 100644 --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -1,8 +1,9 @@ """ Contains functionality needed in every web interface """ -import os, time, logging, re, string, sys, glob -from datetime import datetime, timedelta +import os, time, logging, re, string, sys, glob, shutil, tempfile, subprocess +from datetime import date, datetime, timedelta +from time import strftime from galaxy import config, tools, web, util from galaxy.web import error, form, url_for from galaxy.model.orm import * @@ -11,9 +12,16 @@ from galaxy.web.framework import simplejson from galaxy.web.form_builder import AddressField, CheckboxField, SelectField, TextArea, TextField, WorkflowField, WorkflowMappingField, HistoryField, PasswordField, build_select_field from galaxy.visualization.tracks.data_providers import get_data_provider from galaxy.visualization.tracks.visual_analytics import get_tool_def +from galaxy.security.validate_user_input import validate_username +from paste.httpexceptions import * +from galaxy.exceptions import * from Cheetah.Template import Template + +pkg_resources.require( 'elementtree' ) +from elementtree import ElementTree, ElementInclude + log = logging.getLogger( __name__ ) # States for passing messages @@ -21,7 +29,7 @@ SUCCESS, INFO, WARNING, ERROR = "done", "info", "warning", "error" # RE that tests for valid slug. VALID_SLUG_RE = re.compile( "^[a-z0-9\-]+$" ) - + class BaseController( object ): """ Base class for Galaxy web application controllers. @@ -29,50 +37,169 @@ class BaseController( object ): def __init__( self, app ): """Initialize an interface for application 'app'""" self.app = app + self.sa_session = app.model.context def get_toolbox(self): """Returns the application toolbox""" return self.app.toolbox - def get_class( self, trans, class_name ): + def get_class( self, class_name ): """ Returns the class object that a string denotes. Without this method, we'd have to do eval(). """ if class_name == 'History': - item_class = trans.model.History + item_class = self.app.model.History elif class_name == 'HistoryDatasetAssociation': - item_class = trans.model.HistoryDatasetAssociation + item_class = self.app.model.HistoryDatasetAssociation elif class_name == 'Page': - item_class = trans.model.Page + item_class = self.app.model.Page elif class_name == 'StoredWorkflow': - item_class = trans.model.StoredWorkflow + item_class = self.app.model.StoredWorkflow elif class_name == 'Visualization': - item_class = trans.model.Visualization + item_class = self.app.model.Visualization elif class_name == 'Tool': - item_class = trans.model.Tool + item_class = self.app.model.Tool elif class_name == 'Job': - item_class == trans.model.Job + item_class = self.app.model.Job + elif class_name == 'User': + item_class = self.app.model.User + elif class_name == 'Group': + item_class = self.app.model.Group + elif class_name == 'Role': + item_class = self.app.model.Role + elif class_name == 'Quota': + item_class = self.app.model.Quota + elif class_name == 'Library': + item_class = self.app.model.Library + elif class_name == 'LibraryFolder': + item_class = self.app.model.LibraryFolder + elif class_name == 'LibraryDatasetDatasetAssociation': + item_class = self.app.model.LibraryDatasetDatasetAssociation + elif class_name == 'LibraryDataset': + item_class = self.app.model.LibraryDataset else: item_class = None return item_class - + def get_object( self, trans, id, class_name, check_ownership=False, check_accessible=False, deleted=None ): + """ + Convenience method to get a model object with the specified checks. + """ + try: + decoded_id = trans.security.decode_id( id ) + except: + raise MessageException( "Malformed %s id ( %s ) specified, unable to decode" % ( class_name, str( id ) ), type='error' ) + try: + item_class = self.get_class( class_name ) + assert item_class is not None + item = trans.sa_session.query( item_class ).get( decoded_id ) + assert item is not None + except: + log.exception( "Invalid %s id ( %s ) specified" % ( class_name, id ) ) + raise MessageException( "Invalid %s id ( %s ) specified" % ( class_name, id ), type="error" ) + if check_ownership or check_accessible: + self.security_check( trans, item, check_ownership, check_accessible, encoded_id ) + if deleted == True and not item.deleted: + raise ItemDeletionException( '%s "%s" is not deleted' % ( class_name, getattr( item, 'name', id ) ), type="warning" ) + elif deleted == False and item.deleted: + raise ItemDeletionException( '%s "%s" is deleted' % ( class_name, getattr( item, 'name', id ) ), type="warning" ) + return item + def get_user( self, trans, id, check_ownership=False, check_accessible=False, deleted=None ): + return self.get_object( trans, id, 'User', check_ownership=False, check_accessible=False, deleted=deleted ) + def get_group( self, trans, id, check_ownership=False, check_accessible=False, deleted=None ): + return self.get_object( trans, id, 'Group', check_ownership=False, check_accessible=False, deleted=deleted ) + def get_role( self, trans, id, check_ownership=False, check_accessible=False, deleted=None ): + return self.get_object( trans, id, 'Role', check_ownership=False, check_accessible=False, deleted=deleted ) + def encode_all_ids( self, trans, rval ): + """ + Encodes all integer values in the dict rval whose keys are 'id' or end with '_id' + + It might be useful to turn this in to a decorator + """ + if type( rval ) != dict: + return rval + for k, v in rval.items(): + if k == 'id' or k.endswith( '_id' ): + try: + rval[k] = trans.security.encode_id( v ) + except: + pass # probably already encoded + return rval + Root = BaseController +class BaseUIController( BaseController ): + def get_object( self, trans, id, class_name, check_ownership=False, check_accessible=False, deleted=None ): + try: + return BaseController.get_object( self, trans, id, class_name, check_ownership=False, check_accessible=False, deleted=None ) + except MessageException, e: + raise # handled in the caller + except: + log.exception( "Execption in get_object check for %s %s:" % ( class_name, str( id ) ) ) + raise Exception( 'Server error retrieving %s id ( %s ).' % ( class_name, str( id ) ) ) + +class BaseAPIController( BaseController ): + def get_object( self, trans, id, class_name, check_ownership=False, check_accessible=False, deleted=None ): + try: + return BaseController.get_object( self, trans, id, class_name, check_ownership=False, check_accessible=False, deleted=None ) + except ItemDeletionException, e: + raise HTTPBadRequest( detail="Invalid %s id ( %s ) specified" % ( class_name, str( id ) ) ) + except MessageException, e: + raise HTTPBadRequest( detail=e.err_msg ) + except Exception, e: + log.exception( "Execption in get_object check for %s %s:" % ( class_name, str( id ) ) ) + raise HTTPInternalServerError( comment=str( e ) ) + def validate_in_users_and_groups( self, trans, payload ): + """ + For convenience, in_users and in_groups can be encoded IDs or emails/group names in the API. + """ + def get_id( item, model_class, column ): + try: + return trans.security.decode_id( item ) + except: + pass # maybe an email/group name + # this will raise if the item is invalid + return trans.sa_session.query( model_class ).filter( column == item ).first().id + new_in_users = [] + new_in_groups = [] + invalid = [] + for item in util.listify( payload.get( 'in_users', [] ) ): + try: + new_in_users.append( get_id( item, trans.app.model.User, trans.app.model.User.table.c.email ) ) + except: + invalid.append( item ) + for item in util.listify( payload.get( 'in_groups', [] ) ): + try: + new_in_groups.append( get_id( item, trans.app.model.Group, trans.app.model.Group.table.c.name ) ) + except: + invalid.append( item ) + if invalid: + msg = "The following value(s) for associated users and/or groups could not be parsed: %s." % ', '.join( invalid ) + msg += " Valid values are email addresses of users, names of groups, or IDs of both." + raise Exception( msg ) + payload['in_users'] = map( str, new_in_users ) + payload['in_groups'] = map( str, new_in_groups ) + def not_implemented( self, trans, **kwd ): + raise HTTPNotImplemented() + class SharableItemSecurity: """ Mixin for handling security for sharable items. """ - def security_check( self, user, item, check_ownership=False, check_accessible=False ): + def security_check( self, trans, item, check_ownership=False, check_accessible=False ): """ Security checks for an item: checks if (a) user owns item or (b) item is accessible to user. """ if check_ownership: # Verify ownership. - if not user: - error( "Must be logged in to manage Galaxy items" ) - if item.user != user: - error( "%s is not owned by current user" % item.__class__.__name__ ) + if not trans.user: + raise ItemOwnershipException( "Must be logged in to manage Galaxy items", type='error' ) + if item.user != trans.user: + raise ItemOwnershipException( "%s is not owned by the current user" % item.__class__.__name__, type='error' ) if check_accessible: - # Verify accessible. - if ( item.user != user ) and ( not item.importable ) and ( user not in item.users_shared_with_dot_users ): - error( "%s is not accessible to current user" % item.__class__.__name__ ) + if type( item ) in ( trans.app.model.LibraryFolder, trans.app.model.LibraryDatasetDatasetAssociation, trans.app.model.LibraryDataset ): + if not ( trans.user_is_admin() or trans.app.security_agent.can_access_library_i9tem( trans.get_current_user_roles(), item, trans.user ) ): + raise ItemAccessibilityException( "%s is not accessible to the current user" % item.__class__.__name__, type='error' ) + else: + # Verify accessible. + if ( item.user != trans.user ) and ( not item.importable ) and ( trans.user not in item.users_shared_with_dot_users ): + raise ItemAccessibilityException( "%s is not accessible to the current user" % item.__class__.__name__, type='error' ) return item # # TODO: need to move UsesHistory, etc. mixins to better location - perhaps lib/galaxy/model/XXX ? -# +# class UsesHistoryDatasetAssociation: """ Mixin for controllers that use HistoryDatasetAssociation objects. """ @@ -89,7 +216,7 @@ class UsesHistoryDatasetAssociation: except: data = None if not data: - raise paste.httpexceptions.HTTPRequestRangeNotSatisfiable( "Invalid dataset id: %s." % str( dataset_id ) ) + raise HTTPRequestRangeNotSatisfiable( "Invalid dataset id: %s." % str( dataset_id ) ) if check_ownership: # Verify ownership. user = trans.get_user() @@ -105,6 +232,16 @@ class UsesHistoryDatasetAssociation: else: error( "You are not allowed to access this dataset" ) return data + def get_history_dataset_association( self, trans, dataset_id, check_ownership=True, check_accessible=False ): + """Get a HistoryDatasetAssociation from the database by id, verifying ownership.""" + hda = self.get_object( trans, id, 'HistoryDatasetAssociation', check_ownership=check_ownership, check_accessible=check_accessible, deleted=deleted ) + self.security_check( trans, history, check_ownership=check_ownership, check_accessible=False ) # check accessibility here + if check_accessible: + if trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), hda.dataset ): + if hda.state == trans.model.Dataset.states.UPLOAD: + error( "Please wait until this dataset finishes uploading before attempting to view it." ) + else: + error( "You are not allowed to access this dataset" ) def get_data( self, dataset, preview=True ): """ Gets a dataset's data. """ # Get data from file, truncating if necessary. @@ -119,15 +256,30 @@ class UsesHistoryDatasetAssociation: dataset_data = open( dataset.file_name ).read(max_peek_size) truncated = False return truncated, dataset_data - + +class UsesLibrary: + def get_library( self, trans, id, check_ownership=False, check_accessible=True ): + l = self.get_object( trans, id, 'Library' ) + if check_accessible and not ( trans.user_is_admin() or trans.app.security_agent.can_access_library( trans.get_current_user_roles(), l ) ): + error( "LibraryFolder is not accessible to the current user" ) + return l + +class UsesLibraryItems( SharableItemSecurity ): + def get_library_folder( self, trans, id, check_ownership=False, check_accessible=True ): + return self.get_object( trans, id, 'LibraryFolder', check_ownership=False, check_accessible=check_accessible ) + def get_library_dataset_dataset_association( self, trans, id, check_ownership=False, check_accessible=True ): + return self.get_object( trans, id, 'LibraryDatasetDatasetAssociation', check_ownership=False, check_accessible=check_accessible ) + def get_library_dataset( self, trans, id, check_ownership=False, check_accessible=True ): + return self.get_object( trans, id, 'LibraryDataset', check_ownership=False, check_accessible=check_accessible ) + class UsesVisualization( SharableItemSecurity ): """ Mixin for controllers that use Visualization objects. """ len_files = None - + def _get_dbkeys( self, trans ): """ Returns all valid dbkeys that a user can use in a visualization. """ - + # Read len files. if not self.len_files: len_files = glob.glob( os.path.join(trans.app.config.len_file_path, "*.len") ) @@ -137,10 +289,10 @@ class UsesVisualization( SharableItemSecurity ): user = trans.get_user() if 'dbkeys' in user.preferences: user_keys = from_json_string( user.preferences['dbkeys'] ) - + dbkeys = [ (v, k) for k, v in trans.db_builds if k in self.len_files or k in user_keys ] return dbkeys - + def get_visualization( self, trans, id, check_ownership=True, check_accessible=False ): """ Get a Visualization from the database by id, verifying ownership. """ # Load workflow from database @@ -151,55 +303,80 @@ class UsesVisualization( SharableItemSecurity ): if not visualization: error( "Visualization not found" ) else: - return self.security_check( trans.get_user(), visualization, check_ownership, check_accessible ) - + return self.security_check( trans, visualization, check_ownership, check_accessible ) + def get_visualization_config( self, trans, visualization ): """ Returns a visualization's configuration. Only works for trackster visualizations right now. """ config = None if visualization.type == 'trackster': - # Trackster config; taken from tracks/browser + # Unpack Trackster config. latest_revision = visualization.latest_revision - tracks = [] + bookmarks = latest_revision.config.get( 'bookmarks', [] ) + + def pack_track( track_dict ): + dataset_id = track_dict['dataset_id'] + hda_ldda = track_dict.get('hda_ldda', 'hda') + if hda_ldda == "hda": + dataset = self.get_dataset( trans, dataset_id, check_ownership=False, check_accessible=True ) + else: + dataset = trans.sa_session.query( trans.app.model.LibraryDatasetDatasetAssociation ).get( trans.security.decode_id(dataset_id) ) + + try: + prefs = track_dict['prefs'] + except KeyError: + prefs = {} + + track_type, _ = dataset.datatype.get_track_type() + track_data_provider_class = get_data_provider( original_dataset=dataset ) + track_data_provider = track_data_provider_class( original_dataset=dataset ) + + return { + "track_type": track_type, + "name": track_dict['name'], + "hda_ldda": track_dict.get("hda_ldda", "hda"), + "dataset_id": trans.security.encode_id( dataset.id ), + "prefs": prefs, + "mode": track_dict.get( 'mode', 'Auto' ), + "filters": track_data_provider.get_filters(), + "tool": get_tool_def( trans, dataset ) + } + + def pack_collection( collection_dict ): + drawables = [] + for drawable_dict in collection_dict[ 'drawables' ]: + if 'track_type' in drawable_dict: + drawables.append( pack_track( drawable_dict ) ) + else: + drawables.append( pack_collection( drawable_dict ) ) + return { + 'name': collection_dict.get( 'name', 'dummy' ), + 'obj_type': collection_dict[ 'obj_type' ], + 'drawables': drawables, + 'prefs': collection_dict.get( 'prefs', [] ) + } # Set tracks. + tracks = [] if 'tracks' in latest_revision.config: - for t in visualization.latest_revision.config['tracks']: - dataset_id = t['dataset_id'] - hda_ldda = t.get('hda_ldda', 'hda') - if hda_ldda == "hda": - dataset = self.get_dataset( trans, dataset_id, check_ownership=False, check_accessible=True ) + # Legacy code. + for track_dict in visualization.latest_revision.config[ 'tracks' ]: + tracks.append( pack_track( track_dict ) ) + elif 'view' in latest_revision.config: + for drawable_dict in visualization.latest_revision.config[ 'view' ][ 'drawables' ]: + if 'track_type' in drawable_dict: + tracks.append( pack_track( drawable_dict ) ) else: - dataset = trans.sa_session.query( trans.app.model.LibraryDatasetDatasetAssociation ).get( trans.security.decode_id(dataset_id) ) - - try: - prefs = t['prefs'] - except KeyError: - prefs = {} - - track_type, _ = dataset.datatype.get_track_type() - track_data_provider_class = get_data_provider( original_dataset=dataset ) - track_data_provider = track_data_provider_class( original_dataset=dataset ) - - tracks.append( { - "track_type": track_type, - "name": t['name'], - "hda_ldda": t.get("hda_ldda", "hda"), - "dataset_id": trans.security.encode_id( dataset.id ), - "prefs": prefs, - "filters": track_data_provider.get_filters(), - "tool": get_tool_def( trans, dataset ), - "is_child": t.get('is_child', False) - } ) - - config = { "title": visualization.title, "vis_id": trans.security.encode_id( visualization.id ), - "tracks": tracks, "chrom": "", "dbkey": visualization.dbkey } + tracks.append( pack_collection( drawable_dict ) ) + + config = { "title": visualization.title, "vis_id": trans.security.encode_id( visualization.id ), + "tracks": tracks, "bookmarks": bookmarks, "chrom": "", "dbkey": visualization.dbkey } if 'viewport' in latest_revision.config: config['viewport'] = latest_revision.config['viewport'] - + return config - + class UsesStoredWorkflow( SharableItemSecurity ): """ Mixin for controllers that use StoredWorkflow objects. """ def get_stored_workflow( self, trans, id, check_ownership=True, check_accessible=False ): @@ -212,7 +389,7 @@ class UsesStoredWorkflow( SharableItemSecurity ): if not workflow: error( "Workflow not found" ) else: - return self.security_check( trans.get_user(), workflow, check_ownership, check_accessible ) + return self.security_check( trans, workflow, check_ownership, check_accessible ) def get_stored_workflow_steps( self, trans, stored_workflow ): """ Restores states for a stored workflow's steps. """ for step in stored_workflow.latest_workflow.steps: @@ -224,7 +401,7 @@ class UsesStoredWorkflow( SharableItemSecurity ): step.upgrade_messages = module.check_and_update_state() # Any connected input needs to have value DummyDataset (these # are not persisted so we need to do it every time) - module.add_dummy_datasets( connections=step.input_connections ) + module.add_dummy_datasets( connections=step.input_connections ) # Store state with the step step.module = module step.state = module.state @@ -240,34 +417,29 @@ class UsesStoredWorkflow( SharableItemSecurity ): class UsesHistory( SharableItemSecurity ): """ Mixin for controllers that use History objects. """ - def get_history( self, trans, id, check_ownership=True, check_accessible=False ): + def get_history( self, trans, id, check_ownership=True, check_accessible=False, deleted=None ): """Get a History from the database by id, verifying ownership.""" - # Load history from database - try: - history = trans.sa_session.query( trans.model.History ).get( trans.security.decode_id( id ) ) - except TypeError: - history = None - if not history: - error( "History not found" ) - else: - return self.security_check( trans.get_user(), history, check_ownership, check_accessible ) - def get_history_datasets( self, trans, history, show_deleted=False, show_hidden=False): + history = self.get_object( trans, id, 'History', check_ownership=check_ownership, check_accessible=check_accessible, deleted=deleted ) + return self.security_check( trans, history, check_ownership, check_accessible ) + def get_history_datasets( self, trans, history, show_deleted=False, show_hidden=False, show_purged=False ): """ Returns history's datasets. """ query = trans.sa_session.query( trans.model.HistoryDatasetAssociation ) \ .filter( trans.model.HistoryDatasetAssociation.history == history ) \ .options( eagerload( "children" ) ) \ - .join( "dataset" ).filter( trans.model.Dataset.purged == False ) \ + .join( "dataset" ) \ .options( eagerload_all( "dataset.actions" ) ) \ .order_by( trans.model.HistoryDatasetAssociation.hid ) if not show_deleted: query = query.filter( trans.model.HistoryDatasetAssociation.deleted == False ) + if not show_purged: + query = query.filter( trans.model.Dataset.purged == False ) return query.all() class UsesFormDefinitions: """Mixin for controllers that use Galaxy form objects.""" def get_all_forms( self, trans, all_versions=False, filter=None, form_type='All' ): """ - Return all the latest forms from the form_definition_current table + Return all the latest forms from the form_definition_current table if all_versions is set to True. Otherwise return all the versions of all the forms from the form_definition table. """ @@ -681,7 +853,7 @@ class UsesFormDefinitions: trans.sa_session.flush() info_association = sra.run else: - info_association = assoc.run + info_association = assoc.run else: info_association = None if info_association: @@ -909,7 +1081,7 @@ class UsesFormDefinitions: else: field_value = int( input_text_value ) elif field_type == CheckboxField.__name__: - field_value = CheckboxField.is_checked( input_value ) + field_value = CheckboxField.is_checked( input_value ) elif field_type == PasswordField.__name__: field_value = kwd.get( field_name, '' ) else: @@ -1032,40 +1204,45 @@ class Sharable: @web.require_login( "share Galaxy items" ) def set_public_username( self, trans, id, username, **kwargs ): """ Set user's public username and delegate to sharing() """ - trans.get_user().username = username + user = trans.get_user() + message = validate_username( trans, username, user ) + if message: + return trans.fill_template( '/sharing_base.mako', item=self.get_item( trans, id ), message=message, status='error' ) + user.username = username trans.sa_session.flush return self.sharing( trans, id, **kwargs ) + # Abstract methods. @web.expose @web.require_login( "modify Galaxy items" ) def set_slug_async( self, trans, id, new_slug ): """ Set item slug asynchronously. """ - pass + raise "Unimplemented Method" @web.expose @web.require_login( "share Galaxy items" ) def sharing( self, trans, id, **kwargs ): """ Handle item sharing. """ - pass + raise "Unimplemented Method" @web.expose @web.require_login( "share Galaxy items" ) def share( self, trans, id=None, email="", **kwd ): """ Handle sharing an item with a particular user. """ - pass + raise "Unimplemented Method" @web.expose def display_by_username_and_slug( self, trans, username, slug ): """ Display item by username and slug. """ - pass + raise "Unimplemented Method" @web.expose @web.json @web.require_login( "get item name and link" ) def get_name_and_link_async( self, trans, id=None ): """ Returns item's name and link. """ - pass + raise "Unimplemented Method" @web.expose @web.require_login("get item content asynchronously") def get_item_content_async( self, trans, id ): """ Returns item content in HTML format. """ - pass + raise "Unimplemented Method" # Helper methods. def _make_item_accessible( self, sa_session, item ): """ Makes item accessible--viewable and importable--and sets item's slug. Does not flush/commit changes, however. Item must have name, user, importable, and slug attributes. """ @@ -1096,7 +1273,14 @@ class Sharable: item.slug = slug return True return False - + def get_item( self, trans, id ): + """ Return item based on id. """ + raise "Unimplemented Method" + +class UsesQuota( object ): + def get_quota( self, trans, id, check_ownership=False, check_accessible=False, deleted=None ): + return self.get_object( trans, id, 'Quota', check_ownership=False, check_accessible=False, deleted=deleted ) + """ Deprecated: `BaseController` used to be available under the name `Root` """ @@ -1108,7 +1292,9 @@ class Admin( object ): user_list_grid = None role_list_grid = None group_list_grid = None - + quota_list_grid = None + repository_list_grid = None + @web.expose @web.require_admin def index( self, trans, **kwd ): @@ -1117,8 +1303,12 @@ class Admin( object ): message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) if webapp == 'galaxy': + cloned_repositories = trans.sa_session.query( trans.model.ToolShedRepository ) \ + .filter( trans.model.ToolShedRepository.deleted == False ) \ + .first() return trans.fill_template( '/webapps/galaxy/admin/index.mako', webapp=webapp, + cloned_repositories=cloned_repositories, message=message, status=status ) else: @@ -1140,22 +1330,14 @@ class Admin( object ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) + toolbox = self.app.toolbox + if params.get( 'reload_tool_button', False ): + tool_id = params.tool_id + message, status = toolbox.reload_tool_by_id( tool_id ) return trans.fill_template( '/admin/reload_tool.mako', - toolbox=self.app.toolbox, + toolbox=toolbox, message=message, status=status ) - @web.expose - @web.require_admin - def tool_reload( self, trans, tool_version=None, **kwd ): - params = util.Params( kwd ) - tool_id = params.tool_id - self.app.toolbox.reload( tool_id ) - message = 'Reloaded tool: ' + tool_id - return trans.fill_template( '/admin/reload_tool.mako', - toolbox=self.app.toolbox, - message=message, - status='done' ) - # Galaxy Role Stuff @web.expose @web.require_admin @@ -1278,20 +1460,23 @@ class Admin( object ): if not new_name: message = 'Enter a valid name' status='error' - elif trans.sa_session.query( trans.app.model.Role ).filter( trans.app.model.Role.table.c.name==new_name ).first(): - message = 'A role with that name already exists' - status = 'error' else: - role.name = new_name - role.description = new_description - trans.sa_session.add( role ) - trans.sa_session.flush() - message = "Role '%s' has been renamed to '%s'" % ( old_name, new_name ) - return trans.response.send_redirect( web.url_for( controller='admin', - action='roles', - webapp=webapp, - message=util.sanitize_text( message ), - status='done' ) ) + existing_role = trans.sa_session.query( trans.app.model.Role ).filter( trans.app.model.Role.table.c.name==new_name ).first() + if existing_role and existing_role.id != role.id: + message = 'A role with that name already exists' + status = 'error' + else: + if not ( role.name == new_name and role.description == new_description ): + role.name = new_name + role.description = new_description + trans.sa_session.add( role ) + trans.sa_session.flush() + message = "Role '%s' has been renamed to '%s'" % ( old_name, new_name ) + return trans.response.send_redirect( web.url_for( controller='admin', + action='roles', + webapp=webapp, + message=util.sanitize_text( message ), + status='done' ) ) return trans.fill_template( '/admin/dataset_security/role/role_rename.mako', role=role, webapp=webapp, @@ -1336,7 +1521,7 @@ class Admin( object ): action='roles', webapp=webapp, message=util.sanitize_text( message ), - status=status ) ) + status=status ) ) in_users = [] out_users = [] in_groups = [] @@ -1556,19 +1741,22 @@ class Admin( object ): if not new_name: message = 'Enter a valid name' status = 'error' - elif trans.sa_session.query( trans.app.model.Group ).filter( trans.app.model.Group.table.c.name==new_name ).first(): - message = 'A group with that name already exists' - status = 'error' else: - group.name = new_name - trans.sa_session.add( group ) - trans.sa_session.flush() - message = "Group '%s' has been renamed to '%s'" % ( old_name, new_name ) - return trans.response.send_redirect( web.url_for( controller='admin', - action='groups', - webapp=webapp, - message=util.sanitize_text( message ), - status='done' ) ) + existing_group = trans.sa_session.query( trans.app.model.Group ).filter( trans.app.model.Group.table.c.name==new_name ).first() + if existing_group and existing_group.id != group.id: + message = 'A group with that name already exists' + status = 'error' + else: + if group.name != new_name: + group.name = new_name + trans.sa_session.add( group ) + trans.sa_session.flush() + message = "Group '%s' has been renamed to '%s'" % ( old_name, new_name ) + return trans.response.send_redirect( web.url_for( controller='admin', + action='groups', + webapp=webapp, + message=util.sanitize_text( message ), + status='done' ) ) return trans.fill_template( '/admin/dataset_security/group/group_rename.mako', group=group, webapp=webapp, @@ -1925,7 +2113,7 @@ class Admin( object ): def purge_user( self, trans, **kwd ): # This method should only be called for a User that has previously been deleted. # We keep the User in the database ( marked as purged ), and stuff associated - # with the user's private role in case we want the ability to unpurge the user + # with the user's private role in case we want the ability to unpurge the user # some time in the future. # Purging a deleted User deletes all of the following: # - History where user_id = User.id @@ -2020,16 +2208,6 @@ class Admin( object ): **kwd ) ) elif operation == "manage roles and groups": return self.manage_roles_and_groups_for_user( trans, **kwd ) - elif operation == "tools_by_user": - # This option is called via the ToolsColumn link in a grid subclass, - # so we need to add user_id to kwd since id in the subclass is tool.id, - # and update the current sort filter, using the grid subclass's default - # sort filter instead of this class's. - kwd[ 'user_id' ] = kwd[ 'id' ] - kwd[ 'sort' ] = 'name' - return trans.response.send_redirect( web.url_for( controller='admin', - action='browse_tools', - **kwd ) ) # Render the list view return self.user_list_grid( trans, **kwd ) @web.expose @@ -2159,7 +2337,7 @@ class Admin( object ): @web.expose @web.require_admin - def jobs( self, trans, stop = [], stop_msg = None, cutoff = 180, job_lock = None, **kwd ): + def jobs( self, trans, stop = [], stop_msg = None, cutoff = 180, job_lock = None, ajl_submit = None, **kwd ): deleted = [] msg = None status = None @@ -2182,10 +2360,11 @@ class Admin( object ): msg += ' for deletion: ' msg += ', '.join( deleted ) status = 'done' - if job_lock == 'lock': - trans.app.job_manager.job_queue.job_lock = True - elif job_lock == 'unlock': - trans.app.job_manager.job_queue.job_lock = False + if ajl_submit: + if job_lock == 'on': + trans.app.job_manager.job_queue.job_lock = True + else: + trans.app.job_manager.job_queue.job_lock = False cutoff_time = datetime.utcnow() - timedelta( seconds=int( cutoff ) ) jobs = trans.sa_session.query( trans.app.model.Job ) \ .filter( and_( trans.app.model.Job.table.c.update_time < cutoff_time, @@ -2210,7 +2389,18 @@ class Admin( object ): job_lock = trans.app.job_manager.job_queue.job_lock ) ## ---- Utility methods ------------------------------------------------------- - + +def copy_sample_loc_file( trans, filename ): + """Copy xxx.loc.sample to ~/tool-data/xxx.loc.sample and ~/tool-data/xxx.loc""" + head, sample_loc_file = os.path.split( filename ) + loc_file = sample_loc_file.replace( '.sample', '' ) + tool_data_path = os.path.abspath( trans.app.config.tool_data_path ) + # It's ok to overwrite the .sample version of the file. + shutil.copy( os.path.abspath( filename ), os.path.join( tool_data_path, sample_loc_file ) ) + # Only create the .loc file if it does not yet exist. We don't + # overwrite it in case it contains stuff proprietary to the local instance. + if not os.path.exists( os.path.join( tool_data_path, loc_file ) ): + shutil.copy( os.path.abspath( filename ), os.path.join( tool_data_path, loc_file ) ) def get_user( trans, id ): """Get a User from the database by id.""" # Load user from database @@ -2219,6 +2409,12 @@ def get_user( trans, id ): if not user: return trans.show_error_message( "User not found for id (%s)" % str( id ) ) return user +def get_user_by_username( trans, username ): + """Get a user from the database by username""" + # TODO: Add exception handling here. + return trans.sa_session.query( trans.model.User ) \ + .filter( trans.model.User.table.c.username == username ) \ + .one() def get_role( trans, id ): """Get a Role from the database by id.""" # Load user from database @@ -2235,3 +2431,51 @@ def get_group( trans, id ): if not group: return trans.show_error_message( "Group not found for id (%s)" % str( id ) ) return group +def get_quota( trans, id ): + """Get a Quota from the database by id.""" + # Load user from database + id = trans.security.decode_id( id ) + quota = trans.sa_session.query( trans.model.Quota ).get( id ) + return quota +def handle_sample_tool_data_table_conf_file( trans, filename ): + """ + Parse the incoming filename and add new entries to the in-memory + trans.app.tool_data_tables dictionary as well as appending them + to the shed's tool_data_table_conf.xml file on disk. + """ + # Parse the incoming file and add new entries to the in-memory + # trans.app.tool_data_tables dictionary. + error = False + message = '' + try: + new_table_elems = trans.app.tool_data_tables.add_new_entries_from_config_file( filename ) + except Exception, e: + message = str( e ) + error = True + if not error: + # Add an entry to the end of the tool_data_table_conf.xml file. + tdt_config = "%s/tool_data_table_conf.xml" % trans.app.config.root + if os.path.exists( tdt_config ): + # Make a backup of the file since we're going to be changing it. + today = date.today() + backup_date = today.strftime( "%Y_%m_%d" ) + tdt_config_copy = '%s/tool_data_table_conf.xml_%s_backup' % ( trans.app.config.root, backup_date ) + shutil.copy( os.path.abspath( tdt_config ), os.path.abspath( tdt_config_copy ) ) + # Write each line of the tool_data_table_conf.xml file, except the last line to a temp file. + fh = tempfile.NamedTemporaryFile( 'wb' ) + tmp_filename = fh.name + fh.close() + new_tdt_config = open( tmp_filename, 'wb' ) + for i, line in enumerate( open( tdt_config, 'rb' ) ): + if line.find( '    ' ) >= 0: + for new_table_elem in new_table_elems: + new_tdt_config.write( ' %s\n' % util.xml_to_string( new_table_elem ).rstrip( '\n' ) ) + new_tdt_config.write( '\n' ) + else: + new_tdt_config.write( line ) + new_tdt_config.close() + shutil.move( tmp_filename, os.path.abspath( tdt_config ) ) + else: + message = "The required file named tool_data_table_conf.xml does not exist in the Galaxy install directory." + error = True + return error, message diff --git a/lib/galaxy/web/buildapp.py b/lib/galaxy/web/buildapp.py index 20b8c00ee00..1f6abb828c1 100644 --- a/lib/galaxy/web/buildapp.py +++ b/lib/galaxy/web/buildapp.py @@ -22,12 +22,12 @@ import galaxy.model.mapping import galaxy.datatypes.registry import galaxy.web.framework -def add_controllers( webapp, app ): +def add_ui_controllers( webapp, app ): """ Search for controllers in the 'galaxy.web.controllers' module and add them to the webapp. """ - from galaxy.web.base.controller import BaseController + from galaxy.web.base.controller import BaseUIController from galaxy.web.base.controller import ControllerUnavailable import galaxy.web.controllers controller_dir = galaxy.web.controllers.__path__[0] @@ -45,11 +45,11 @@ def add_controllers( webapp, app ): # Look for a controller inside the modules for key in dir( module ): T = getattr( module, key ) - if isclass( T ) and T is not BaseController and issubclass( T, BaseController ): - webapp.add_controller( name, T( app ) ) + if isclass( T ) and T is not BaseUIController and issubclass( T, BaseUIController ): + webapp.add_ui_controller( name, T( app ) ) def add_api_controllers( webapp, app ): - from galaxy.web.base.controller import BaseController + from galaxy.web.base.controller import BaseAPIController from galaxy.web.base.controller import ControllerUnavailable import galaxy.web.api controller_dir = galaxy.web.api.__path__[0] @@ -66,7 +66,7 @@ def add_api_controllers( webapp, app ): module = getattr( module, comp ) for key in dir( module ): T = getattr( module, key ) - if isclass( T ) and T is not BaseController and issubclass( T, BaseController ): + if isclass( T ) and T is not BaseAPIController and issubclass( T, BaseAPIController ): webapp.add_api_controller( name, T( app ) ) def app_factory( global_conf, **kwargs ): @@ -87,13 +87,15 @@ def app_factory( global_conf, **kwargs ): atexit.register( app.shutdown ) # Create the universe WSGI application webapp = galaxy.web.framework.WebApplication( app, session_cookie='galaxysession' ) - add_controllers( webapp, app ) + add_ui_controllers( webapp, app ) # Force /history to go to /root/history -- needed since the tests assume this webapp.add_route( '/history', controller='root', action='history' ) # These two routes handle our simple needs at the moment webapp.add_route( '/async/:tool_id/:data_id/:data_secret', controller='async', action='index', tool_id=None, data_id=None, data_secret=None ) webapp.add_route( '/:controller/:action', action='index' ) webapp.add_route( '/:action', controller='root', action='index' ) + # allow for subdirectories in extra_files_path + webapp.add_route( '/datasets/:dataset_id/display/{filename:.+?}', controller='dataset', action='display', dataset_id=None, filename=None) webapp.add_route( '/datasets/:dataset_id/:action/:filename', controller='dataset', action='index', dataset_id=None, filename=None) webapp.add_route( '/display_application/:dataset_id/:app_name/:link_name/:user_id/:app_action/:action_param', controller='dataset', action='display_application', dataset_id=None, user_id=None, app_name = None, link_name = None, app_action = None, action_param = None ) webapp.add_route( '/u/:username/d/:slug', controller='dataset', action='display_by_username_and_slug' ) @@ -104,19 +106,34 @@ def app_factory( global_conf, **kwargs ): # If enabled, add the web API if asbool( kwargs.get( 'enable_api', False ) ): add_api_controllers( webapp, app ) - webapp.api_mapper.resource( 'content', - 'contents', + webapp.api_mapper.resource( 'content', + 'contents', + controller='library_contents', + name_prefix='library_', path_prefix='/api/libraries/:library_id', parent_resources=dict( member_name='library', collection_name='libraries' ) ) + webapp.api_mapper.resource( 'content', + 'contents', + controller='history_contents', + name_prefix='history_', + path_prefix='/api/histories/:history_id', + parent_resources=dict( member_name='history', collection_name='histories' ) ) + webapp.api_mapper.resource( 'permission', + 'permissions', + path_prefix='/api/libraries/:library_id', + parent_resources=dict( member_name='library', collection_name='libraries' ) ) webapp.api_mapper.resource( 'library', 'libraries', path_prefix='/api' ) webapp.api_mapper.resource( 'sample', 'samples', path_prefix='/api' ) webapp.api_mapper.resource( 'request', 'requests', path_prefix='/api' ) webapp.api_mapper.resource( 'form', 'forms', path_prefix='/api' ) webapp.api_mapper.resource( 'request_type', 'request_types', path_prefix='/api' ) webapp.api_mapper.resource( 'role', 'roles', path_prefix='/api' ) - webapp.api_mapper.resource( 'user', 'users', path_prefix='/api' ) + webapp.api_mapper.resource_with_deleted( 'quota', 'quotas', path_prefix='/api' ) + webapp.api_mapper.resource_with_deleted( 'user', 'users', path_prefix='/api' ) webapp.api_mapper.resource( 'workflow', 'workflows', path_prefix='/api' ) - + webapp.api_mapper.resource_with_deleted( 'history', 'histories', path_prefix='/api' ) + #webapp.api_mapper.connect( 'run_workflow', '/api/workflow/{workflow_id}/library/{library_id}', controller='workflows', action='run', workflow_id=None, library_id=None, conditions=dict(method=["GET"]) ) + webapp.finalize_config() # Wrap the webapp in some useful middleware if kwargs.get( 'middleware', True ): diff --git a/lib/galaxy/web/controllers/admin.py b/lib/galaxy/web/controllers/admin.py index b2297cf668b..6ca4cb6e328 100644 --- a/lib/galaxy/web/controllers/admin.py +++ b/lib/galaxy/web/controllers/admin.py @@ -5,6 +5,10 @@ from galaxy.web.framework.helpers import time_ago, iff, grids import logging log = logging.getLogger( __name__ ) +from galaxy.actions.admin import AdminActions +from galaxy.web.params import QuotaParamParser +from galaxy.exceptions import * + class UserListGrid( grids.Grid ): class EmailColumn( grids.TextColumn ): def get_value( self, trans, grid, user ): @@ -169,7 +173,7 @@ class RoleListGrid( grids.Grid ): global_actions = [ grids.GridAction( "Add new role", dict( controller='admin', action='roles', operation='create' ) ) ] - operations = [ grids.GridOperation( "Rename", + operations = [ grids.GridOperation( "Edit", condition=( lambda item: not item.deleted ), allow_multiple=False, url_args=dict( webapp="galaxy", action="rename_role" ) ), @@ -268,8 +272,890 @@ class GroupListGrid( grids.Grid ): preserve_state = False use_paging = True -class AdminGalaxy( BaseController, Admin ): +class QuotaListGrid( grids.Grid ): + class NameColumn( grids.TextColumn ): + def get_value( self, trans, grid, quota ): + return quota.name + class DescriptionColumn( grids.TextColumn ): + def get_value( self, trans, grid, quota ): + if quota.description: + return quota.description + return '' + class AmountColumn( grids.TextColumn ): + def get_value( self, trans, grid, quota ): + return quota.operation + quota.display_amount + class StatusColumn( grids.GridColumn ): + def get_value( self, trans, grid, quota ): + if quota.deleted: + return "deleted" + elif quota.default: + return "default for %s users" % quota.default[0].type + return "" + class UsersColumn( grids.GridColumn ): + def get_value( self, trans, grid, quota ): + if quota.users: + return len( quota.users ) + return 0 + class GroupsColumn( grids.GridColumn ): + def get_value( self, trans, grid, quota ): + if quota.groups: + return len( quota.groups ) + return 0 + + # Grid definition + webapp = "galaxy" + title = "Quotas" + model_class = model.Quota + template='/admin/quota/grid.mako' + default_sort_key = "name" + columns = [ + NameColumn( "Name", + key="name", + link=( lambda item: dict( operation="Change amount", id=item.id, webapp="galaxy" ) ), + model_class=model.Quota, + attach_popup=True, + filterable="advanced" ), + DescriptionColumn( "Description", + key='description', + model_class=model.Quota, + attach_popup=False, + filterable="advanced" ), + AmountColumn( "Amount", + key='amount', + model_class=model.Quota, + attach_popup=False, + filterable="advanced" ), + UsersColumn( "Users", attach_popup=False ), + GroupsColumn( "Groups", attach_popup=False ), + StatusColumn( "Status", attach_popup=False ), + # Columns that are valid for filtering but are not visible. + grids.DeletedColumn( "Deleted", key="deleted", visible=False, filterable="advanced" ) + ] + columns.append( grids.MulticolFilterColumn( "Search", + cols_to_filter=[ columns[0], columns[1], columns[2] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) + global_actions = [ + grids.GridAction( "Add new quota", dict( controller='admin', action='quotas', operation='create' ) ) + ] + operations = [ grids.GridOperation( "Rename", + condition=( lambda item: not item.deleted ), + allow_multiple=False, + url_args=dict( webapp="galaxy", action="rename_quota" ) ), + grids.GridOperation( "Change amount", + condition=( lambda item: not item.deleted ), + allow_multiple=False, + url_args=dict( webapp="galaxy", action="edit_quota" ) ), + grids.GridOperation( "Manage users and groups", + condition=( lambda item: not item.default and not item.deleted ), + allow_multiple=False, + url_args=dict( webapp="galaxy", action="manage_users_and_groups_for_quota" ) ), + grids.GridOperation( "Set as different type of default", + condition=( lambda item: item.default ), + allow_multiple=False, + url_args=dict( webapp="galaxy", action="set_quota_default" ) ), + grids.GridOperation( "Set as default", + condition=( lambda item: not item.default and not item.deleted ), + allow_multiple=False, + url_args=dict( webapp="galaxy", action="set_quota_default" ) ), + grids.GridOperation( "Unset as default", + condition=( lambda item: item.default and not item.deleted ), + allow_multiple=False, + url_args=dict( webapp="galaxy", action="unset_quota_default" ) ), + grids.GridOperation( "Delete", + condition=( lambda item: not item.deleted and not item.default ), + allow_multiple=True, + url_args=dict( webapp="galaxy", action="mark_quota_deleted" ) ), + grids.GridOperation( "Undelete", + condition=( lambda item: item.deleted ), + allow_multiple=True, + url_args=dict( webapp="galaxy", action="undelete_quota" ) ), + grids.GridOperation( "Purge", + condition=( lambda item: item.deleted ), + allow_multiple=True, + url_args=dict( webapp="galaxy", action="purge_quota" ) ) ] + standard_filters = [ + grids.GridColumnFilter( "Active", args=dict( deleted=False ) ), + grids.GridColumnFilter( "Deleted", args=dict( deleted=True ) ), + grids.GridColumnFilter( "All", args=dict( deleted='All' ) ) + ] + num_rows_per_page = 50 + preserve_state = False + use_paging = True + +class RepositoryListGrid( grids.Grid ): + class NameColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_shed_repository ): + return tool_shed_repository.name + class DescriptionColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_shed_repository ): + return tool_shed_repository.description + class OwnerColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_shed_repository ): + return tool_shed_repository.owner + class RevisionColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_shed_repository ): + return tool_shed_repository.changeset_revision + class ToolShedColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_shed_repository ): + return tool_shed_repository.tool_shed + # Grid definition + title = "Tool shed repositories" + model_class = model.ToolShedRepository + template='/admin/tool_shed_repository/grid.mako' + default_sort_key = "name" + columns = [ + NameColumn( "Name", + key="name", + attach_popup=True ), + DescriptionColumn( "Description" ), + OwnerColumn( "Owner" ), + RevisionColumn( "Revision" ), + ToolShedColumn( "Tool shed" ), + # Columns that are valid for filtering but are not visible. + grids.DeletedColumn( "Deleted", + key="deleted", + visible=False, + filterable="advanced" ) + ] + columns.append( grids.MulticolFilterColumn( "Search repository name", + cols_to_filter=[ columns[0] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) + operations = [ grids.GridOperation( "Get updates", + allow_multiple=False, + condition=( lambda item: not item.deleted ), + async_compatible=False ) ] + standard_filters = [] + default_filter = dict( deleted="False" ) + num_rows_per_page = 50 + preserve_state = False + use_paging = True + def build_initial_query( self, trans, **kwd ): + return trans.sa_session.query( self.model_class ) + +class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuota, QuotaParamParser ): user_list_grid = UserListGrid() role_list_grid = RoleListGrid() group_list_grid = GroupListGrid() + quota_list_grid = QuotaListGrid() + repository_list_grid = RepositoryListGrid() + + @web.expose + @web.require_admin + def quotas( self, trans, **kwargs ): + if 'operation' in kwargs: + operation = kwargs.pop('operation').lower() + if operation == "quotas": + return self.quota( trans, **kwargs ) + if operation == "create": + return self.create_quota( trans, **kwargs ) + if operation == "delete": + return self.mark_quota_deleted( trans, **kwargs ) + if operation == "undelete": + return self.undelete_quota( trans, **kwargs ) + if operation == "purge": + return self.purge_quota( trans, **kwargs ) + if operation == "change amount": + return self.edit_quota( trans, **kwargs ) + if operation == "manage users and groups": + return self.manage_users_and_groups_for_quota( trans, **kwargs ) + if operation == "rename": + return self.rename_quota( trans, **kwargs ) + if operation == "edit": + return self.edit_quota( trans, **kwargs ) + # Render the list view + return self.quota_list_grid( trans, **kwargs ) + @web.expose + @web.require_admin + def create_quota( self, trans, **kwd ): + params = self.get_quota_params( kwd ) + if params.get( 'create_quota_button', False ): + try: + quota, message = self._create_quota( params ) + return trans.response.send_redirect( web.url_for( controller='admin', + action='quotas', + webapp=params.webapp, + message=util.sanitize_text( message ), + status='done' ) ) + except MessageException, e: + params.message = str( e ) + params.status = 'error' + in_users = map( int, params.in_users ) + in_groups = map( int, params.in_groups ) + new_in_users = [] + new_in_groups = [] + for user in trans.sa_session.query( trans.app.model.User ) \ + .filter( trans.app.model.User.table.c.deleted==False ) \ + .order_by( trans.app.model.User.table.c.email ): + if user.id in in_users: + new_in_users.append( ( user.id, user.email ) ) + else: + params.out_users.append( ( user.id, user.email ) ) + for group in trans.sa_session.query( trans.app.model.Group ) \ + .filter( trans.app.model.Group.table.c.deleted==False ) \ + .order_by( trans.app.model.Group.table.c.name ): + if group.id in in_groups: + new_in_groups.append( ( group.id, group.name ) ) + else: + params.out_groups.append( ( group.id, group.name ) ) + return trans.fill_template( '/admin/quota/quota_create.mako', + webapp=params.webapp, + name=params.name, + description=params.description, + amount=params.amount, + operation=params.operation, + default=params.default, + in_users=new_in_users, + out_users=params.out_users, + in_groups=new_in_groups, + out_groups=params.out_groups, + message=params.message, + status=params.status ) + @web.expose + @web.require_admin + def rename_quota( self, trans, **kwd ): + quota, params = self._quota_op( trans, 'rename_quota_button', self._rename_quota, kwd ) + if not quota: + return + return trans.fill_template( '/admin/quota/quota_rename.mako', + id=params.id, + name=params.name or quota.name, + description=params.description or quota.description, + webapp=params.webapp, + message=params.message, + status=params.status ) + @web.expose + @web.require_admin + def manage_users_and_groups_for_quota( self, trans, **kwd ): + quota, params = self._quota_op( trans, 'quota_members_edit_button', self._manage_users_and_groups_for_quota, kwd ) + if not quota: + return + in_users = [] + out_users = [] + in_groups = [] + out_groups = [] + for user in trans.sa_session.query( trans.app.model.User ) \ + .filter( trans.app.model.User.table.c.deleted==False ) \ + .order_by( trans.app.model.User.table.c.email ): + if user in [ x.user for x in quota.users ]: + in_users.append( ( user.id, user.email ) ) + else: + out_users.append( ( user.id, user.email ) ) + for group in trans.sa_session.query( trans.app.model.Group ) \ + .filter( trans.app.model.Group.table.c.deleted==False ) \ + .order_by( trans.app.model.Group.table.c.name ): + if group in [ x.group for x in quota.groups ]: + in_groups.append( ( group.id, group.name ) ) + else: + out_groups.append( ( group.id, group.name ) ) + return trans.fill_template( '/admin/quota/quota.mako', + id=params.id, + name=quota.name, + in_users=in_users, + out_users=out_users, + in_groups=in_groups, + out_groups=out_groups, + webapp=params.webapp, + message=params.message, + status=params.status ) + @web.expose + @web.require_admin + def edit_quota( self, trans, **kwd ): + quota, params = self._quota_op( trans, 'edit_quota_button', self._edit_quota, kwd ) + if not quota: + return + return trans.fill_template( '/admin/quota/quota_edit.mako', + id=params.id, + operation=params.operation or quota.operation, + display_amount=params.amount or quota.display_amount, + webapp=params.webapp, + message=params.message, + status=params.status ) + @web.expose + @web.require_admin + def set_quota_default( self, trans, **kwd ): + quota, params = self._quota_op( trans, 'set_default_quota_button', self._set_quota_default, kwd ) + if not quota: + return + if params.default: + default = params.default + elif quota.default: + default = quota.default[0].type + else: + default = "no" + return trans.fill_template( '/admin/quota/quota_set_default.mako', + id=params.id, + default=default, + webapp=params.webapp, + message=params.message, + status=params.status ) + @web.expose + @web.require_admin + def unset_quota_default( self, trans, **kwd ): + quota, params = self._quota_op( trans, True, self._unset_quota_default, kwd ) + if not quota: + return + return trans.response.send_redirect( web.url_for( controller='admin', + action='quotas', + webapp=params.webapp, + message=util.sanitize_text( params.message ), + status='error' ) ) + @web.expose + @web.require_admin + def mark_quota_deleted( self, trans, **kwd ): + quota, params = self._quota_op( trans, True, self._mark_quota_deleted, kwd, listify=True ) + if not quota: + return + return trans.response.send_redirect( web.url_for( controller='admin', + action='quotas', + webapp=params.webapp, + message=util.sanitize_text( params.message ), + status='error' ) ) + @web.expose + @web.require_admin + def undelete_quota( self, trans, **kwd ): + quota, params = self._quota_op( trans, True, self._undelete_quota, kwd, listify=True ) + if not quota: + return + return trans.response.send_redirect( web.url_for( controller='admin', + action='quotas', + webapp=params.webapp, + message=util.sanitize_text( params.message ), + status='error' ) ) + @web.expose + @web.require_admin + def purge_quota( self, trans, **kwd ): + quota, params = self._quota_op( trans, True, self._purge_quota, kwd, listify=True ) + if not quota: + return + return trans.response.send_redirect( web.url_for( controller='admin', + action='quotas', + webapp=params.webapp, + message=util.sanitize_text( params.message ), + status='error' ) ) + def _quota_op( self, trans, do_op, op_method, kwd, listify=False ): + params = self.get_quota_params( kwd ) + if listify: + quota = [] + messages = [] + for id in util.listify( params.id ): + try: + quota.append( self.get_quota( trans, id ) ) + except MessageException, e: + messages.append( str( e ) ) + if messages: + return None, trans.response.send_redirect( web.url_for( controller='admin', + action='quotas', + webapp=params.webapp, + message=util.sanitize_text( ', '.join( messages ) ), + status='error' ) ) + else: + try: + quota = self.get_quota( trans, params.id, deleted=False ) + except MessageException, e: + return None, trans.response.send_redirect( web.url_for( controller='admin', + action='quotas', + webapp=params.webapp, + message=util.sanitize_text( str( e ) ), + status='error' ) ) + if do_op == True or ( do_op != False and params.get( do_op, False ) ): + try: + message = op_method( quota, params ) + return None, trans.response.send_redirect( web.url_for( controller='admin', + action='quotas', + webapp=params.webapp, + message=util.sanitize_text( message ), + status='done' ) ) + except MessageException, e: + params.message = e.err_msg + params.status = e.type + return quota, params + @web.expose + @web.require_admin + def browse_repositories( self, trans, **kwd ): + if 'operation' in kwd: + operation = kwd.pop('operation').lower() + if operation == "get updates": + return self.check_for_updates( trans, **kwd ) + # Render the list view + return self.repository_list_grid( trans, **kwd ) + @web.expose + @web.require_admin + def browse_tool_shed( self, trans, **kwd ): + tool_shed_url = kwd[ 'tool_shed_url' ] + galaxy_url = trans.request.host + url = '%s/repository/browse_downloadable_repositories?galaxy_url=%s&webapp=community' % ( tool_shed_url, galaxy_url ) + return trans.response.send_redirect( url ) + @web.expose + @web.require_admin + def install_tool_shed_repository( self, trans, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + tool_shed_url = kwd[ 'tool_shed_url' ] + name = kwd[ 'name' ] + description = kwd[ 'description' ] + changeset_revision = kwd[ 'changeset_revision' ] + repository_clone_url = kwd[ 'repository_clone_url' ] + if kwd.get( 'select_tool_panel_section_button', False ): + shed_tool_conf = kwd[ 'shed_tool_conf' ] + # Get the tool path. + for k, tool_path in trans.app.toolbox.shed_tool_confs.items(): + if k == shed_tool_conf: + break + if 'tool_panel_section' in kwd: + section_key = 'section_%s' % kwd[ 'tool_panel_section' ] + tool_section = trans.app.toolbox.tool_panel[ section_key ] + # Clone the repository to the configured location. + current_working_dir = os.getcwd() + clone_dir = os.path.join( tool_path, self.__generate_tool_path( repository_clone_url, changeset_revision ) ) + if os.path.exists( clone_dir ): + # Repository and revision has already been cloned. + # TODO: implement the ability to re-install or revert an existing repository. + message = 'Revision %s of repository %s has already been installed. Updating an existing repository is not yet supported.' % \ + ( changeset_revision, name ) + status = 'error' + else: + os.makedirs( clone_dir ) + log.debug( 'Cloning %s...' % repository_clone_url ) + cmd = 'hg clone %s' % repository_clone_url + tmp_name = tempfile.NamedTemporaryFile().name + tmp_stderr = open( tmp_name, 'wb' ) + os.chdir( clone_dir ) + proc = subprocess.Popen( args=cmd, shell=True, stderr=tmp_stderr.fileno() ) + returncode = proc.wait() + os.chdir( current_working_dir ) + tmp_stderr.close() + if returncode == 0: + # Add a new record to the tool_shed_repository table. + tool_shed_repository = self.__create_tool_shed_repository( trans, + name, + description, + changeset_revision, + repository_clone_url ) + # Update the cloned repository to changeset_revision. + repo_files_dir = os.path.join( clone_dir, name ) + log.debug( 'Updating cloned repository to revision "%s"...' % changeset_revision ) + cmd = 'hg update -r %s' % changeset_revision + tmp_name = tempfile.NamedTemporaryFile().name + tmp_stderr = open( tmp_name, 'wb' ) + os.chdir( repo_files_dir ) + proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) + returncode = proc.wait() + os.chdir( current_working_dir ) + tmp_stderr.close() + if returncode == 0: + sample_files, repository_tools_tups = self.__get_repository_tools_and_sample_files( trans, tool_path, repo_files_dir ) + if repository_tools_tups: + # Handle missing data table entries for tool parameters that are dynamically generated select lists. + repository_tools_tups = self.__handle_missing_data_table_entry( trans, tool_path, sample_files, repository_tools_tups ) + # Handle missing index files for tool parameters that are dynamically generated select lists. + repository_tools_tups = self.__handle_missing_index_file( trans, tool_path, sample_files, repository_tools_tups ) + # Handle tools that use fabric scripts to install dependencies. + self.__handle_tool_dependencies( current_working_dir, repo_files_dir, repository_tools_tups ) + # Generate an in-memory tool conf section that includes the new tools. + new_tool_section = self.__generate_tool_panel_section( name, + repository_clone_url, + changeset_revision, + tool_section, + repository_tools_tups ) + # Create a temporary file to persist the in-memory tool section + # TODO: Figure out how to do this in-memory using xml.etree. + tmp_name = tempfile.NamedTemporaryFile().name + persisted_new_tool_section = open( tmp_name, 'wb' ) + persisted_new_tool_section.write( new_tool_section ) + persisted_new_tool_section.close() + # Parse the persisted tool panel section + tree = ElementTree.parse( tmp_name ) + root = tree.getroot() + ElementInclude.include( root ) + # Load the tools in the section into the tool panel. + trans.app.toolbox.load_section_tag_set( root, trans.app.toolbox.tool_panel, tool_path ) + # Remove the temporary file + try: + os.unlink( tmp_name ) + except: + pass + # Append the new section to the shed_tool_config file. + self.__add_shed_tool_conf_entry( trans, shed_tool_conf, new_tool_section ) + message = 'Revision %s of repository %s has been installed in tool panel section %s.' % \ + ( changeset_revision, name, tool_section.name ) + return trans.show_ok_message( message ) + else: + tmp_stderr = open( tmp_name, 'rb' ) + message = tmp_stderr.read() + tmp_stderr.close() + status = 'error' + else: + tmp_stderr = open( tmp_name, 'rb' ) + message = tmp_stderr.read() + tmp_stderr.close() + status = 'error' + else: + message = 'Choose the section in your tool panel to contain the installed tools.' + status = 'error' + if len( trans.app.toolbox.shed_tool_confs.keys() ) > 1: + shed_tool_conf_select_field = build_shed_tool_conf_select_field( trans ) + shed_tool_conf = None + else: + shed_tool_conf = trans.app.toolbox.shed_tool_confs.keys()[0].lstrip( './' ) + shed_tool_conf_select_field = None + tool_panel_section_select_field = build_tool_panel_section_select_field( trans ) + return trans.fill_template( '/admin/select_tool_panel_section.mako', + tool_shed_url=tool_shed_url, + name=name, + description=description, + changeset_revision=changeset_revision, + repository_clone_url=repository_clone_url, + shed_tool_conf=shed_tool_conf, + shed_tool_conf_select_field=shed_tool_conf_select_field, + tool_panel_section_select_field=tool_panel_section_select_field, + message=message, + status=status ) + @web.expose + @web.require_admin + def check_for_updates( self, trans, **kwd ): + params = util.Params( kwd ) + repository_id = params.get( 'id', None ) + repository = get_repository( trans, repository_id ) + galaxy_url = trans.request.host + # Send a request to the relevant tool shed to see if there are any updates. + # TODO: support https in the following url. + url = 'http://%s/repository/check_for_updates?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=community' % \ + ( repository.tool_shed, galaxy_url, repository.name, repository.owner, repository.changeset_revision ) + return trans.response.send_redirect( url ) + @web.expose + @web.require_admin + def update_to_changeset_revision( self, trans, **kwd ): + """Update a cloned repository to the latest revision possible.""" + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + tool_shed_url = kwd[ 'tool_shed_url' ] + name = params.get( 'name', None ) + owner = params.get( 'owner', None ) + changeset_revision = params.get( 'changeset_revision', None ) + latest_changeset_revision = params.get( 'latest_changeset_revision', None ) + if changeset_revision and latest_changeset_revision: + if changeset_revision == latest_changeset_revision: + message = "The cloned tool shed repository named '%s' is current (there are no updates available)." % name + else: + repository = get_repository_by_name_owner_changeset_revision( trans, name, owner, changeset_revision ) + current_working_dir = os.getcwd() + # Get the directory where the repository is cloned. + cleaned_tool_shed_url = self.__clean_tool_shed_url( tool_shed_url ) + partial_cloned_dir = '%s/repos/%s/%s/%s' % ( cleaned_tool_shed_url, owner, name, changeset_revision ) + # Get the relative tool installation paths from each of the shed tool configs. + shed_tool_confs = trans.app.toolbox.shed_tool_confs + relative_cloned_dir = None + # The shed_tool_confs dictionary contains shed_conf_filename : tool_path pairs. + for shed_conf_filename, tool_path in shed_tool_confs.items(): + relative_cloned_dir = os.path.join( tool_path, partial_cloned_dir ) + if os.path.isdir( relative_cloned_dir ): + break + if relative_cloned_dir: + # Update the cloned repository to changeset_revision. + repo_files_dir = os.path.join( relative_cloned_dir, name ) + log.debug( "Updating cloned repository named '%s' from revision '%s' to revision '%s'..." % \ + ( name, changeset_revision, latest_changeset_revision ) ) + cmd = 'hg pull' + tmp_name = tempfile.NamedTemporaryFile().name + tmp_stderr = open( tmp_name, 'wb' ) + os.chdir( repo_files_dir ) + proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) + returncode = proc.wait() + os.chdir( current_working_dir ) + tmp_stderr.close() + if returncode == 0: + cmd = 'hg update -r %s' % latest_changeset_revision + tmp_name = tempfile.NamedTemporaryFile().name + tmp_stderr = open( tmp_name, 'wb' ) + os.chdir( repo_files_dir ) + proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) + returncode = proc.wait() + os.chdir( current_working_dir ) + tmp_stderr.close() + if returncode == 0: + # Update the repository changeset_revision in the database. + repository.changeset_revision = latest_changeset_revision + trans.sa_session.add( repository ) + trans.sa_session.flush() + message = "The cloned repository named '%s' has been updated to change set revision '%s'." % \ + ( name, latest_changeset_revision ) + else: + tmp_stderr = open( tmp_name, 'rb' ) + message = tmp_stderr.read() + tmp_stderr.close() + status = 'error' + else: + tmp_stderr = open( tmp_name, 'rb' ) + message = tmp_stderr.read() + tmp_stderr.close() + status = 'error' + else: + message = "The directory containing the cloned repository named '%s' cannot be found." % name + status = 'error' + else: + message = "The latest changeset revision could not be retrieved for the repository named '%s'." % name + status = 'error' + return trans.response.send_redirect( web.url_for( controller='admin', + action='browse_repositories', + message=message, + status=status ) ) + def __handle_missing_data_table_entry( self, trans, tool_path, sample_files, repository_tools_tups ): + # Inspect each tool to see if any have input parameters that are dynamically + # generated select lists that require entries in the tool_data_table_conf.xml file. + missing_data_table_entry = False + for index, repository_tools_tup in enumerate( repository_tools_tups ): + tup_path, repository_tool = repository_tools_tup + if repository_tool.params_with_missing_data_table_entry: + missing_data_table_entry = True + break + if missing_data_table_entry: + # The repository must contain a tool_data_table_conf.xml.sample file that includes + # all required entries for all tools in the repository. + for sample_file in sample_files: + head, tail = os.path.split( sample_file ) + if tail == 'tool_data_table_conf.xml.sample': + break + error, correction_msg = handle_sample_tool_data_table_conf_file( trans, sample_file ) + if error: + # TODO: Do more here than logging an exception. + log.debug( exception_msg ) + # Reload the tool into the local list of repository_tools_tups. + repository_tool = trans.app.toolbox.load_tool( os.path.join( tool_path, tup_path ) ) + repository_tools_tups[ index ] = ( tup_path, repository_tool ) + return repository_tools_tups + def __handle_missing_index_file( self, trans, tool_path, sample_files, repository_tools_tups ): + # Inspect each tool to see if it has any input parameters that + # are dynamically generated select lists that depend on a .loc file. + missing_files_handled = [] + for index, repository_tools_tup in enumerate( repository_tools_tups ): + tup_path, repository_tool = repository_tools_tup + params_with_missing_index_file = repository_tool.params_with_missing_index_file + for param in params_with_missing_index_file: + options = param.options + missing_head, missing_tail = os.path.split( options.missing_index_file ) + if missing_tail not in missing_files_handled: + # The repository must contain the required xxx.loc.sample file. + for sample_file in sample_files: + sample_head, sample_tail = os.path.split( sample_file ) + if sample_tail == '%s.sample' % missing_tail: + copy_sample_loc_file( trans, sample_file ) + if options.tool_data_table and options.tool_data_table.missing_index_file: + options.tool_data_table.handle_found_index_file( options.missing_index_file ) + missing_files_handled.append( missing_tail ) + break + # Reload the tool into the local list of repository_tools_tups. + repository_tool = trans.app.toolbox.load_tool( os.path.join( tool_path, tup_path ) ) + repository_tools_tups[ index ] = ( tup_path, repository_tool ) + return repository_tools_tups + def __handle_tool_dependencies( self, current_working_dir, repo_files_dir, repository_tools_tups ): + # Inspect each tool to see if it includes a "requirement" that refers to a fabric + # script. For those that do, execute the fabric script to install tool dependencies. + for index, repository_tools_tup in enumerate( repository_tools_tups ): + tup_path, repository_tool = repository_tools_tup + for requirement in repository_tool.requirements: + if requirement.type == 'fabfile': + log.debug( 'Executing fabric script to install dependencies for tool "%s"...' % repository_tool.name ) + fabfile = requirement.fabfile + method = requirement.method + # Find the relative path to the fabfile. + relative_fabfile_path = None + for root, dirs, files in os.walk( repo_files_dir ): + for name in files: + if name == fabfile: + relative_fabfile_path = os.path.join( root, name ) + break + if relative_fabfile_path: + # cmd will look something like: fab -f fabfile.py install_bowtie + cmd = 'fab -f %s %s' % ( relative_fabfile_path, method ) + tmp_name = tempfile.NamedTemporaryFile().name + tmp_stderr = open( tmp_name, 'wb' ) + os.chdir( repo_files_dir ) + proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) + returncode = proc.wait() + os.chdir( current_working_dir ) + tmp_stderr.close() + if returncode != 0: + # TODO: do something more here than logging the problem. + tmp_stderr = open( tmp_name, 'rb' ) + error = tmp_stderr.read() + tmp_stderr.close() + log.debug( 'Problem installing dependencies for tool "%s"\n%s' % ( repository_tool.name, error ) ) + def __get_repository_tools_and_sample_files( self, trans, tool_path, repo_files_dir ): + # The sample_files list contains all files whose name ends in .sample + sample_files = [] + # The repository_tools_tups list contains tuples of ( relative_path_to_tool_config, tool ) pairs + repository_tools_tups = [] + for root, dirs, files in os.walk( repo_files_dir ): + if not root.find( '.hg' ) >= 0 and not root.find( 'hgrc' ) >= 0: + if '.hg' in dirs: + # Don't visit .hg directories - should be impossible since we don't + # allow uploaded archives that contain .hg dirs, but just in case... + dirs.remove( '.hg' ) + if 'hgrc' in files: + # Don't include hgrc files in commit. + files.remove( 'hgrc' ) + # Find all special .sample files first. + for name in files: + if name.endswith( '.sample' ): + sample_files.append( os.path.abspath( os.path.join( root, name ) ) ) + for name in files: + # Find all tool configs. + if name.endswith( '.xml' ): + relative_path = os.path.join( root, name ) + full_path = os.path.abspath( os.path.join( root, name ) ) + try: + repository_tool = trans.app.toolbox.load_tool( full_path ) + if repository_tool: + # At this point, we need to lstrip tool_path from relative_path. + tup_path = relative_path.replace( tool_path, '' ).lstrip( '/' ) + repository_tools_tups.append( ( tup_path, repository_tool ) ) + except Exception, e: + # We have an invalid .xml file, so not a tool config. + log.debug( "Ignoring invalid tool config (%s). Error: %s" % ( str( relative_path ), str( e ) ) ) + return sample_files, repository_tools_tups + def __create_tool_shed_repository( self, trans, name, description, changeset_revision, repository_clone_url ): + tmp_url = self.__clean_repository_clone_url( repository_clone_url ) + tool_shed = tmp_url.split( 'repos' )[ 0 ].rstrip( '/' ) + owner = self.__get_repository_owner( tmp_url ) + tool_shed_repository = trans.model.ToolShedRepository( tool_shed=tool_shed, + name=name, + description=description, + owner=owner, + changeset_revision=changeset_revision ) + trans.sa_session.add( tool_shed_repository ) + trans.sa_session.flush() + def __add_shed_tool_conf_entry( self, trans, shed_tool_conf, new_tool_section ): + # Add an entry in the shed_tool_conf file. An entry looks something like: + #
    + # + #
    + # Make a backup of the hgweb.config file since we're going to be changing it. + if not os.path.exists( shed_tool_conf ): + output = open( shed_tool_conf, 'w' ) + output.write( '\n' ) + output.write( '\n' % tool_path ) + output.write( '\n' ) + output.close() + self.__make_shed_tool_conf_copy( trans, shed_tool_conf ) + tmp_fd, tmp_fname = tempfile.mkstemp() + new_shed_tool_conf = open( tmp_fname, 'wb' ) + for i, line in enumerate( open( shed_tool_conf ) ): + if line.startswith( '' ): + # We're at the end of the original config file, so add our entry. + new_shed_tool_conf.write( new_tool_section ) + new_shed_tool_conf.write( line ) + else: + new_shed_tool_conf.write( line ) + new_shed_tool_conf.close() + shutil.move( tmp_fname, os.path.abspath( shed_tool_conf ) ) + def __make_shed_tool_conf_copy( self, trans, shed_tool_conf ): + # Make a backup of the shed_tool_conf file. + today = date.today() + backup_date = today.strftime( "%Y_%m_%d" ) + shed_tool_conf_copy = '%s/%s_%s_backup' % ( trans.app.config.root, shed_tool_conf, backup_date ) + shutil.copy( os.path.abspath( shed_tool_conf ), os.path.abspath( shed_tool_conf_copy ) ) + def __clean_tool_shed_url( self, tool_shed_url ): + if tool_shed_url.find( ':' ) > 0: + # Eliminate the port, if any, since it will result in an invalid directory name. + return tool_shed_url.split( ':' )[ 0 ] + return tool_shed_url.rstrip( '/' ) + def __clean_repository_clone_url( self, repository_clone_url ): + if repository_clone_url.find( '@' ) > 0: + # We have an url that includes an authenticated user, something like: + # http://test@bx.psu.edu:9009/repos/some_username/column + items = repository_clone_url.split( '@' ) + tmp_url = items[ 1 ] + elif repository_clone_url.find( '\/\/' ) > 0: + # We have an url that includes only a protocol, something like: + # http://bx.psu.edu:9009/repos/some_username/column + items = repository_clone_url.split( '\/\/' ) + tmp_url = items[ 1 ] + else: + tmp_url = repository_clone_url + return tmp_url + def __get_repository_owner( self, cleaned_repository_url ): + items = cleaned_repository_url.split( 'repos' ) + repo_path = items[ 1 ] + return repo_path.lstrip( '/' ).split( '/' )[ 0 ] + def __generate_tool_path( self, repository_clone_url, changeset_revision ): + """ + Generate a tool path that guarantees repositories with the same name will always be installed + in different directories. The tool path will be of the form: + /repos/// + http://test@bx.psu.edu:9009/repos/test/filter + """ + tmp_url = self.__clean_repository_clone_url( repository_clone_url ) + # Now tmp_url is something like: bx.psu.edu:9009/repos/some_username/column + items = tmp_url.split( 'repos' ) + tool_shed_url = items[ 0 ] + repo_path = items[ 1 ] + tool_shed_url = self.__clean_tool_shed_url( tool_shed_url ) + return '%s/repos%s/%s' % ( tool_shed_url, repo_path, changeset_revision ) + def __generate_tool_guid( self, repository_clone_url, tool ): + """ + Generate a guid for the installed tool. It is critical that this guid matches the guid for + the tool in the Galaxy tool shed from which it is being installed. The form of the guid is + /repos//// + """ + tmp_url = self.__clean_repository_clone_url( repository_clone_url ) + return '%s/%s/%s' % ( tmp_url, tool.id, tool.version ) + def __generate_tool_panel_section( self, repository_name, repository_clone_url, changeset_revision, tool_section, repository_tools_tups ): + """ + Write an in-memory tool panel section so we can load it into the tool panel and then + append it to the appropriate shed tool config. + TODO: re-write using ElementTree. + """ + tmp_url = self.__clean_repository_clone_url( repository_clone_url ) + section_str = '' + section_str += '
    \n' % ( tool_section.name, tool_section.id ) + for repository_tool_tup in repository_tools_tups: + tool_file_path, tool = repository_tool_tup + guid = self.__generate_tool_guid( repository_clone_url, tool ) + section_str += ' \n' % ( tool_file_path, guid ) + section_str += ' %s\n' % tmp_url.split( 'repos' )[ 0 ].rstrip( '/' ) + section_str += ' %s\n' % repository_name + section_str += ' %s\n' % self.__get_repository_owner( tmp_url ) + section_str += ' %s\n' % changeset_revision + section_str += ' %s\n' % tool.id + section_str += ' %s\n' % tool.version + section_str += ' \n' + section_str += '
    \n' + return section_str + +## ---- Utility methods ------------------------------------------------------- + +def build_shed_tool_conf_select_field( trans ): + """Build a SelectField whose options are the keys in trans.app.toolbox.shed_tool_confs.""" + options = [] + for shed_tool_conf_filename, tool_path in trans.app.toolbox.shed_tool_confs.items(): + options.append( ( shed_tool_conf_filename.lstrip( './' ), shed_tool_conf_filename ) ) + select_field = SelectField( name='shed_tool_conf' ) + for option_tup in options: + select_field.add_option( option_tup[0], option_tup[1] ) + return select_field +def build_tool_panel_section_select_field( trans ): + """Build a SelectField whose options are the sections of the current in-memory toolbox.""" + options = [] + for k, tool_section in trans.app.toolbox.tool_panel.items(): + options.append( ( tool_section.name, tool_section.id ) ) + select_field = SelectField( name='tool_panel_section', display='radio' ) + for option_tup in options: + select_field.add_option( option_tup[0], option_tup[1] ) + return select_field +def get_repository( trans, id ): + """Get a tool_shed_repository from the database via id""" + return trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( id ) ) +def get_repository_by_name_owner_changeset_revision( trans, name, owner, changeset_revision ): + """Get a repository from the database via name owner and changeset_revision""" + return trans.sa_session.query( trans.model.ToolShedRepository ) \ + .filter( and_( trans.model.ToolShedRepository.table.c.name == name, + trans.model.ToolShedRepository.table.c.owner == owner, + trans.model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \ + .first() diff --git a/lib/galaxy/web/controllers/async.py b/lib/galaxy/web/controllers/async.py index 3ae5233611b..e17d15ffcf1 100644 --- a/lib/galaxy/web/controllers/async.py +++ b/lib/galaxy/web/controllers/async.py @@ -11,7 +11,7 @@ from galaxy.util.hash_util import * log = logging.getLogger( __name__ ) -class ASync( BaseController ): +class ASync( BaseUIController ): @web.expose def default(self, trans, tool_id=None, data_id=None, data_secret=None, **kwd): diff --git a/lib/galaxy/web/controllers/dataset.py b/lib/galaxy/web/controllers/dataset.py index 8d3a2fab60d..32b6ed0de43 100644 --- a/lib/galaxy/web/controllers/dataset.py +++ b/lib/galaxy/web/controllers/dataset.py @@ -1,4 +1,4 @@ -import logging, os, string, shutil, re, socket, mimetypes, smtplib, urllib, tempfile, zipfile, glob, sys +import logging, os, string, shutil, re, socket, mimetypes, urllib, tempfile, zipfile, glob, sys from galaxy.web.base.controller import * from galaxy.web.framework.helpers import time_ago, iff, grids @@ -9,8 +9,8 @@ from galaxy.util.sanitize_html import sanitize_html from galaxy.util import inflector from galaxy.model.item_attrs import * from galaxy.model import LibraryDatasetDatasetAssociation, HistoryDatasetAssociation +from galaxy.web.framework.helpers import to_unicode -from email.MIMEText import MIMEText import pkg_resources; pkg_resources.require( "Paste" ) import paste.httpexceptions @@ -145,7 +145,7 @@ class HistoryDatasetAssociationListGrid( grids.Grid ): .filter( model.History.deleted==False ) \ .filter( self.model_class.visible==True ) -class DatasetInterface( BaseController, UsesAnnotations, UsesHistory, UsesHistoryDatasetAssociation, UsesItemRatings ): +class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHistoryDatasetAssociation, UsesItemRatings ): stored_list_grid = HistoryDatasetAssociationListGrid() @@ -174,7 +174,7 @@ class DatasetInterface( BaseController, UsesAnnotations, UsesHistory, UsesHistor host = trans.request.host history_view_link = "%s/history/view?id=%s" % ( str( host ), trans.security.encode_id( hda.history_id ) ) # Build the email message - msg = MIMEText( string.Template( error_report_template ) + body = string.Template( error_report_template ) \ .safe_substitute( host=host, dataset_id=hda.dataset_id, history_id=hda.history_id, @@ -189,7 +189,7 @@ class DatasetInterface( BaseController, UsesAnnotations, UsesHistory, UsesHistor job_info=job.info, job_traceback=job.traceback, email=email, - message=message ) ) + message=message ) frm = to_address # Check email a bit email = email.strip() @@ -198,15 +198,10 @@ class DatasetInterface( BaseController, UsesAnnotations, UsesHistory, UsesHistor to = to_address + ", " + email else: to = to_address - msg[ 'To' ] = to - msg[ 'From' ] = frm - msg[ 'Subject' ] = "Galaxy tool error report from " + email + subject = "Galaxy tool error report from " + email # Send it try: - s = smtplib.SMTP() - s.connect( smtp_server ) - s.sendmail( frm, [ to ], msg.as_string() ) - s.close() + util.send_mail( frm, to, subject, body, trans.app.config ) return trans.show_ok_message( "Your error report has been sent" ) except Exception, e: return trans.show_error_message( "An error occurred sending the report by email: %s" % str( e ) ) @@ -223,7 +218,7 @@ class DatasetInterface( BaseController, UsesAnnotations, UsesHistory, UsesHistor outfname = data.name[0:150] outfname = ''.join(c in valid_chars and c or '_' for c in outfname) if (params.do_action == None): - params.do_action = 'zip' # default + params.do_action = 'zip' # default msg = util.restore_text( params.get( 'msg', '' ) ) messagetype = params.get( 'messagetype', 'done' ) if not data: @@ -267,17 +262,18 @@ class DatasetInterface( BaseController, UsesAnnotations, UsesHistory, UsesHistor log.exception( "Unable to add composite parent %s to temporary library download archive" % data.file_name) msg = "Unable to create archive for download, please report this error" messagetype = 'error' - flist = glob.glob(os.path.join(efp,'*.*')) # glob returns full paths - for fpath in flist: - efp,fname = os.path.split(fpath) - try: - archive.add( fpath,fname ) - except IOError: - error = True - log.exception( "Unable to add %s to temporary library download archive" % fname) - msg = "Unable to create archive for download, please report this error" - messagetype = 'error' - continue + for root, dirs, files in os.walk(efp): + for fname in files: + fpath = os.path.join(root,fname) + rpath = os.path.relpath(fpath,efp) + try: + archive.add( fpath,rpath ) + except IOError: + error = True + log.exception( "Unable to add %s to temporary library download archive" % rpath) + msg = "Unable to create archive for download, please report this error" + messagetype = 'error' + continue if not error: if params.do_action == 'zip': archive.close() @@ -308,7 +304,7 @@ class DatasetInterface( BaseController, UsesAnnotations, UsesHistory, UsesHistor @web.expose - def get_metadata_file(self, trans, hda_id, metadata_type): + def get_metadata_file(self, trans, hda_id, metadata_name): """ Allows the downloading of metadata files associated with datasets (eg. bai index for bam files) """ data = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( trans.security.decode_id( hda_id ) ) if not data or not trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), data.dataset ): @@ -316,8 +312,11 @@ class DatasetInterface( BaseController, UsesAnnotations, UsesHistory, UsesHistor valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' fname = ''.join(c in valid_chars and c or '_' for c in data.name)[0:150] - trans.response.headers["Content-Disposition"] = "attachment; filename=Galaxy%s-[%s].%s" % (data.hid, fname, metadata_type) - return open(data.metadata.get(metadata_type).file_name) + + file_ext = data.metadata.spec.get(metadata_name).get("file_ext", metadata_name) + trans.response.headers["Content-Type"] = "application/octet-stream" + trans.response.headers["Content-Disposition"] = "attachment; filename=Galaxy%s-[%s].%s" % (data.hid, fname, file_ext) + return open(data.metadata.get(metadata_name).file_name) @web.expose def display(self, trans, dataset_id=None, preview=False, filename=None, to_ext=None, **kwd): @@ -346,20 +345,20 @@ class DatasetInterface( BaseController, UsesAnnotations, UsesHistory, UsesHistor # For files in extra_files_path file_path = os.path.join( data.extra_files_path, filename ) if os.path.exists( file_path ): + if os.path.isdir( file_path ): + return trans.show_error_message( "Directory listing is not allowed." ) #TODO: Reconsider allowing listing of directories? mime, encoding = mimetypes.guess_type( file_path ) if not mime: try: mime = trans.app.datatypes_registry.get_mimetype_by_extension( ".".split( file_path )[-1] ) except: mime = "text/plain" - trans.response.set_content_type( mime ) return open( file_path ) else: - return "Could not find '%s' on the extra files path %s." % (filename,file_path) + return trans.show_error_message( "Could not find '%s' on the extra files path %s." % ( filename, file_path ) ) - mime = trans.app.datatypes_registry.get_mimetype_by_extension( data.extension.lower() ) - trans.response.set_content_type(mime) + trans.response.set_content_type(data.get_mime()) trans.log_event( "Display dataset id: %s" % str( dataset_id ) ) if to_ext or isinstance(data.datatype, datatypes.binary.Binary): # Saving the file, or binary file @@ -371,12 +370,15 @@ class DatasetInterface( BaseController, UsesAnnotations, UsesHistory, UsesHistor to_ext = data.extension valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' fname = ''.join(c in valid_chars and c or '_' for c in data.name)[0:150] + trans.response.set_content_type( "application/octet-stream" ) #force octet-stream so Safari doesn't append mime extensions to filename trans.response.headers["Content-Disposition"] = "attachment; filename=Galaxy%s-[%s].%s" % (data.hid, fname, to_ext) return open( data.file_name ) if not os.path.exists( data.file_name ): raise paste.httpexceptions.HTTPNotFound( "File Not Found (%s)." % data.file_name ) max_peek_size = 1000000 # 1 MB + if isinstance(data.datatype, datatypes.images.Html): + max_peek_size = 10000000 # 10 MB for html if not preview or isinstance(data.datatype, datatypes.images.Image) or os.stat( data.file_name ).st_size < max_peek_size: return open( data.file_name ) else: @@ -384,6 +386,188 @@ class DatasetInterface( BaseController, UsesAnnotations, UsesHistory, UsesHistor return trans.stream_template_mako( "/dataset/large_file.mako", truncated_data = open( data.file_name ).read(max_peek_size), data = data ) + + @web.expose + def edit(self, trans, dataset_id=None, filename=None, hid=None, **kwd): + """Allows user to modify parameters of an HDA.""" + message = None + status = 'done' + refresh_frames = [] + error = False + def __ok_to_edit_metadata( dataset_id ): + #prevent modifying metadata when dataset is queued or running as input/output + #This code could be more efficient, i.e. by using mappers, but to prevent slowing down loading a History panel, we'll leave the code here for now + for job_to_dataset_association in trans.sa_session.query( self.app.model.JobToInputDatasetAssociation ) \ + .filter_by( dataset_id=dataset_id ) \ + .all() \ + + trans.sa_session.query( self.app.model.JobToOutputDatasetAssociation ) \ + .filter_by( dataset_id=dataset_id ) \ + .all(): + if job_to_dataset_association.job.state not in [ job_to_dataset_association.job.states.OK, job_to_dataset_association.job.states.ERROR, job_to_dataset_association.job.states.DELETED ]: + return False + return True + if hid is not None: + history = trans.get_history() + # TODO: hid handling + data = history.datasets[ int( hid ) - 1 ] + id = None + elif dataset_id is not None: + id = trans.app.security.decode_id( dataset_id ) + data = trans.sa_session.query( self.app.model.HistoryDatasetAssociation ).get( id ) + else: + trans.log_event( "dataset_id and hid are both None, cannot load a dataset to edit" ) + return trans.show_error_message( "You must provide a history dataset id to edit" ) + if data is None: + trans.log_event( "Problem retrieving dataset (encoded: %s, decoded: %s) with history id %s." % ( str( dataset_id ), str( id ), str( hid ) ) ) + return trans.show_error_message( "History dataset id is invalid" ) + if dataset_id is not None and data.history.user is not None and data.history.user != trans.user: + trans.log_event( "User attempted to edit an HDA they do not own (encoded: %s, decoded: %s)" % ( dataset_id, id ) ) + # Do not reveal the dataset's existence + return trans.show_error_message( "History dataset id is invalid" ) + current_user_roles = trans.get_current_user_roles() + if data.history.user and not data.dataset.has_manage_permissions_roles( trans ): + # Permission setting related to DATASET_MANAGE_PERMISSIONS was broken for a period of time, + # so it is possible that some Datasets have no roles associated with the DATASET_MANAGE_PERMISSIONS + # permission. In this case, we'll reset this permission to the hda user's private role. + manage_permissions_action = trans.app.security_agent.get_action( trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS.action ) + permissions = { manage_permissions_action : [ trans.app.security_agent.get_private_user_role( data.history.user ) ] } + trans.app.security_agent.set_dataset_permission( data.dataset, permissions ) + if trans.app.security_agent.can_access_dataset( current_user_roles, data.dataset ): + if data.state == trans.model.Dataset.states.UPLOAD: + return trans.show_error_message( "Please wait until this dataset finishes uploading before attempting to edit its metadata." ) + params = util.Params( kwd, sanitize=False ) + if params.change: + # The user clicked the Save button on the 'Change data type' form + if data.datatype.allow_datatype_change and trans.app.datatypes_registry.get_datatype_by_extension( params.datatype ).allow_datatype_change: + #prevent modifying datatype when dataset is queued or running as input/output + if not __ok_to_edit_metadata( data.id ): + message = "This dataset is currently being used as input or output. You cannot change datatype until the jobs have completed or you have canceled them." + error = True + else: + trans.app.datatypes_registry.change_datatype( data, params.datatype, set_meta = not trans.app.config.set_metadata_externally ) + trans.sa_session.flush() + if trans.app.config.set_metadata_externally: + trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, trans, incoming = { 'input1':data }, overwrite = False ) #overwrite is False as per existing behavior + message = "Changed the type of dataset '%s' to %s" % ( to_unicode( data.name ), params.datatype ) + refresh_frames=['history'] + else: + message = "You are unable to change datatypes in this manner. Changing %s to %s is not allowed." % ( data.extension, params.datatype ) + error = True + elif params.save: + # The user clicked the Save button on the 'Edit Attributes' form + data.name = params.name + data.info = params.info + message = '' + if __ok_to_edit_metadata( data.id ): + # The following for loop will save all metadata_spec items + for name, spec in data.datatype.metadata_spec.items(): + if spec.get("readonly"): + continue + optional = params.get("is_"+name, None) + other = params.get("or_"+name, None) + if optional and optional == 'true': + # optional element... == 'true' actually means it is NOT checked (and therefore omitted) + setattr(data.metadata, name, None) + else: + if other: + setattr( data.metadata, name, other ) + else: + setattr( data.metadata, name, spec.unwrap( params.get (name, None) ) ) + data.datatype.after_setting_metadata( data ) + # Sanitize annotation before adding it. + if params.annotation: + annotation = sanitize_html( params.annotation, 'utf-8', 'text/html' ) + self.add_item_annotation( trans.sa_session, trans.get_user(), data, annotation ) + # If setting metadata previously failed and all required elements have now been set, clear the failed state. + if data._state == trans.model.Dataset.states.FAILED_METADATA and not data.missing_meta(): + data._state = None + trans.sa_session.flush() + message = "Attributes updated%s" % message + refresh_frames=['history'] + else: + trans.sa_session.flush() + message = "Attributes updated, but metadata could not be changed because this dataset is currently being used as input or output. You must cancel or wait for these jobs to complete before changing metadata." + status = "warning" + refresh_frames=['history'] + elif params.detect: + # The user clicked the Auto-detect button on the 'Edit Attributes' form + #prevent modifying metadata when dataset is queued or running as input/output + if not __ok_to_edit_metadata( data.id ): + message = "This dataset is currently being used as input or output. You cannot change metadata until the jobs have completed or you have canceled them." + error = True + else: + for name, spec in data.metadata.spec.items(): + # We need to be careful about the attributes we are resetting + if name not in [ 'name', 'info', 'dbkey', 'base_name' ]: + if spec.get( 'default' ): + setattr( data.metadata, name, spec.unwrap( spec.get( 'default' ) ) ) + if trans.app.config.set_metadata_externally: + message = 'Attributes have been queued to be updated' + trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, trans, incoming = { 'input1':data } ) + else: + message = 'Attributes updated' + data.set_meta() + data.datatype.after_setting_metadata( data ) + trans.sa_session.flush() + refresh_frames=['history'] + elif params.convert_data: + target_type = kwd.get("target_type", None) + if target_type: + message = data.datatype.convert_dataset(trans, data, target_type) + refresh_frames=['history'] + elif params.update_roles_button: + if not trans.user: + return trans.show_error_message( "You must be logged in if you want to change permissions." ) + if trans.app.security_agent.can_manage_dataset( current_user_roles, data.dataset ): + access_action = trans.app.security_agent.get_action( trans.app.security_agent.permitted_actions.DATASET_ACCESS.action ) + manage_permissions_action = trans.app.security_agent.get_action( trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS.action ) + # The user associated the DATASET_ACCESS permission on the dataset with 1 or more roles. We + # need to ensure that they did not associate roles that would cause accessibility problems. + permissions, in_roles, error, message = \ + trans.app.security_agent.derive_roles_from_access( trans, data.dataset.id, 'root', **kwd ) + if error: + # Keep the original role associations for the DATASET_ACCESS permission on the dataset. + permissions[ access_action ] = data.dataset.get_access_roles( trans ) + status = 'error' + else: + error = trans.app.security_agent.set_all_dataset_permissions( data.dataset, permissions ) + if error: + message += error + status = 'error' + else: + message = 'Your changes completed successfully.' + trans.sa_session.refresh( data.dataset ) + else: + message = "You are not authorized to change this dataset's permissions" + error = True + else: + if "dbkey" in data.datatype.metadata_spec and not data.metadata.dbkey: + # Copy dbkey into metadata, for backwards compatability + # This looks like it does nothing, but getting the dbkey + # returns the metadata dbkey unless it is None, in which + # case it resorts to the old dbkey. Setting the dbkey + # sets it properly in the metadata + #### This is likely no longer required, since the dbkey exists entirely within metadata (the old_dbkey field is gone): REMOVE ME? + data.metadata.dbkey = data.dbkey + # let's not overwrite the imported datatypes module with the variable datatypes? + # the built-in 'id' is overwritten in lots of places as well + ldatatypes = [ dtype_name for dtype_name, dtype_value in trans.app.datatypes_registry.datatypes_by_extension.iteritems() if dtype_value.allow_datatype_change ] + ldatatypes.sort() + all_roles = trans.app.security_agent.get_legitimate_roles( trans, data.dataset, 'root' ) + if error: + status = 'error' + return trans.fill_template( "/dataset/edit_attributes.mako", + data=data, + data_annotation=self.get_item_annotation_str( trans.sa_session, trans.user, data ), + datatypes=ldatatypes, + current_user_roles=current_user_roles, + all_roles=all_roles, + message=message, + status=status, + dataset_id=dataset_id, + refresh_frames=refresh_frames ) + else: + return trans.show_error_message( "You do not have permission to edit this dataset's ( id: %s ) information." % str( dataset_id ) ) @web.expose @web.require_login( "see all available datasets" ) @@ -509,8 +693,7 @@ class DatasetInterface( BaseController, UsesAnnotations, UsesHistory, UsesHistor # If data is binary or an image, stream without template; otherwise, use display template. # TODO: figure out a way to display images in display template. if isinstance(dataset.datatype, datatypes.binary.Binary) or isinstance(dataset.datatype, datatypes.images.Image) or isinstance(dataset.datatype, datatypes.images.Html): - mime = trans.app.datatypes_registry.get_mimetype_by_extension( dataset.extension.lower() ) - trans.response.set_content_type( mime ) + trans.response.set_content_type( dataset.get_mime() ) return open( dataset.file_name ) else: # Get rating data. @@ -655,65 +838,192 @@ class DatasetInterface( BaseController, UsesAnnotations, UsesHistory, UsesHistor return trans.fill_template_mako( "dataset/display_application/display.mako", msg = msg, display_app = display_app, display_link = display_link, refresh = refresh ) return trans.show_error_message( 'You do not have permission to view this dataset at an external display application.' ) - def _undelete( self, trans, id ): + def _delete( self, trans, dataset_id ): + message = None + status = 'done' + id = None try: - id = int( id ) - except ValueError, e: - return False - history = trans.get_history() - data = trans.sa_session.query( self.app.model.HistoryDatasetAssociation ).get( id ) - if data and data.undeletable: + id = trans.app.security.decode_id( dataset_id ) + history = trans.get_history() + hda = trans.sa_session.query( self.app.model.HistoryDatasetAssociation ).get( id ) + assert hda, 'Invalid HDA: %s' % id # Walk up parent datasets to find the containing history - topmost_parent = data + topmost_parent = hda + while topmost_parent.parent: + topmost_parent = topmost_parent.parent + assert topmost_parent in trans.history.datasets, "Data does not belong to current history" + # Mark deleted and cleanup + hda.mark_deleted() + hda.clear_associated_files() + trans.log_event( "Dataset id %s marked as deleted" % str(id) ) + if hda.parent_id is None and len( hda.creating_job_associations ) > 0: + # Mark associated job for deletion + job = hda.creating_job_associations[0].job + if job.state in [ self.app.model.Job.states.QUEUED, self.app.model.Job.states.RUNNING, self.app.model.Job.states.NEW ]: + # Are *all* of the job's other output datasets deleted? + if job.check_if_output_datasets_deleted(): + job.mark_deleted( self.app.config.get_bool( 'enable_job_running', True ), + self.app.config.get_bool( 'track_jobs_in_database', False ) ) + self.app.job_manager.job_stop_queue.put( job.id ) + trans.sa_session.flush() + except Exception, e: + msg = 'HDA deletion failed (encoded: %s, decoded: %s)' % ( dataset_id, id ) + log.exception( msg ) + trans.log_event( msg ) + message = 'Dataset deletion failed' + status = 'error' + return ( message, status ) + + def _undelete( self, trans, dataset_id ): + message = None + status = 'done' + id = None + try: + id = trans.app.security.decode_id( dataset_id ) + history = trans.get_history() + hda = trans.sa_session.query( self.app.model.HistoryDatasetAssociation ).get( id ) + assert hda and hda.undeletable, 'Invalid HDA: %s' % id + # Walk up parent datasets to find the containing history + topmost_parent = hda while topmost_parent.parent: topmost_parent = topmost_parent.parent assert topmost_parent in history.datasets, "Data does not belong to current history" # Mark undeleted - data.mark_undeleted() + hda.mark_undeleted() trans.sa_session.flush() trans.log_event( "Dataset id %s has been undeleted" % str(id) ) - return True - return False + except Exception, e: + msg = 'HDA undeletion failed (encoded: %s, decoded: %s)' % ( dataset_id, id ) + log.exception( msg ) + trans.log_event( msg ) + message = 'Dataset undeletion failed' + status = 'error' + return ( message, status ) - def _unhide( self, trans, id ): + def _unhide( self, trans, dataset_id ): try: - id = int( id ) - except ValueError, e: + id = trans.app.security.decode_id( dataset_id ) + except: return False history = trans.get_history() - data = trans.sa_session.query( self.app.model.HistoryDatasetAssociation ).get( id ) - if data: + hda = trans.sa_session.query( self.app.model.HistoryDatasetAssociation ).get( id ) + if hda: # Walk up parent datasets to find the containing history - topmost_parent = data + topmost_parent = hda while topmost_parent.parent: topmost_parent = topmost_parent.parent assert topmost_parent in history.datasets, "Data does not belong to current history" # Mark undeleted - data.mark_unhidden() + hda.mark_unhidden() trans.sa_session.flush() trans.log_event( "Dataset id %s has been unhidden" % str(id) ) return True return False - @web.expose - def undelete( self, trans, id ): - if self._undelete( trans, id ): - return trans.response.send_redirect( web.url_for( controller='root', action='history', show_deleted = True ) ) - raise "Error undeleting" + def _purge( self, trans, dataset_id ): + message = None + status = 'done' + try: + id = trans.app.security.decode_id( dataset_id ) + history = trans.get_history() + user = trans.get_user() + hda = trans.sa_session.query( self.app.model.HistoryDatasetAssociation ).get( id ) + # Invalid HDA + assert hda, 'Invalid history dataset ID' + # Walk up parent datasets to find the containing history + topmost_parent = hda + while topmost_parent.parent: + topmost_parent = topmost_parent.parent + assert topmost_parent in history.datasets, "Data does not belong to current history" + # If the user is anonymous, make sure the HDA is owned by the current session. + if not user: + assert trans.galaxy_session.current_history_id == trans.history.id, 'Invalid history dataset ID' + # If the user is known, make sure the HDA is owned by the current user. + else: + assert topmost_parent.history.user == trans.user, 'Invalid history dataset ID' + # HDA is not deleted + assert hda.deleted, 'History dataset is not marked as deleted' + # HDA is purgeable + # Decrease disk usage first + if user: + user.total_disk_usage -= hda.quota_amount( user ) + # Mark purged + hda.purged = True + trans.sa_session.add( hda ) + trans.log_event( "HDA id %s has been purged" % hda.id ) + trans.sa_session.flush() + # Don't delete anything if there are active HDAs or any LDDAs, even if + # the LDDAs are deleted. Let the cleanup scripts get it in the latter + # case. + if hda.dataset.user_can_purge: + try: + hda.dataset.full_delete() + trans.log_event( "Dataset id %s has been purged upon the the purge of HDA id %s" % ( hda.dataset.id, hda.id ) ) + trans.sa_session.add( hda.dataset ) + except: + log.exception( 'Unable to purge dataset (%s) on purge of HDA (%s):' % ( hda.dataset.id, hda.id ) ) + trans.sa_session.flush() + except Exception, e: + msg = 'HDA purge failed (encoded: %s, decoded: %s)' % ( dataset_id, id ) + log.exception( msg ) + trans.log_event( msg ) + message = 'Dataset removal from disk failed' + status = 'error' + return ( message, status ) @web.expose - def unhide( self, trans, id ): - if self._unhide( trans, id ): - return trans.response.send_redirect( web.url_for( controller='root', action='history', show_hidden = True ) ) - raise "Error unhiding" - + def delete( self, trans, dataset_id, filename, show_deleted_on_refresh = False ): + message, status = self._delete( trans, dataset_id ) + return trans.response.send_redirect( web.url_for( controller='root', action='history', show_deleted=show_deleted_on_refresh, message=message, status=status ) ) @web.expose - def undelete_async( self, trans, id ): - if self._undelete( trans, id ): + def delete_async( self, trans, dataset_id, filename ): + message, status = self._delete( trans, dataset_id ) + if status == 'done': return "OK" - raise "Error undeleting" + else: + raise Exception( message ) + + @web.expose + def undelete( self, trans, dataset_id, filename ): + message, status = self._undelete( trans, dataset_id ) + return trans.response.send_redirect( web.url_for( controller='root', action='history', show_deleted = True, message=message, status=status ) ) + + @web.expose + def undelete_async( self, trans, dataset_id, filename ): + message, status =self._undelete( trans, dataset_id ) + if status == 'done': + return "OK" + else: + raise Exception( message ) + @web.expose + def unhide( self, trans, dataset_id, filename ): + if self._unhide( trans, dataset_id ): + return trans.response.send_redirect( web.url_for( controller='root', action='history', show_hidden = True ) ) + raise Exception( "Error unhiding" ) + + @web.expose + def purge( self, trans, dataset_id, filename, show_deleted_on_refresh = False ): + if trans.app.config.allow_user_dataset_purge: + message, status = self._purge( trans, dataset_id ) + else: + message = "Removal of datasets by users is not allowed in this Galaxy instance. Please contact your Galaxy administrator." + status = 'error' + return trans.response.send_redirect( web.url_for( controller='root', action='history', show_deleted=show_deleted_on_refresh, message=message, status=status ) ) + + @web.expose + def purge_async( self, trans, dataset_id, filename ): + if trans.app.config.allow_user_dataset_purge: + message, status = self._purge( trans, dataset_id ) + else: + message = "Removal of datasets by users is not allowed in this Galaxy instance. Please contact your Galaxy administrator." + status = 'error' + if status == 'done': + return "OK" + else: + raise Exception( message ) + @web.expose def show_params( self, trans, dataset_id=None, from_noframe=None, **kwd ): """ @@ -790,10 +1100,11 @@ class DatasetInterface( BaseController, UsesAnnotations, UsesHistory, UsesHistor else: target_history_ids = [] done_msg = error_msg = "" + new_history = None if do_copy: invalid_datasets = 0 if not source_dataset_ids or not ( target_history_ids or new_history_name ): - error_msg = "You must provide both source datasets and target histories." + error_msg = "You must provide both source datasets and target histories. " else: if new_history_name: new_history = trans.app.model.History() @@ -808,23 +1119,28 @@ class DatasetInterface( BaseController, UsesAnnotations, UsesHistory, UsesHistor target_histories = [ history ] if len( target_histories ) != len( target_history_ids ): error_msg = error_msg + "You do not have permission to add datasets to %i requested histories. " % ( len( target_history_ids ) - len( target_histories ) ) - for data in map( trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get, source_dataset_ids ): - if data is None: - error_msg = error_msg + "You tried to copy a dataset that does not exist. " + source_hdas = map( trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get, source_dataset_ids ) + source_hdas.sort(key=lambda hda: hda.hid) + for hda in source_hdas: + if hda is None: + error_msg = error_msg + "You tried to copy a dataset that does not exist. " invalid_datasets += 1 - elif data.history != history: - error_msg = error_msg + "You tried to copy a dataset which is not in your current history. " + elif hda.history != history: + error_msg = error_msg + "You tried to copy a dataset which is not in your current history. " invalid_datasets += 1 else: for hist in target_histories: - hist.add_dataset( data.copy( copy_children = True ) ) + hist.add_dataset( hda.copy( copy_children = True ) ) if history in target_histories: refresh_frames = ['history'] trans.sa_session.flush() hist_names_str = ", ".join( [ hist.name for hist in target_histories ] ) num_source = len( source_dataset_ids ) - invalid_datasets num_target = len(target_histories) - done_msg = "%i %s copied to %i %s: %s" % (num_source, inflector.cond_plural(num_source, "dataset"), num_target, inflector.cond_plural(num_target, "history"), hist_names_str ) + done_msg = "%i %s copied to %i %s: %s." % (num_source, inflector.cond_plural(num_source, "dataset"), num_target, inflector.cond_plural(num_target, "history"), hist_names_str ) + if new_history is not None: + done_msg += " Switch to the new history." % url_for( + controller="history", action="switch_to_history", hist_id=trans.security.encode_id( new_history.id ) ) trans.sa_session.refresh( history ) source_datasets = history.visible_datasets target_histories = [history] diff --git a/lib/galaxy/web/controllers/error.py b/lib/galaxy/web/controllers/error.py index ac36200dbf4..47053c39ceb 100644 --- a/lib/galaxy/web/controllers/error.py +++ b/lib/galaxy/web/controllers/error.py @@ -1,6 +1,6 @@ from galaxy.web.base.controller import * -class Error( BaseController ): +class Error( BaseUIController ): @web.expose def index( self, trans ): raise Exception, "Fake error" \ No newline at end of file diff --git a/lib/galaxy/web/controllers/external_service.py b/lib/galaxy/web/controllers/external_service.py index 85a010d32f0..d3f520b0f85 100644 --- a/lib/galaxy/web/controllers/external_service.py +++ b/lib/galaxy/web/controllers/external_service.py @@ -63,7 +63,7 @@ class ExternalServiceGrid( grids.Grid ): grids.GridAction( "Create new external service", dict( controller='external_service', action='create_external_service' ) ) ] -class ExternalService( BaseController, UsesFormDefinitions ): +class ExternalService( BaseUIController, UsesFormDefinitions ): external_service_grid = ExternalServiceGrid() @web.expose diff --git a/lib/galaxy/web/controllers/external_services.py b/lib/galaxy/web/controllers/external_services.py index bc9ecb57bc7..00867498027 100644 --- a/lib/galaxy/web/controllers/external_services.py +++ b/lib/galaxy/web/controllers/external_services.py @@ -9,7 +9,7 @@ class_name_to_class = {} for model_class in [Sample]: class_name_to_class[ model_class.__name__ ] = model_class -class ExternalServiceController( BaseController ): +class ExternalServiceController( BaseUIController ): @web.expose @web.require_admin def access_action( self, trans, external_service_action, item, item_type, **kwd ): diff --git a/lib/galaxy/web/controllers/forms.py b/lib/galaxy/web/controllers/forms.py index 392506d00dd..3d667d10890 100644 --- a/lib/galaxy/web/controllers/forms.py +++ b/lib/galaxy/web/controllers/forms.py @@ -66,7 +66,7 @@ class FormsGrid( grids.Grid ): grids.GridAction( "Create new form", dict( controller='forms', action='create_form_definition' ) ) ] -class Forms( BaseController ): +class Forms( BaseUIController ): # Empty TextField empty_field = { 'name': '', 'label': '', diff --git a/lib/galaxy/web/controllers/history.py b/lib/galaxy/web/controllers/history.py index 1a84ba5b3d2..7e2f27f637b 100644 --- a/lib/galaxy/web/controllers/history.py +++ b/lib/galaxy/web/controllers/history.py @@ -1,5 +1,6 @@ from galaxy.web.base.controller import * from galaxy.web.framework.helpers import time_ago, iff, grids +from galaxy.datatypes.data import nice_size from galaxy import model, util from galaxy.util.odict import odict from galaxy.model.mapping import desc @@ -10,7 +11,8 @@ from galaxy.util.sanitize_html import sanitize_html from galaxy.tools.parameters.basic import UnvalidatedValue from galaxy.tools.actions import upload_common from galaxy.tags.tag_handler import GalaxyTagHandler -from sqlalchemy.sql.expression import ClauseElement +from sqlalchemy.sql.expression import ClauseElement, func +from sqlalchemy.sql import select import webhelpers, logging, operator, os, tempfile, subprocess, shutil, tarfile from datetime import datetime from cgi import escape @@ -25,11 +27,29 @@ class HistoryListGrid( grids.Grid ): # Custom column types class DatasetsByStateColumn( grids.GridColumn ): def get_value( self, trans, grid, history ): + # Build query to get (state, count) pairs. + cols_to_select = [ trans.app.model.Dataset.table.c.state, func.count( '*' ) ] + from_obj = trans.app.model.HistoryDatasetAssociation.table.join( trans.app.model.Dataset.table ) + where_clause = and_( trans.app.model.HistoryDatasetAssociation.table.c.history_id == history.id, + trans.app.model.HistoryDatasetAssociation.table.c.deleted == False, + trans.app.model.HistoryDatasetAssociation.table.c.visible == True, + ) + group_by = trans.app.model.Dataset.table.c.state + query = select( columns=cols_to_select, + from_obj=from_obj, + whereclause=where_clause, + group_by=group_by ) + + # Process results. + state_count_dict = {} + for row in trans.sa_session.execute( query ): + state, count = row + state_count_dict[ state ] = count rval = [] for state in ( 'ok', 'running', 'queued', 'error' ): - total = sum( 1 for d in history.active_datasets if d.state == state ) - if total: - rval.append( '
    %s
    ' % ( state, total ) ) + count = state_count_dict.get( state, 0 ) + if count: + rval.append( '
    %s
    ' % ( state, count ) ) else: rval.append( '' ) return rval @@ -51,23 +71,24 @@ class HistoryListGrid( grids.Grid ): grids.IndividualTagsColumn( "Tags", key="tags", model_tag_association_class=model.HistoryTagAssociation, \ filterable="advanced", grid_name="HistoryListGrid" ), grids.SharingStatusColumn( "Sharing", key="sharing", filterable="advanced", sortable=False ), + grids.GridColumn( "Size on Disk", key="get_disk_size_bytes", format=nice_size, sortable=False ), grids.GridColumn( "Created", key="create_time", format=time_ago ), grids.GridColumn( "Last Updated", key="update_time", format=time_ago ), # Columns that are valid for filtering but are not visible. - grids.DeletedColumn( "Deleted", key="deleted", visible=False, filterable="advanced" ) + grids.DeletedColumn( "Status", key="deleted", visible=False, filterable="advanced" ) ] - columns.append( - grids.MulticolFilterColumn( - "search history names and tags", - cols_to_filter=[ columns[0], columns[2] ], + columns.append( + grids.MulticolFilterColumn( + "search history names and tags", + cols_to_filter=[ columns[0], columns[2] ], key="free-text-search", visible=False, filterable="standard" ) ) - operations = [ grids.GridOperation( "Switch", allow_multiple=False, condition=( lambda item: not item.deleted ), async_compatible=False ), grids.GridOperation( "Share or Publish", allow_multiple=False, condition=( lambda item: not item.deleted ), async_compatible=False ), grids.GridOperation( "Rename", condition=( lambda item: not item.deleted ), async_compatible=False ), grids.GridOperation( "Delete", condition=( lambda item: not item.deleted ), async_compatible=True ), + grids.GridOperation( "Delete Permanently", confirm="History contents will be removed from disk, this cannot be undone. Continue?", async_compatible=True ), grids.GridOperation( "Undelete", condition=( lambda item: item.deleted ), async_compatible=True ), ] standard_filters = [ @@ -122,11 +143,11 @@ class SharedHistoryListGrid( grids.Grid ): return trans.sa_session.query( self.model_class ).join( 'users_shared_with' ) def apply_query_filter( self, trans, query, **kwargs ): return query.filter( model.HistoryUserShareAssociation.user == trans.user ) - + class HistoryAllPublishedGrid( grids.Grid ): class NameURLColumn( grids.PublicURLColumn, NameColumn ): pass - + title = "Published Histories" model_class = model.History default_sort_key = "update_time" @@ -135,15 +156,15 @@ class HistoryAllPublishedGrid( grids.Grid ): columns = [ NameURLColumn( "Name", key="name", filterable="advanced" ), grids.OwnerAnnotationColumn( "Annotation", key="annotation", model_annotation_association_class=model.HistoryAnnotationAssociation, filterable="advanced" ), - grids.OwnerColumn( "Owner", key="username", model_class=model.User, filterable="advanced" ), + grids.OwnerColumn( "Owner", key="username", model_class=model.User, filterable="advanced" ), grids.CommunityRatingColumn( "Community Rating", key="rating" ), grids.CommunityTagsColumn( "Community Tags", key="tags", model_tag_association_class=model.HistoryTagAssociation, filterable="advanced", grid_name="PublicHistoryListGrid" ), grids.ReverseSortColumn( "Last Updated", key="update_time", format=time_ago ) ] - columns.append( - grids.MulticolFilterColumn( - "Search name, annotation, owner, and tags", - cols_to_filter=[ columns[0], columns[1], columns[2], columns[4] ], + columns.append( + grids.MulticolFilterColumn( + "Search name, annotation, owner, and tags", + cols_to_filter=[ columns[0], columns[1], columns[2], columns[4] ], key="free-text-search", visible=False, filterable="standard" ) ) operations = [] @@ -153,8 +174,8 @@ class HistoryAllPublishedGrid( grids.Grid ): def apply_query_filter( self, trans, query, **kwargs ): # A public history is published, has a slug, and is not deleted. return query.filter( self.model_class.published == True ).filter( self.model_class.slug != None ).filter( self.model_class.deleted == False ) - -class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRatings, UsesHistory ): + +class HistoryController( BaseUIController, Sharable, UsesAnnotations, UsesItemRatings, UsesHistory ): @web.expose def index( self, trans ): return "" @@ -163,11 +184,11 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati """XML history list for functional tests""" trans.response.set_content_type( 'text/xml' ) return trans.fill_template( "/history/list_as_xml.mako" ) - + stored_list_grid = HistoryListGrid() shared_list_grid = SharedHistoryListGrid() published_list_grid = HistoryAllPublishedGrid() - + @web.expose def list_published( self, trans, **kwargs ): grid = self.published_list_grid( trans, **kwargs ) @@ -176,7 +197,7 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati else: # Render grid wrapped in panels return trans.fill_template( "history/list_published.mako", grid=grid ) - + @web.expose @web.require_login( "work with multiple histories" ) def list( self, trans, **kwargs ): @@ -197,7 +218,7 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati refresh_history = False # Load the histories and ensure they all belong to the current user histories = [] - for history_id in history_ids: + for history_id in history_ids: history = self.get_history( trans, history_id ) if history: # Ensure history is owned by current user @@ -206,21 +227,24 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati histories.append( history ) else: log.warn( "Invalid history id '%r' passed to list", history_id ) - if histories: + if histories: if operation == "switch": status, message = self._list_switch( trans, histories ) - # Take action to update UI to reflect history switch. If + # Take action to update UI to reflect history switch. If # grid is using panels, it is standalone and hence a redirect # to root is needed; if grid is not using panels, it is nested - # in the main Galaxy UI and refreshing the history frame + # in the main Galaxy UI and refreshing the history frame # is sufficient. use_panels = kwargs.get('use_panels', False) == 'True' if use_panels: return trans.response.send_redirect( url_for( "/" ) ) - else: + else: trans.template_context['refresh_frames'] = ['history'] - elif operation == "delete": - status, message = self._list_delete( trans, histories ) + elif operation in ( "delete", "delete permanently" ): + if operation == "delete permanently": + status, message = self._list_delete( trans, histories, purge=True ) + else: + status, message = self._list_delete( trans, histories ) if current_history in histories: # Deleted the current history, so a new, empty history was # created automatically, and we need to refresh the history frame @@ -245,7 +269,7 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati trans.sa_session.flush() # Render the list view return self.stored_list_grid( trans, status=status, message=message, **kwargs ) - def _list_delete( self, trans, histories ): + def _list_delete( self, trans, histories, purge=False ): """Delete histories""" n_deleted = 0 deleted_current = False @@ -264,8 +288,32 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati trans.new_history() trans.log_event( "History (%s) marked as deleted" % history.name ) n_deleted += 1 + if purge and trans.app.config.allow_user_dataset_purge: + for hda in history.datasets: + if trans.user: + trans.user.total_disk_usage -= hda.quota_amount( trans.user ) + hda.purged = True + trans.sa_session.add( hda ) + trans.log_event( "HDA id %s has been purged" % hda.id ) + trans.sa_session.flush() + if hda.dataset.user_can_purge: + try: + hda.dataset.full_delete() + trans.log_event( "Dataset id %s has been purged upon the the purge of HDA id %s" % ( hda.dataset.id, hda.id ) ) + trans.sa_session.add( hda.dataset ) + except: + log.exception( 'Unable to purge dataset (%s) on purge of hda (%s):' % ( hda.dataset.id, hda.id ) ) + history.purged = True + self.sa_session.add( history ) + self.sa_session.flush() + trans.sa_session.flush() if n_deleted: - message_parts.append( "Deleted %d %s. " % ( n_deleted, iff( n_deleted != 1, "histories", "history" ) ) ) + part = "Deleted %d %s" % ( n_deleted, iff( n_deleted != 1, "histories", "history" ) ) + if purge and trans.app.config.allow_user_dataset_purge: + part += " and removed %s datasets from disk" % iff( n_deleted != 1, "their", "its" ) + elif purge: + part += " but the datasets were not removed from disk because that feature is not enabled in this Galaxy instance" + message_parts.append( "%s. " % part ) if deleted_current: message_parts.append( "Your active history was deleted, a new empty history is now active. " ) status = INFO @@ -314,7 +362,7 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati trans.set_history( new_history ) # No message return None, None - + @web.expose @web.require_login( "work with shared histories" ) def list_shared( self, trans, **kwargs ): @@ -349,7 +397,7 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati status = 'done' # Render the list view return self.shared_list_grid( trans, status=status, message=message, **kwargs ) - + @web.expose def display_structured( self, trans, id=None ): """ @@ -420,9 +468,32 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati items.sort( key=( lambda x: x[0].create_time ), reverse=True ) # return trans.fill_template( "history/display_structured.mako", items=items ) - + @web.expose - def delete_current( self, trans ): + def purge_deleted_datasets( self, trans ): + count = 0 + if trans.app.config.allow_user_dataset_purge: + for hda in trans.history.datasets: + if not hda.deleted or hda.purged: + continue + if trans.user: + trans.user.total_disk_usage -= hda.quota_amount( trans.user ) + hda.purged = True + trans.sa_session.add( hda ) + trans.log_event( "HDA id %s has been purged" % hda.id ) + trans.sa_session.flush() + if hda.dataset.user_can_purge: + try: + hda.dataset.full_delete() + trans.log_event( "Dataset id %s has been purged upon the the purge of HDA id %s" % ( hda.dataset.id, hda.id ) ) + trans.sa_session.add( hda.dataset ) + except: + log.exception( 'Unable to purge dataset (%s) on purge of hda (%s):' % ( hda.dataset.id, hda.id ) ) + count += 1 + return trans.show_ok_message( "%d datasets have been deleted permanently" % count, refresh_frames=['history'] ) + + @web.expose + def delete_current( self, trans, purge=False ): """Delete just the active history -- this does not require a logged in user.""" history = trans.get_history() if history.users_shared_with: @@ -432,25 +503,40 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati trans.sa_session.add( history ) trans.sa_session.flush() trans.log_event( "History id %d marked as deleted" % history.id ) - # Regardless of whether it was previously deleted, we make a new history active + if purge and trans.app.config.allow_user_dataset_purge: + for hda in history.datasets: + if trans.user: + trans.user.total_disk_usage -= hda.quota_amount( trans.user ) + hda.purged = True + trans.sa_session.add( hda ) + trans.log_event( "HDA id %s has been purged" % hda.id ) + trans.sa_session.flush() + if hda.dataset.user_can_purge: + try: + hda.dataset.full_delete() + trans.log_event( "Dataset id %s has been purged upon the the purge of HDA id %s" % ( hda.dataset.id, hda.id ) ) + trans.sa_session.add( hda.dataset ) + except: + log.exception( 'Unable to purge dataset (%s) on purge of hda (%s):' % ( hda.dataset.id, hda.id ) ) + history.purged = True + self.sa_session.add( history ) + self.sa_session.flush() + # Regardless of whether it was previously deleted, we make a new history active trans.new_history() - return trans.show_ok_message( "History deleted, a new history is active", refresh_frames=['history'] ) - + return trans.show_ok_message( "History deleted, a new history is active", refresh_frames=['history'] ) + @web.expose @web.require_login( "rate items" ) @web.json def rate_async( self, trans, id, rating ): """ Rate a history asynchronously and return updated community data. """ - history = self.get_history( trans, id, check_ownership=False, check_accessible=True ) if not history: return trans.show_error_message( "The specified history does not exist." ) - # Rate history. history_rating = self.rate_item( trans.sa_session, trans.get_user(), history, rating ) - return self.get_ave_item_rating_data( trans.sa_session, history ) - + @web.expose def rename_async( self, trans, id=None, new_name=None ): history = self.get_history( trans, id ) @@ -462,11 +548,11 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati else: assert history.user == trans.user # Rename - history.name = new_name + history.name = sanitize_html( new_name ) trans.sa_session.add( history ) trans.sa_session.flush() return history.name - + @web.expose @web.require_login( "use Galaxy histories" ) def annotate_async( self, trans, id, new_annotation=None, **kwargs ): @@ -479,12 +565,11 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati return new_annotation @web.expose - # TODO: Remove require_login when users are warned that, if they are not + # TODO: Remove require_login when users are warned that, if they are not # logged in, this will remove their current history. @web.require_login( "use Galaxy histories" ) def import_archive( self, trans, **kwargs ): """ Import a history from a file archive. """ - # Set archive source and type. archive_file = kwargs.get( 'archive_file', None ) archive_url = kwargs.get( 'archive_url', None ) @@ -495,37 +580,34 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati elif archive_url: archive_source = archive_url archive_type = 'url' - # If no source to create archive from, show form to upload archive or specify URL. if not archive_source: - return trans.show_form( + return trans.show_form( web.FormBuilder( web.url_for(), "Import a History from an Archive", submit_text="Submit" ) \ .add_input( "text", "Archived History URL", "archive_url", value="", error=None ) # TODO: add support for importing via a file. - #.add_input( "file", "Archived History File", "archive_file", value=None, error=None ) + #.add_input( "file", "Archived History File", "archive_file", value=None, error=None ) ) - # Run job to do import. history_imp_tool = trans.app.toolbox.tools_by_id[ '__IMPORT_HISTORY__' ] incoming = { '__ARCHIVE_SOURCE__' : archive_source, '__ARCHIVE_TYPE__' : archive_type } history_imp_tool.execute( trans, incoming=incoming ) return trans.show_message( "Importing history from '%s'. \ This history will be visible when the import is complete" % archive_source ) - - @web.expose + + @web.expose def export_archive( self, trans, id=None, gzip=True, include_hidden=False, include_deleted=False ): """ Export a history to an archive. """ - - # + # # Convert options to booleans. # if isinstance( gzip, basestring ): - gzip = ( gzip in [ 'True', 'true', 'T', 't' ] ) + gzip = ( gzip in [ 'True', 'true', 'T', 't' ] ) if isinstance( include_hidden, basestring ): include_hidden = ( include_hidden in [ 'True', 'true', 'T', 't' ] ) if isinstance( include_deleted, basestring ): - include_deleted = ( include_deleted in [ 'True', 'true', 'T', 't' ] ) - + include_deleted = ( include_deleted in [ 'True', 'true', 'T', 't' ] ) + # # Get history to export. # @@ -535,10 +617,10 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati # Use current history. history = trans.history id = trans.security.encode_id( history.id ) - + if not history: return trans.show_error_message( "This history does not exist or you cannot export this history." ) - + # # If history has already been exported and it has not changed since export, stream it. # @@ -561,40 +643,38 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati elif jeha.job.state in [ model.Job.states.RUNNING, model.Job.states.QUEUED, model.Job.states.WAITING ]: return trans.show_message( "Still exporting history %(n)s; please check back soon. Link: %(s)s" \ % ( { 'n' : history.name, 's' : url_for( action="export_archive", id=id, qualified=True ) } ) ) - + # Run job to do export. history_exp_tool = trans.app.toolbox.tools_by_id[ '__EXPORT_HISTORY__' ] - params = { - 'history_to_export' : history, - 'compress' : gzip, - 'include_hidden' : include_hidden, + params = { + 'history_to_export' : history, + 'compress' : gzip, + 'include_hidden' : include_hidden, 'include_deleted' : include_deleted } history_exp_tool.execute( trans, incoming = params, set_output_hid = True ) return trans.show_message( "Exporting History '%(n)s'. Use this link to download \ the archive or import it to another Galaxy server: \ %(u)s" \ % ( { 'n' : history.name, 'u' : url_for( action="export_archive", id=id, qualified=True ) } ) ) - + @web.expose @web.json @web.require_login( "get history name and link" ) def get_name_and_link_async( self, trans, id=None ): """ Returns history's name and link. """ history = self.get_history( trans, id, False ) - if self.create_item_slug( trans.sa_session, history ): trans.sa_session.flush() - return_dict = { - "name" : history.name, + return_dict = { + "name" : history.name, "link" : url_for( action="display_by_username_and_slug", username=history.user.username, slug=history.slug ) } return return_dict - + @web.expose @web.require_login( "set history's accessible flag" ) def set_accessible_async( self, trans, id=None, accessible=False ): """ Set history's importable attribute and slug. """ history = self.get_history( trans, id, True ) - # Only set if importable value would change; this prevents a change in the update_time unless attribute really changed. importable = accessible in ['True', 'true', 't', 'T']; if history and history.importable != importable: @@ -603,7 +683,6 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati else: history.importable = importable trans.sa_session.flush() - return @web.expose @@ -614,7 +693,7 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati history.slug = new_slug trans.sa_session.flush() return history.slug - + @web.expose def get_item_content_async( self, trans, id ): """ Returns item content in HTML format. """ @@ -622,7 +701,7 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati history = self.get_history( trans, id, False, True ) if history is None: raise web.httpexceptions.HTTPNotFound() - + # Get datasets. datasets = self.get_history_datasets( trans, history ) # Get annotations. @@ -630,7 +709,7 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati for dataset in datasets: dataset.annotation = self.get_item_annotation_str( trans.sa_session, history.user, dataset ) return trans.stream_template_mako( "/history/item_content.mako", item = history, item_data = datasets ) - + @web.expose def name_autocomplete_data( self, trans, q=None, limit=None, timestamp=None ): """Return autocomplete data for history names""" @@ -642,7 +721,7 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati for history in trans.sa_session.query( model.History ).filter_by( user=user ).filter( func.lower( model.History.name ) .like(q.lower() + "%") ): ac_data = ac_data + history.name + "\n" return ac_data - + @web.expose def imp( self, trans, id=None, confirm=False, **kwd ): """Import another user's history via a shared URL""" @@ -658,7 +737,7 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati referer_message = "return to the previous page" % referer else: referer_message = "go to Galaxy's start page" % url_for( '/' ) - + # Do import. if not id: return trans.show_error_message( "You must specify a history you want to import.
    You can %s." % referer_message, use_panels=True ) @@ -666,7 +745,7 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati if not import_history: return trans.show_error_message( "The specified history does not exist.
    You can %s." % referer_message, use_panels=True ) # History is importable if user is admin or it's accessible. TODO: probably want to have app setting to enable admin access to histories. - if not trans.user_is_admin() and not self.security_check( user, import_history, check_ownership=False, check_accessible=True ): + if not trans.user_is_admin() and not self.security_check( trans, import_history, check_ownership=False, check_accessible=True ): return trans.show_error_message( "You cannot access this history.
    You can %s." % referer_message, use_panels=True ) if user: #dan: I can import my own history. @@ -688,7 +767,7 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati # Set imported history to be user's current history. trans.set_history( new_history ) return trans.show_ok_message( - message="""History "%s" has been imported.
    You can start using this history or %s.""" + message="""History "%s" has been imported.
    You can start using this history or %s.""" % ( new_history.name, web.url_for( '/' ), referer_message ), use_panels=True ) elif not user_history or not user_history.datasets or confirm: new_history = import_history.copy() @@ -706,15 +785,15 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati trans.sa_session.flush() trans.set_history( new_history ) return trans.show_ok_message( - message="""History "%s" has been imported.
    You can start using this history or %s.""" + message="""History "%s" has been imported.
    You can start using this history or %s.""" % ( new_history.name, web.url_for( '/' ), referer_message ), use_panels=True ) return trans.show_warn_message( """ Warning! If you import this history, you will lose your current history.
    You can continue and import this history or %s. """ % ( web.url_for( id=id, confirm=True, referer=trans.request.referer ), referer_message ), use_panels=True ) - + @web.expose - def view( self, trans, id=None ): + def view( self, trans, id=None, show_deleted=False ): """View a history. If a history is importable, then it is viewable by any user.""" # Get history to view. if not id: @@ -727,16 +806,17 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati if not trans.user_is_admin() and not history_to_view.importable: error( "Either you are not allowed to view this history or the owner of this history has not made it accessible." ) # View history. - datasets = self.get_history_datasets( trans, history_to_view ) + show_deleted = util.string_as_bool( show_deleted ) + datasets = self.get_history_datasets( trans, history_to_view, show_deleted=show_deleted ) return trans.stream_template_mako( "history/view.mako", history = history_to_view, datasets = datasets, - show_deleted = False ) - + show_deleted = show_deleted ) + @web.expose def display_by_username_and_slug( self, trans, username, slug ): - """ Display history based on a username and slug. """ - + """ Display history based on a username and slug. """ + # Get history. session = trans.sa_session user = session.query( model.User ).filter_by( username=username ).first() @@ -744,15 +824,15 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati if history is None: raise web.httpexceptions.HTTPNotFound() # Security check raises error if user cannot access history. - self.security_check( trans.get_user(), history, False, True) - + self.security_check( trans, history, False, True) + # Get datasets. datasets = self.get_history_datasets( trans, history ) # Get annotations. history.annotation = self.get_item_annotation_str( trans.sa_session, history.user, history ) for dataset in datasets: dataset.annotation = self.get_item_annotation_str( trans.sa_session, history.user, dataset ) - + # Get rating data. user_item_rating = 0 if trans.get_user(): @@ -762,9 +842,9 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati else: user_item_rating = 0 ave_item_rating, num_ratings = self.get_ave_item_rating_data( trans.sa_session, history ) - return trans.stream_template_mako( "history/display.mako", item = history, item_data = datasets, + return trans.stream_template_mako( "history/display.mako", item = history, item_data = datasets, user_item_rating = user_item_rating, ave_item_rating=ave_item_rating, num_ratings=num_ratings ) - + @web.expose @web.require_login( "share Galaxy histories" ) def sharing( self, trans, id=None, histories=[], **kwargs ): @@ -779,7 +859,7 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati histories = [ self.get_history( trans, history_id ) for history_id in ids ] elif not histories: histories = [ trans.history ] - + # Do operation on histories. for history in histories: if 'make_accessible_via_link' in kwargs: @@ -812,17 +892,17 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati message = "History '%s' does not seem to be shared with user '%s'" % ( history.name, user.email ) return trans.fill_template( '/sharing_base.mako', item=history, message=message, status='error' ) - - + + # Legacy issue: histories made accessible before recent updates may not have a slug. Create slug for any histories that need them. for history in histories: if history.importable and not history.slug: self._make_item_accessible( trans.sa_session, history ) - + session.flush() - + return trans.fill_template( "/sharing_base.mako", item=history ) - + @web.expose @web.require_login( "share histories with other users" ) def share( self, trans, id=None, email="", **kwd ): @@ -865,11 +945,11 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati send_to_err = "The histories you are sharing do not contain any datasets that can be accessed by the users with which you are sharing." return trans.fill_template( "/history/share.mako", histories=histories, email=email, send_to_err=send_to_err ) if can_change or cannot_change: - return trans.fill_template( "/history/share.mako", - histories=histories, - email=email, - send_to_err=send_to_err, - can_change=can_change, + return trans.fill_template( "/history/share.mako", + histories=histories, + email=email, + send_to_err=send_to_err, + can_change=can_change, cannot_change=cannot_change, no_change_needed=unique_no_change_needed ) if no_change_needed: @@ -878,11 +958,11 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati # User seems to be sharing an empty history send_to_err = "You cannot share an empty history. " return trans.fill_template( "/history/share.mako", histories=histories, email=email, send_to_err=send_to_err ) - + @web.expose @web.require_login( "share restricted histories with other users" ) def share_restricted( self, trans, id=None, email="", **kwd ): - if 'action' in kwd: + if 'action' in kwd: action = kwd[ 'action' ] else: err_msg = "Select an action. " @@ -913,10 +993,10 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati # The action here is either 'public' or 'private', so we'll continue to populate the # histories_for_sharing dictionary from the can_change dictionary. for send_to_user, history_dict in can_change.items(): - for history in history_dict: + for history in history_dict: # Make sure the current history has not already been shared with the current send_to_user if trans.sa_session.query( trans.app.model.HistoryUserShareAssociation ) \ - .filter( and_( trans.app.model.HistoryUserShareAssociation.table.c.user_id == send_to_user.id, + .filter( and_( trans.app.model.HistoryUserShareAssociation.table.c.user_id == send_to_user.id, trans.app.model.HistoryUserShareAssociation.table.c.history_id == history.id ) ) \ .count() > 0: send_to_err += "History (%s) already shared with user (%s)" % ( history.name, send_to_user.email ) @@ -929,7 +1009,7 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati # The user with which we are sharing the history does not have access permission on the current dataset if trans.app.security_agent.can_manage_dataset( user_roles, hda.dataset ) and not hda.dataset.library_associations: # The current user has authority to change permissions on the current dataset because - # they have permission to manage permissions on the dataset and the dataset is not associated + # they have permission to manage permissions on the dataset and the dataset is not associated # with a library. if action == "private": trans.app.security_agent.privately_share_dataset( hda.dataset, users=[ user, send_to_user ] ) @@ -961,7 +1041,7 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati send_to_user = trans.sa_session.query( trans.app.model.User ) \ .filter( and_( trans.app.model.User.table.c.email==email_address, trans.app.model.User.table.c.deleted==False ) ) \ - .first() + .first() if send_to_user: send_to_users.append( send_to_user ) else: @@ -979,7 +1059,7 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati for history in history_dict: # Make sure the current history has not already been shared with the current send_to_user if trans.sa_session.query( trans.app.model.HistoryUserShareAssociation ) \ - .filter( and_( trans.app.model.HistoryUserShareAssociation.table.c.user_id == send_to_user.id, + .filter( and_( trans.app.model.HistoryUserShareAssociation.table.c.user_id == send_to_user.id, trans.app.model.HistoryUserShareAssociation.table.c.history_id == history.id ) ) \ .count() > 0: send_to_err += "History (%s) already shared with user (%s)" % ( history.name, send_to_user.email ) @@ -994,7 +1074,7 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati # The user may be attempting to share histories whose datasets cannot all be accessed by other users. # If this is the case, the user sharing the histories can: # 1) action=='public': choose to make the datasets public if he is permitted to do so - # 2) action=='private': automatically create a new "sharing role" allowing protected + # 2) action=='private': automatically create a new "sharing role" allowing protected # datasets to be accessed only by the desired users # This method will populate the can_change, cannot_change and no_change_needed dictionaries, which # are used for either displaying to the user, letting them make 1 of the choices above, or sharing @@ -1011,7 +1091,7 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati for send_to_user in send_to_users: # Make sure the current history has not already been shared with the current send_to_user if trans.sa_session.query( trans.app.model.HistoryUserShareAssociation ) \ - .filter( and_( trans.app.model.HistoryUserShareAssociation.table.c.user_id == send_to_user.id, + .filter( and_( trans.app.model.HistoryUserShareAssociation.table.c.user_id == send_to_user.id, trans.app.model.HistoryUserShareAssociation.table.c.history_id == history.id ) ) \ .count() > 0: send_to_err += "History (%s) already shared with user (%s)" % ( history.name, send_to_user.email ) @@ -1100,7 +1180,7 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati if send_to_err: msg += send_to_err return self.sharing( trans, histories=shared_histories, msg=msg ) - + @web.expose @web.require_login( "rename histories" ) def rename( self, trans, id=None, name=None, **kwd ): @@ -1129,7 +1209,7 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati change_msg = change_msg + "

    History: "+cur_names[i]+" is already named: "+name[i]+"

    " elif name[i] not in [None,'',' ']: name[i] = escape(name[i]) - histories[i].name = name[i] + histories[i].name = sanitize_html( name[i] ) trans.sa_session.add( histories[i] ) trans.sa_session.flush() change_msg = change_msg + "

    History: "+cur_names[i]+" renamed to: "+name[i]+"

    " @@ -1139,7 +1219,7 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati else: change_msg = change_msg + "

    History: "+cur_names[i]+" does not appear to belong to you.

    " return trans.show_message( "

    %s" % change_msg, refresh_frames=['history'] ) - + @web.expose @web.require_login( "clone shared Galaxy history" ) def clone( self, trans, id=None, **kwd ): @@ -1178,7 +1258,19 @@ class HistoryController( BaseController, Sharable, UsesAnnotations, UsesItemRati name += " (active items only)" new_history = history.copy( name=name, target_user=user ) if len( histories ) == 1: - msg = 'Clone with name "%s" is now included in your previously stored histories.' % new_history.name + msg = 'Clone with name "%s" is now included in your previously stored histories.' % ( url_for( controller="history", action="switch_to_history", hist_id=trans.security.encode_id( new_history.id ) ) , new_history.name ) else: msg = '%d cloned histories are now included in your previously stored histories.' % len( histories ) return trans.show_ok_message( msg ) + + @web.expose + @web.require_login( "switch to a history" ) + def switch_to_history( self, trans, hist_id=None ): + decoded_id = trans.security.decode_id(hist_id) + hist = trans.sa_session.query( trans.app.model.History ).get( decoded_id ) + trans.set_history( hist ) + return trans.response.send_redirect( url_for( "/" ) ) + + def get_item( self, trans, id ): + return self.get_history( trans, id ) + diff --git a/lib/galaxy/web/controllers/library.py b/lib/galaxy/web/controllers/library.py index 419fda59908..4f8fe70544b 100644 --- a/lib/galaxy/web/controllers/library.py +++ b/lib/galaxy/web/controllers/library.py @@ -65,7 +65,7 @@ class LibraryListGrid( grids.Grid ): # public libraries and restricted libraries accessible by the current user. return query.filter( or_( not_( trans.model.Library.table.c.id.in_( restricted_library_ids ) ), trans.model.Library.table.c.id.in_( accessible_restricted_library_ids ) ) ) -class Library( BaseController ): +class Library( BaseUIController ): library_list_grid = LibraryListGrid() diff --git a/lib/galaxy/web/controllers/library_admin.py b/lib/galaxy/web/controllers/library_admin.py index 00aa8890e32..2f05646b35f 100644 --- a/lib/galaxy/web/controllers/library_admin.py +++ b/lib/galaxy/web/controllers/library_admin.py @@ -69,7 +69,7 @@ class LibraryListGrid( grids.Grid ): preserve_state = False use_paging = True -class LibraryAdmin( BaseController ): +class LibraryAdmin( BaseUIController ): library_list_grid = LibraryListGrid() diff --git a/lib/galaxy/web/controllers/library_common.py b/lib/galaxy/web/controllers/library_common.py index 6b3d544a220..fcda23aa4e1 100644 --- a/lib/galaxy/web/controllers/library_common.py +++ b/lib/galaxy/web/controllers/library_common.py @@ -11,6 +11,7 @@ from galaxy.util import inflector from galaxy.web.form_builder import AddressField, CheckboxField, SelectField, TextArea, TextField, WorkflowField, WorkflowMappingField, HistoryField import logging, tempfile, zipfile, tarfile, os, sys, operator from galaxy.eggs import require +from galaxy.security import Action # Whoosh is compatible with Python 2.5+ Try to import Whoosh and set flag to indicate whether tool search is enabled. try: require( "Whoosh" ) @@ -67,7 +68,7 @@ except OSError: pass os.rmdir( tmpd ) -class LibraryCommon( BaseController, UsesFormDefinitions ): +class LibraryCommon( BaseUIController, UsesFormDefinitions ): @web.json def library_item_updates( self, trans, ids=None, states=None ): # Avoid caching @@ -231,10 +232,10 @@ class LibraryCommon( BaseController, UsesFormDefinitions ): for k, v in trans.app.model.Library.permitted_actions.items(): in_roles = [ trans.sa_session.query( trans.app.model.Role ).get( x ) for x in util.listify( params.get( k + '_in', [] ) ) ] permissions[ trans.app.security_agent.get_action( v.action ) ] = in_roles - trans.app.security_agent.set_all_library_permissions( library, permissions ) + trans.app.security_agent.set_all_library_permissions( trans, library, permissions ) trans.sa_session.refresh( library ) # Copy the permissions to the root folder - trans.app.security_agent.copy_library_permissions( library, library.root_folder ) + trans.app.security_agent.copy_library_permissions( trans, library, library.root_folder ) message = "Permissions updated for library '%s'." % library.name return trans.response.send_redirect( web.url_for( controller='library_common', action='library_permissions', @@ -245,12 +246,14 @@ class LibraryCommon( BaseController, UsesFormDefinitions ): message=util.sanitize_text( message ), status='done' ) ) roles = trans.app.security_agent.get_legitimate_roles( trans, library, cntrller ) + all_roles = trans.app.security_agent.get_all_roles( trans, cntrller ) return trans.fill_template( '/library/common/library_permissions.mako', cntrller=cntrller, use_panels=use_panels, library=library, current_user_roles=current_user_roles, roles=roles, + all_roles=all_roles, show_deleted=show_deleted, message=message, status=status ) @@ -284,7 +287,7 @@ class LibraryCommon( BaseController, UsesFormDefinitions ): trans.sa_session.add( new_folder ) trans.sa_session.flush() # New folders default to having the same permissions as their parent folder - trans.app.security_agent.copy_library_permissions( parent_folder, new_folder ) + trans.app.security_agent.copy_library_permissions( trans, parent_folder, new_folder ) # If we're creating in the API, we're done if cntrller == 'api': return 200, dict( created=new_folder ) @@ -410,7 +413,7 @@ class LibraryCommon( BaseController, UsesFormDefinitions ): # and it is not inherited. in_roles = [ trans.sa_session.query( trans.app.model.Role ).get( int( x ) ) for x in util.listify( params.get( k + '_in', [] ) ) ] permissions[ trans.app.security_agent.get_action( v.action ) ] = in_roles - trans.app.security_agent.set_all_library_permissions( folder, permissions ) + trans.app.security_agent.set_all_library_permissions( trans, folder, permissions ) trans.sa_session.refresh( folder ) message = "Permissions updated for folder '%s'." % folder.name return trans.response.send_redirect( web.url_for( controller='library_common', @@ -624,33 +627,50 @@ class LibraryCommon( BaseController, UsesFormDefinitions ): else: roles = trans.app.security_agent.get_legitimate_roles( trans, ldda.dataset, cntrller ) if params.get( 'update_roles_button', False ): - a = trans.app.security_agent.get_action( trans.app.security_agent.permitted_actions.DATASET_ACCESS.action ) + # Dataset permissions + access_action = trans.app.security_agent.get_action( trans.app.security_agent.permitted_actions.DATASET_ACCESS.action ) + manage_permissions_action = trans.app.security_agent.get_action( trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS.action ) permissions, in_roles, error, message = \ trans.app.security_agent.derive_roles_from_access( trans, trans.app.security.decode_id( library_id ), cntrller, library=True, **kwd ) + # Keep roles for DATASET_MANAGE_PERMISSIONS on the dataset + if not ldda.has_manage_permissions_roles( trans ): + # Permission setting related to DATASET_MANAGE_PERMISSIONS was broken for a period of time, + # so it is possible that some Datasets have no roles associated with the DATASET_MANAGE_PERMISSIONS + # permission. In this case, we'll reset this permission to the ldda user's private role. + #dataset_manage_permissions_roles = [ trans.app.security_agent.get_private_user_role( ldda.user ) ] + permissions[ manage_permissions_action ] = [ trans.app.security_agent.get_private_user_role( ldda.user ) ] + else: + permissions[ manage_permissions_action ] = ldda.get_manage_permissions_roles( trans ) for ldda in lddas: # Set the DATASET permissions on the Dataset. if error: # Keep the original role associations for the DATASET_ACCESS permission on the ldda. - permissions[ a ] = ldda.get_access_roles( trans ) - trans.app.security_agent.set_all_dataset_permissions( ldda.dataset, permissions ) - trans.sa_session.refresh( ldda.dataset ) - # Set the LIBRARY permissions on the LibraryDataset. The LibraryDataset and - # LibraryDatasetDatasetAssociation will be set with the same permissions. - permissions = {} - for k, v in trans.app.model.Library.permitted_actions.items(): - if k != 'LIBRARY_ACCESS': - # LIBRARY_ACCESS is a special permission set only at the library level and it is not inherited. - in_roles = [ trans.sa_session.query( trans.app.model.Role ).get( x ) for x in util.listify( kwd.get( k + '_in', [] ) ) ] - permissions[ trans.app.security_agent.get_action( v.action ) ] = in_roles - for ldda in lddas: - trans.app.security_agent.set_all_library_permissions( ldda.library_dataset, permissions ) - trans.sa_session.refresh( ldda.library_dataset ) - # Set the LIBRARY permissions on the LibraryDatasetDatasetAssociation - trans.app.security_agent.set_all_library_permissions( ldda, permissions ) - trans.sa_session.refresh( ldda ) - if error: - status = 'error' - else: + permissions[ access_action ] = ldda.get_access_roles( trans ) + status = 'error' + else: + error = trans.app.security_agent.set_all_dataset_permissions( ldda.dataset, permissions ) + if error: + message += error + status = 'error' + trans.sa_session.refresh( ldda.dataset ) + if not error: + # Set the LIBRARY permissions on the LibraryDataset. The LibraryDataset and + # LibraryDatasetDatasetAssociation will be set with the same permissions. + permissions = {} + for k, v in trans.app.model.Library.permitted_actions.items(): + if k != 'LIBRARY_ACCESS': + # LIBRARY_ACCESS is a special permission set only at the library level and it is not inherited. + in_roles = [ trans.sa_session.query( trans.app.model.Role ).get( x ) for x in util.listify( kwd.get( k + '_in', [] ) ) ] + permissions[ trans.app.security_agent.get_action( v.action ) ] = in_roles + for ldda in lddas: + error = trans.app.security_agent.set_all_library_permissions( trans, ldda.library_dataset, permissions ) + trans.sa_session.refresh( ldda.library_dataset ) + if error: + message = error + else: + # Set the LIBRARY permissions on the LibraryDatasetDatasetAssociation + trans.app.security_agent.set_all_library_permissions( trans, ldda, permissions ) + trans.sa_session.refresh( ldda ) if len( lddas ) == 1: message = "Permissions updated for dataset '%s'." % ldda.name else: @@ -829,9 +849,11 @@ class LibraryCommon( BaseController, UsesFormDefinitions ): **kwd ) if created_outputs_dict: if cntrller == 'api': - # created_outputs_dict can only ever be a string if cntrller == 'api' + # created_outputs_dict can be a string only if cntrller == 'api' if type( created_outputs_dict ) == str: return 400, created_outputs_dict + elif type( created_outputs_dict ) == tuple: + return created_outputs_dict[0], created_outputs_dict[1] return 200, created_outputs_dict total_added = len( created_outputs_dict.keys() ) ldda_id_list = [ str( v.id ) for k, v in created_outputs_dict.items() ] @@ -904,7 +926,8 @@ class LibraryCommon( BaseController, UsesFormDefinitions ): dbkeys = get_dbkey_options( last_used_build ) # Send the current history to the form to enable importing datasets from history to library history = trans.get_history() - trans.sa_session.refresh( history ) + if history is not None: + trans.sa_session.refresh( history ) if upload_option == 'upload_file' and trans.app.config.nginx_upload_path: # If we're using nginx upload, override the form action - # url_for is intentionally not used on the base URL here - @@ -1209,10 +1232,31 @@ class LibraryCommon( BaseController, UsesFormDefinitions ): if not replace_dataset: # If replace_dataset is None, the Library level permissions will be taken from the folder and applied to the new # LDDA and LibraryDataset. - trans.app.security_agent.copy_library_permissions( folder, ldda ) - trans.app.security_agent.copy_library_permissions( folder, ldda.library_dataset ) + trans.app.security_agent.copy_library_permissions( trans, folder, ldda ) + trans.app.security_agent.copy_library_permissions( trans, folder, ldda.library_dataset ) + # Make sure to apply any defined dataset permissions, allowing the permissions inherited from the folder to + # over-ride the same permissions on the dataset, if they exist. + dataset_permissions_dict = trans.app.security_agent.get_permissions( hda.dataset ) + current_library_dataset_actions = [ permission.action for permission in ldda.library_dataset.actions ] + # The DATASET_MANAGE_PERMISSIONS permission on a dataset is a special case because if + # it exists, then we need to apply the LIBRARY_MANAGE permission to the library dataset. + dataset_manage_permissions_action = trans.app.security_agent.get_action( 'DATASET_MANAGE_PERMISSIONS' ).action + flush_needed = False + for action, dataset_permissions_roles in dataset_permissions_dict.items(): + if isinstance( action, Action ): + action = action.action + if action == dataset_manage_permissions_action: + # Apply the LIBRARY_MANAGE permission to the library dataset. + action = trans.app.security_agent.get_action( 'LIBRARY_MANAGE' ).action + # Allow the permissions inherited from the folder to over-ride the same permissions on the dataset. + if action not in current_library_dataset_actions: + for ldp in [ trans.model.LibraryDatasetPermissions( action, ldda.library_dataset, role ) for role in dataset_permissions_roles ]: + trans.sa_session.add( ldp ) + flush_needed = True + if flush_needed: + trans.sa_session.flush() # Permissions must be the same on the LibraryDatasetDatasetAssociation and the associated LibraryDataset - trans.app.security_agent.copy_library_permissions( ldda.library_dataset, ldda ) + trans.app.security_agent.copy_library_permissions( trans, ldda.library_dataset, ldda ) if created_ldda_ids: created_ldda_ids = created_ldda_ids.lstrip( ',' ) ldda_id_list = created_ldda_ids.split( ',' ) @@ -1360,8 +1404,7 @@ class LibraryCommon( BaseController, UsesFormDefinitions ): kwd['do_action'] = 'zip' return self.act_on_multiple_datasets( trans, cntrller, library_id, ldda_ids=[id,], **kwd ) else: - mime = trans.app.datatypes_registry.get_mimetype_by_extension( ldda.extension.lower() ) - trans.response.set_content_type( mime ) + trans.response.set_content_type( ldda.get_mime() ) fStat = os.stat( ldda.file_name ) trans.response.headers[ 'Content-Length' ] = int( fStat.st_size ) valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' @@ -1455,13 +1498,17 @@ class LibraryCommon( BaseController, UsesFormDefinitions ): permissions[ trans.app.security_agent.get_action( v.action ) ] = in_roles # Set the LIBRARY permissions on the LibraryDataset # NOTE: the LibraryDataset and LibraryDatasetDatasetAssociation will be set with the same permissions - trans.app.security_agent.set_all_library_permissions( library_dataset, permissions ) + error = trans.app.security_agent.set_all_library_permissions( trans, library_dataset, permissions ) trans.sa_session.refresh( library_dataset ) - # Set the LIBRARY permissions on the LibraryDatasetDatasetAssociation - trans.app.security_agent.set_all_library_permissions( library_dataset.library_dataset_dataset_association, permissions ) - trans.sa_session.refresh( library_dataset.library_dataset_dataset_association ) - message = "Permisisons updated for library dataset '%s'." % library_dataset.name - status = 'done' + if error: + message = error + status = 'error' + else: + # Set the LIBRARY permissions on the LibraryDatasetDatasetAssociation + trans.app.security_agent.set_all_library_permissions( trans, library_dataset.library_dataset_dataset_association, permissions ) + trans.sa_session.refresh( library_dataset.library_dataset_dataset_association ) + message = "Permisisons updated for library dataset '%s'." % library_dataset.name + status = 'done' roles = trans.app.security_agent.get_legitimate_roles( trans, library_dataset, cntrller ) return trans.fill_template( '/library/common/library_dataset_permissions.mako', cntrller=cntrller, @@ -1605,7 +1652,7 @@ class LibraryCommon( BaseController, UsesFormDefinitions ): valid_lddas = [] invalid_lddas = [] for ldda in lddas: - if trans.app.security_agent.can_manage_library_item( current_user_roles, ldda ): + if is_admin or trans.app.security_agent.can_manage_library_item( current_user_roles, ldda ): valid_lddas.append( ldda ) valid_ldda_ids.append( ldda.id ) else: @@ -1634,7 +1681,7 @@ class LibraryCommon( BaseController, UsesFormDefinitions ): valid_lddas = [] invalid_lddas = [] for ldda in lddas: - if trans.app.security_agent.can_modify_library_item( current_user_roles, ldda ): + if is_admin or trans.app.security_agent.can_modify_library_item( current_user_roles, ldda ): valid_lddas.append( ldda ) else: invalid_lddas.append( ldda ) @@ -1850,14 +1897,16 @@ class LibraryCommon( BaseController, UsesFormDefinitions ): new_history.user = user trans.sa_session.add( new_history ) trans.sa_session.flush() - target_history_ids.append( new_history.id ) - if user: + target_history_ids = [ new_history.id ] + target_histories = [ new_history ] + elif user: target_histories = [ hist for hist in map( trans.sa_session.query( trans.app.model.History ).get, target_history_ids ) if ( hist is not None and hist.user == user )] else: target_histories = [ current_history ] if len( target_histories ) != len( target_history_ids ): message += "You do not have permission to add datasets to %i requested histories. " % ( len( target_history_ids ) - len( target_histories ) ) status = 'error' + flush_needed = False for ldda in map( trans.sa_session.query( trans.app.model.LibraryDatasetDatasetAssociation ).get, ldda_ids ): if ldda is None: message += "You tried to import a dataset that does not exist. " @@ -1874,15 +1923,18 @@ class LibraryCommon( BaseController, UsesFormDefinitions ): else: for target_history in target_histories: hda = ldda.to_history_dataset_association( target_history=target_history, add_to_history=True ) - trans.sa_session.flush() - hist_names_str = ", ".join( [ target_history.name for target_history in target_histories ] ) - num_source = len( ldda_ids ) - invalid_datasets - num_target = len( target_histories ) - message = "%i %s imported into %i %s: %s" % ( num_source, - inflector.cond_plural( num_source, "dataset" ), - num_target, - inflector.cond_plural( num_target, "history" ), - hist_names_str ) + if not flush_needed: + flush_needed = True + if flush_needed: + trans.sa_session.flush() + hist_names_str = ", ".join( [ target_history.name for target_history in target_histories ] ) + num_source = len( ldda_ids ) - invalid_datasets + num_target = len( target_histories ) + message += "%i %s imported into %i %s: %s" % ( num_source, + inflector.cond_plural( num_source, "dataset" ), + num_target, + inflector.cond_plural( num_target, "history" ), + hist_names_str ) trans.sa_session.refresh( current_history ) current_user_roles = trans.get_current_user_roles() source_lddas = [] diff --git a/lib/galaxy/web/controllers/mobile.py b/lib/galaxy/web/controllers/mobile.py index 218e77d0bbf..e8a72194176 100644 --- a/lib/galaxy/web/controllers/mobile.py +++ b/lib/galaxy/web/controllers/mobile.py @@ -1,6 +1,6 @@ from galaxy.web.base.controller import * -class Mobile( BaseController ): +class Mobile( BaseUIController ): @web.expose def index( self, trans, **kwargs ): return trans.fill_template( "mobile/index.mako" ) @@ -47,7 +47,7 @@ class Mobile( BaseController ): error = password_error = None user = trans.sa_session.query( model.User ).filter_by( email = email ).first() if not user: - error = "No such user" + error = "No such user (please note that login is case sensitive)" elif user.deleted: error = "This account has been marked deleted, contact your Galaxy administrator to restore the account." elif user.external: diff --git a/lib/galaxy/web/controllers/page.py b/lib/galaxy/web/controllers/page.py index 7ab59bb4c23..a0eea040e6d 100644 --- a/lib/galaxy/web/controllers/page.py +++ b/lib/galaxy/web/controllers/page.py @@ -272,7 +272,7 @@ class _PageContentProcessor( _BaseHTMLProcessor ): # Default behavior: _BaseHTMLProcessor.unknown_endtag( self, tag ) -class PageController( BaseController, Sharable, UsesAnnotations, UsesHistory, +class PageController( BaseUIController, Sharable, UsesAnnotations, UsesHistory, UsesStoredWorkflow, UsesHistoryDatasetAssociation, UsesVisualization, UsesItemRatings ): _page_list = PageListGrid() @@ -533,7 +533,7 @@ class PageController( BaseController, Sharable, UsesAnnotations, UsesHistory, annotations = from_json_string( annotations ) for annotation_dict in annotations: item_id = trans.security.decode_id( annotation_dict[ 'item_id' ] ) - item_class = self.get_class( trans, annotation_dict[ 'item_class' ] ) + item_class = self.get_class( annotation_dict[ 'item_class' ] ) item = trans.sa_session.query( item_class ).filter_by( id=item_id ).first() if not item: raise RuntimeError( "cannot find annotated item" ) @@ -582,7 +582,7 @@ class PageController( BaseController, Sharable, UsesAnnotations, UsesHistory, if page is None: raise web.httpexceptions.HTTPNotFound() # Security check raises error if user cannot access page. - self.security_check( trans.get_user(), page, False, True) + self.security_check( trans, page, False, True) # Process page content. processor = _PageContentProcessor( trans, 'utf-8', 'text/html', self._get_embed_html ) @@ -716,18 +716,21 @@ class PageController( BaseController, Sharable, UsesAnnotations, UsesHistory, return trans.fill_template( "page/wymiframe.mako" ) def get_page( self, trans, id, check_ownership=True, check_accessible=False ): - """Get a page from the database by id, verifying ownership.""" + """Get a page from the database by id.""" # Load history from database id = trans.security.decode_id( id ) page = trans.sa_session.query( model.Page ).get( id ) if not page: err+msg( "Page not found" ) else: - return self.security_check( trans.get_user(), page, check_ownership, check_accessible ) + return self.security_check( trans, page, check_ownership, check_accessible ) + + def get_item( self, trans, id ): + return self.get_page( trans, id ) def _get_embed_html( self, trans, item_class, item_id ): """ Returns HTML for embedding an item in a page. """ - item_class = self.get_class( trans, item_class ) + item_class = self.get_class( item_class ) if item_class == model.History: history = self.get_history( trans, item_id, False, True ) history.annotation = self.get_item_annotation_str( trans.sa_session, history.user, history ) diff --git a/lib/galaxy/web/controllers/request_type.py b/lib/galaxy/web/controllers/request_type.py index 38633d64702..d327ca8771d 100644 --- a/lib/galaxy/web/controllers/request_type.py +++ b/lib/galaxy/web/controllers/request_type.py @@ -72,7 +72,7 @@ class RequestTypeGrid( grids.Grid ): grids.GridAction( "Create new request type", dict( controller='request_type', action='create_request_type' ) ) ] -class RequestType( BaseController, UsesFormDefinitions ): +class RequestType( BaseUIController, UsesFormDefinitions ): request_type_grid = RequestTypeGrid() @web.expose diff --git a/lib/galaxy/web/controllers/requests.py b/lib/galaxy/web/controllers/requests.py index f1d1cd7045d..dfaf580ab2c 100644 --- a/lib/galaxy/web/controllers/requests.py +++ b/lib/galaxy/web/controllers/requests.py @@ -1,7 +1,6 @@ from galaxy.web.base.controller import * from galaxy.web.framework.helpers import grids from galaxy.model.orm import * -from galaxy import model, util from galaxy.web.form_builder import * from galaxy.web.controllers.requests_common import RequestsGrid import logging @@ -16,7 +15,7 @@ class UserRequestsGrid( RequestsGrid ): def apply_query_filter( self, trans, query, **kwd ): return query.filter_by( user=trans.user ) -class Requests( BaseController ): +class Requests( BaseUIController ): request_grid = UserRequestsGrid() @web.expose @@ -89,4 +88,4 @@ class Requests( BaseController ): self.request_grid.global_actions = [] # Render the list view return self.request_grid( trans, **kwd ) - \ No newline at end of file + diff --git a/lib/galaxy/web/controllers/requests_admin.py b/lib/galaxy/web/controllers/requests_admin.py index 03e59b571c5..9903a314f68 100644 --- a/lib/galaxy/web/controllers/requests_admin.py +++ b/lib/galaxy/web/controllers/requests_admin.py @@ -94,7 +94,7 @@ class DataTransferGrid( grids.Grid ): return query return query.filter_by( sample_id=trans.security.decode_id( sample_id ) ) -class RequestsAdmin( BaseController, UsesFormDefinitions ): +class RequestsAdmin( BaseUIController, UsesFormDefinitions ): request_grid = AdminRequestsGrid() datatx_grid = DataTransferGrid() diff --git a/lib/galaxy/web/controllers/requests_common.py b/lib/galaxy/web/controllers/requests_common.py index c2d45d9ec45..5e989154baf 100644 --- a/lib/galaxy/web/controllers/requests_common.py +++ b/lib/galaxy/web/controllers/requests_common.py @@ -4,6 +4,7 @@ from galaxy.model.orm import * from galaxy import model, util from galaxy.util.odict import odict from galaxy.web.form_builder import * +from galaxy.security.validate_user_input import validate_email import logging, os, csv log = logging.getLogger( __name__ ) @@ -92,7 +93,7 @@ class RequestsGrid( grids.Grid ): confirm="Samples cannot be added to this request after it is submitted. Click OK to submit." ) ] -class RequestsCommon( BaseController, UsesFormDefinitions ): +class RequestsCommon( BaseUIController, UsesFormDefinitions ): @web.json def sample_state_updates( self, trans, ids=None, states=None ): # Avoid caching @@ -643,7 +644,7 @@ class RequestsCommon( BaseController, UsesFormDefinitions ): # Make sure email addresses are valid err_msg = '' for email_address in email_addresses: - err_msg += self.__validate_email( email_address ) + err_msg += validate_email( trans, email_address, check_dup=False ) if err_msg: status = 'error' message += err_msg @@ -826,7 +827,9 @@ class RequestsCommon( BaseController, UsesFormDefinitions ): displayable_sample_widgets = self.__get_sample_widgets( trans, request, request.samples, **kwd ) if params.get( 'import_samples_button', False ): # Import sample field values from a csv file - return self.__import_samples( trans, cntrller, request, displayable_sample_widgets, libraries, **kwd ) + # TODO: should this be a mapper? + workflows = [ w.latest_workflow for w in trans.user.stored_workflows if not w.deleted ] + return self.__import_samples( trans, cntrller, request, displayable_sample_widgets, libraries, workflows, **kwd ) elif params.get( 'add_sample_button', False ): return self.add_sample( trans, cntrller, request_id, **kwd ) elif params.get( 'save_samples_button', False ): @@ -905,17 +908,17 @@ class RequestsCommon( BaseController, UsesFormDefinitions ): folder_id=folder_id, **kwd ) history_select_field = self.__build_history_select_field( trans=trans, - user=request.user, - sample_index=len( displayable_sample_widgets ), - history_id=history_id, - **kwd) + user=request.user, + sample_index=len( displayable_sample_widgets ), + history_id=history_id, + **kwd ) workflow_select_field = self.__build_workflow_select_field( trans=trans, - user=request.user, - request=request, - sample_index=len( displayable_sample_widgets ), - workflow_id=workflow_id, - history_id=history_id, - **kwd) + user=request.user, + request=request, + sample_index=len( displayable_sample_widgets ), + workflow_id=workflow_id, + history_id=history_id, + **kwd ) # Append the new sample to the current list of samples for the request displayable_sample_widgets.append( dict( id=None, name=name, @@ -1045,47 +1048,106 @@ class RequestsCommon( BaseController, UsesFormDefinitions ): transfer_status=transfer_status, message=message, status=status ) - def __import_samples( self, trans, cntrller, request, displayable_sample_widgets, libraries, **kwd ): + def __import_samples( self, trans, cntrller, request, displayable_sample_widgets, libraries, workflows, **kwd ): """ - Reads the samples csv file and imports all the samples. The format of the csv file is: - SampleName,DataLibrary,DataLibraryFolder,Field1,Field2.... + Reads the samples csv file and imports all the samples. The csv file must be in the following format. The [:FieldValue] + is optional, the form field will contain the value after the ':' if included. + SampleName,DataLibraryName,DataLibraryFolderName,HistoryName,WorkflowName,Field1Name:Field1Value,Field2Name:Field2Value... """ params = util.Params( kwd ) + current_user_roles = trans.get_current_user_roles() + is_admin = trans.user_is_admin() and cntrller == 'requests_admin' file_obj = params.get( 'file_data', '' ) try: reader = csv.reader( file_obj.file ) for row in reader: library_id = None + library = None folder_id = None - # FIXME: this is bad - what happens when multiple libraries have the same name?? - lib = trans.sa_session.query( trans.model.Library ) \ - .filter( and_( trans.model.Library.table.c.name==row[1], - trans.model.Library.table.c.deleted==False ) ) \ - .first() - if lib: - folder = trans.sa_session.query( trans.model.LibraryFolder ) \ - .filter( and_( trans.model.LibraryFolder.table.c.name==row[2], - trans.model.LibraryFolder.table.c.deleted==False ) ) \ - .first() + folder = None + history_id = None + history = None + workflow_id = None + workflow = None + # Get the library + library = trans.sa_session.query( trans.model.Library ) \ + .filter( and_( trans.model.Library.table.c.name==row[1], + trans.model.Library.table.c.deleted==False ) ) \ + .first() + if library: + # Get the folder + for folder in trans.sa_session.query( trans.model.LibraryFolder ) \ + .filter( and_( trans.model.LibraryFolder.table.c.name==row[2], + trans.model.LibraryFolder.table.c.deleted==False ) ): + if folder.parent_library == library: + break if folder: - library_id = lib.id - folder_id = folder.id + library_id = trans.security.encode_id( library.id ) + folder_id = trans.security.encode_id( folder.id ) library_select_field, folder_select_field = self.__build_library_and_folder_select_fields( trans, request.user, - len( displayable_sample_widgets ), + len( displayable_sample_widgets ), libraries, None, library_id, folder_id, **kwd ) + # Get the history + history = trans.sa_session.query( trans.model.History ) \ + .filter( and_( trans.model.History.table.c.name==row[3], + trans.model.History.table.c.deleted==False, + trans.model.History.user_id == trans.user.id ) ) \ + .first() + if history: + history_id = trans.security.encode_id( history.id ) + else: + history_id = 'none' + history_select_field = self.__build_history_select_field( trans=trans, + user=request.user, + sample_index=len( displayable_sample_widgets ), + history_id=history_id ) + # Get the workflow + workflow = trans.sa_session.query( trans.model.StoredWorkflow ) \ + .filter( and_( trans.model.StoredWorkflow.table.c.name==row[4], + trans.model.StoredWorkflow.table.c.deleted==False, + trans.model.StoredWorkflow.user_id == trans.user.id ) ) \ + .first() + if workflow: + workflow_id = trans.security.encode_id( workflow.id ) + else: + workflow_id = 'none' + workflow_select_field = self.__build_workflow_select_field( trans=trans, + user=request.user, + request=request, + sample_index=len( displayable_sample_widgets ), + workflow_id=workflow_id, + history_id=history_id ) + field_values = {} + field_names = row[5:] + for field_name in field_names: + if field_name.find( ':' ) >= 0: + field_list = field_name.split( ':' ) + field_name = field_list[0] + field_value = field_list[1] + else: + field_value = '' + field_values[ field_name ] = field_value displayable_sample_widgets.append( dict( id=None, - name=row[0], + name=row[0], bar_code='', - library=None, - folder=None, + library=library, + library_id=library_id, library_select_field=library_select_field, + folder=folder, + folder_id=folder_id, folder_select_field=folder_select_field, - field_values=row[3:] ) ) + history=history, + history_id=history_id, + history_select_field=history_select_field, + workflow=workflow, + workflow_id=workflow_id, + workflow_select_field=workflow_select_field, + field_values=field_values ) ) except Exception, e: if str( e ) == "'unicode' object has no attribute 'file'": message = "Select a file" @@ -1124,6 +1186,8 @@ class RequestsCommon( BaseController, UsesFormDefinitions ): redirect_action = 'edit_samples' # Check for duplicate sample names within the request self.__validate_sample_names( trans, cntrller, request, sample_widgets, **kwd ) + print "SAVING SAMPLES!" + print "saving_new_samples is %s" % saving_new_samples if not saving_new_samples: library = None folder = None @@ -1170,7 +1234,7 @@ class RequestsCommon( BaseController, UsesFormDefinitions ): handle_error( **kwd ) self.update_sample_state( trans, cntrller, encoded_selected_sample_ids, new_state, comment=sample_event_comment ) return trans.response.send_redirect( web.url_for( controller='requests_common', - cntrller=cntrller, + cntrller=cntrller, action='update_request_state', request_id=trans.security.encode_id( request.id ) ) ) elif sample_operation == trans.model.Sample.bulk_operations.SELECT_LIBRARY: @@ -1180,7 +1244,7 @@ class RequestsCommon( BaseController, UsesFormDefinitions ): for sample_index in range( len( sample_widgets ) ): current_sample = sample_widgets[ sample_index ] if current_sample is None: - # We have a None value because the user did not select this sample + # We have a None value because the user did not select this sample # on which to perform the action. continue current_sample[ 'library' ] = library @@ -1189,7 +1253,7 @@ class RequestsCommon( BaseController, UsesFormDefinitions ): message = 'Changes made to the samples have been saved. ' else: # Saving a newly created sample. The sample will not have an associated SampleState - # until the request is submitted, at which time all samples of the request will be + # until the request is submitted, at which time all samples of the request will be # set to the first SampleState configured for the request's RequestType configured # by the admin ( i.e., the sample's SampleState would be set to request.type.states[0] ). new_samples = [] @@ -1204,9 +1268,9 @@ class RequestsCommon( BaseController, UsesFormDefinitions ): else: bar_code = '' sample = trans.model.Sample( name=sample_widget[ 'name' ], - desc='', + desc='', request=request, - form_values=form_values, + form_values=form_values, bar_code=bar_code, library=sample_widget[ 'library' ], folder=sample_widget[ 'folder' ], @@ -1486,7 +1550,7 @@ class RequestsCommon( BaseController, UsesFormDefinitions ): library_select_field=library_select_field, folder_select_field=folder_select_field, history_select_field=history_select_field, - workflow_select_field=workflow_select_field, ) ) + workflow_select_field=workflow_select_field ) ) # There may be additional new samples on the form that have not yet been associated with the request. # TODO: factor this code so it is not duplicating what's above. index = len( samples ) @@ -1749,7 +1813,7 @@ class RequestsCommon( BaseController, UsesFormDefinitions ): select_field.add_option( label, value ) wf_fieldset.append((step.tool_inputs['name'], select_field)) return wf_fieldset - + def __build_sample_state_id_select_field( self, trans, request, selected_value ): if selected_value == 'none': if request.samples: @@ -1771,10 +1835,21 @@ class RequestsCommon( BaseController, UsesFormDefinitions ): for index, field in enumerate( request.type.request_form.fields ): if field[ 'required' ] == 'required' and request.values.content[ field[ 'name' ] ] in [ '', None ]: empty_fields.append( field[ 'label' ] ) - if empty_fields: - message = 'Complete the following fields of the request before submitting: ' - for ef in empty_fields: - message += '' + ef + ' ' + empty_sample_fields = [] + for s in request.samples: + for field in request.type.sample_form.fields: + print "field:", field + print "svc:", s.values.content + if field['required'] == 'required' and s.values.content[field['name']] in ['', None]: + empty_sample_fields.append((s.name, field['label'])) + if empty_fields or empty_sample_fields: + message = 'Complete the following fields of the request before submitting:
    ' + if empty_fields: + for ef in empty_fields: + message += '%s
    ' % ef + if empty_sample_fields: + for sname, ef in empty_sample_fields: + message = message + '%s field of sample %s
    ' % (ef, sname) return message return None def __validate_sample_names( self, trans, cntrller, request, displayable_sample_widgets, **kwd ): @@ -1791,7 +1866,7 @@ class RequestsCommon( BaseController, UsesFormDefinitions ): for i in range( len( displayable_sample_widgets ) ): if sample_name == displayable_sample_widgets[ i ][ 'name' ]: count += 1 - if count > 1: + if count > 1: message = "You tried to add %i samples with the name (%s). Samples belonging to a request must have unique names." % ( count, sample_name ) break if message: @@ -1825,13 +1900,6 @@ class RequestsCommon( BaseController, UsesFormDefinitions ): if not unique: break return message - def __validate_email( self, email ): - error = '' - if len( email ) == 0 or "@" not in email or "." not in email: - error = "(%s) is not a valid email address. " % str( email ) - elif len( email ) > 255: - error = "(%s) exceeds maximum allowable length. " % str( email ) - return error # ===== Other miscellaneous utility methods ===== def __get_encoded_selected_sample_ids( self, trans, request, **kwd ): encoded_selected_sample_ids = [] diff --git a/lib/galaxy/web/controllers/root.py b/lib/galaxy/web/controllers/root.py index 4fa3e90bb70..da9873b8722 100644 --- a/lib/galaxy/web/controllers/root.py +++ b/lib/galaxy/web/controllers/root.py @@ -11,7 +11,7 @@ from galaxy.model.item_attrs import UsesAnnotations log = logging.getLogger( __name__ ) -class RootController( BaseController, UsesHistory, UsesAnnotations ): +class RootController( BaseUIController, UsesHistory, UsesAnnotations ): @web.expose def default(self, trans, target1=None, target2=None, **kwd): @@ -98,11 +98,14 @@ class RootController( BaseController, UsesHistory, UsesAnnotations ): return trans.fill_template_mako( "/my_data.mako" ) @web.expose - def history( self, trans, as_xml=False, show_deleted=False, show_hidden=False, hda_id=None ): + def history( self, trans, as_xml=False, show_deleted=False, show_hidden=False, hda_id=None, **kwd ): """ Display the current history, creating a new history if necessary. NOTE: No longer accepts "id" or "template" options for security reasons. """ + params = util.Params( kwd ) + message = params.get( 'message', None ) + status = params.get( 'status', 'done' ) if trans.app.config.require_login and not trans.user: return trans.fill_template( '/no_access.mako', message = 'Please log in to access Galaxy histories.' ) history = trans.get_history( create=True ) @@ -113,16 +116,19 @@ class RootController( BaseController, UsesHistory, UsesAnnotations ): show_deleted=util.string_as_bool( show_deleted ), show_hidden=util.string_as_bool( show_hidden ) ) else: - show_deleted = util.string_as_bool( show_deleted ) + show_deleted = show_purged = util.string_as_bool( show_deleted ) show_hidden = util.string_as_bool( show_hidden ) - datasets = self.get_history_datasets( trans, history, show_deleted, show_hidden ) + datasets = self.get_history_datasets( trans, history, show_deleted, show_hidden, show_purged ) return trans.stream_template_mako( "root/history.mako", history = history, annotation = self.get_item_annotation_str( trans.sa_session, trans.user, history ), datasets = datasets, hda_id = hda_id, show_deleted = show_deleted, - show_hidden=show_hidden ) + show_hidden=show_hidden, + over_quota=trans.app.quota_agent.get_percent( trans=trans ) >= 100, + message=message, + status=status ) @web.expose def dataset_state ( self, trans, id=None, stamp=None ): @@ -159,9 +165,13 @@ class RootController( BaseController, UsesHistory, UsesAnnotations ): # Create new HTML for any that have changed rval = {} if ids is not None and states is not None: - ids = map( int, ids.split( "," ) ) + ids = ids.split( "," ) states = states.split( "," ) - for id, state in zip( ids, states ): + for encoded_id, state in zip( ids, states ): + try: + id = int( trans.app.security.decode_id( encoded_id ) ) + except: + id = int( encoded_id ) data = trans.sa_session.query( self.app.model.HistoryDatasetAssociation ).get( id ) if data.state != state: job_hda = data @@ -174,13 +184,35 @@ class RootController( BaseController, UsesHistory, UsesAnnotations ): force_history_refresh = tool.force_history_refresh if not job_hda.visible: force_history_refresh = True - rval[id] = { + rval[encoded_id] = { "state": data.state, "html": unicode( trans.fill_template( "root/history_item.mako", data=data, hid=data.hid ), 'utf-8' ), "force_history_refresh": force_history_refresh } return rval + @web.json + def history_get_disk_size( self, trans ): + rval = { 'history' : trans.history.get_disk_size( nice_size=True ) } + for k, v in self.__user_get_usage( trans ).items(): + rval['global_' + k] = v + return rval + + @web.json + def user_get_usage( self, trans ): + return self.__user_get_usage( trans ) + + def __user_get_usage( self, trans ): + usage = trans.app.quota_agent.get_usage( trans ) + percent = trans.app.quota_agent.get_percent( trans=trans, usage=usage ) + rval = {} + if percent is None: + rval['usage'] = util.nice_size( usage ) + else: + rval['percent'] = percent + return rval + + ## ---- Dataset display / editing ---------------------------------------- @web.expose @@ -209,8 +241,7 @@ class RootController( BaseController, UsesHistory, UsesAnnotations ): if data: current_user_roles = trans.get_current_user_roles() if trans.app.security_agent.can_access_dataset( current_user_roles, data.dataset ): - mime = trans.app.datatypes_registry.get_mimetype_by_extension( data.extension.lower() ) - trans.response.set_content_type(mime) + trans.response.set_content_type(data.get_mime()) if tofile: fStat = os.stat(data.file_name) trans.response.headers['Content-Length'] = int(fStat.st_size) @@ -283,207 +314,6 @@ class RootController( BaseController, UsesHistory, UsesAnnotations ): else: yield "No data with id=%d" % id - @web.expose - def edit(self, trans, id=None, hid=None, **kwd): - """Allows user to modify parameters of an HDA.""" - message = '' - error = False - def __ok_to_edit_metadata( dataset_id ): - #prevent modifying metadata when dataset is queued or running as input/output - #This code could be more efficient, i.e. by using mappers, but to prevent slowing down loading a History panel, we'll leave the code here for now - for job_to_dataset_association in trans.sa_session.query( self.app.model.JobToInputDatasetAssociation ) \ - .filter_by( dataset_id=dataset_id ) \ - .all() \ - + trans.sa_session.query( self.app.model.JobToOutputDatasetAssociation ) \ - .filter_by( dataset_id=dataset_id ) \ - .all(): - if job_to_dataset_association.job.state not in [ job_to_dataset_association.job.states.OK, job_to_dataset_association.job.states.ERROR, job_to_dataset_association.job.states.DELETED ]: - return False - return True - if hid is not None: - history = trans.get_history() - # TODO: hid handling - data = history.datasets[ int( hid ) - 1 ] - elif id is not None: - data = trans.sa_session.query( self.app.model.HistoryDatasetAssociation ).get( id ) - else: - trans.log_event( "Problem loading dataset id %s with history id %s." % ( str( id ), str( hid ) ) ) - return trans.show_error_message( "Problem loading dataset." ) - if data is None: - trans.log_event( "Problem retrieving dataset id %s with history id." % ( str( id ), str( hid ) ) ) - return trans.show_error_message( "Problem retrieving dataset." ) - if id is not None and data.history.user is not None and data.history.user != trans.user: - return trans.show_error_message( "This instance of a dataset (%s) in a history does not belong to you." % ( data.id ) ) - current_user_roles = trans.get_current_user_roles() - if trans.app.security_agent.can_access_dataset( current_user_roles, data.dataset ): - if data.state == trans.model.Dataset.states.UPLOAD: - return trans.show_error_message( "Please wait until this dataset finishes uploading before attempting to edit its metadata." ) - params = util.Params( kwd, sanitize=False ) - if params.change: - # The user clicked the Save button on the 'Change data type' form - if data.datatype.allow_datatype_change and trans.app.datatypes_registry.get_datatype_by_extension( params.datatype ).allow_datatype_change: - #prevent modifying datatype when dataset is queued or running as input/output - if not __ok_to_edit_metadata( data.id ): - return trans.show_error_message( "This dataset is currently being used as input or output. You cannot change datatype until the jobs have completed or you have canceled them." ) - trans.app.datatypes_registry.change_datatype( data, params.datatype, set_meta = not trans.app.config.set_metadata_externally ) - trans.sa_session.flush() - if trans.app.config.set_metadata_externally: - trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, trans, incoming = { 'input1':data }, overwrite = False ) #overwrite is False as per existing behavior - return trans.show_ok_message( "Changed the type of dataset '%s' to %s" % ( data.name, params.datatype ), refresh_frames=['history'] ) - else: - return trans.show_error_message( "You are unable to change datatypes in this manner. Changing %s to %s is not allowed." % ( data.extension, params.datatype ) ) - elif params.save: - # The user clicked the Save button on the 'Edit Attributes' form - data.name = params.name - data.info = params.info - message = '' - if __ok_to_edit_metadata( data.id ): - # The following for loop will save all metadata_spec items - for name, spec in data.datatype.metadata_spec.items(): - if spec.get("readonly"): - continue - optional = params.get("is_"+name, None) - other = params.get("or_"+name, None) - if optional and optional == 'true': - # optional element... == 'true' actually means it is NOT checked (and therefore omitted) - setattr(data.metadata, name, None) - else: - if other: - setattr( data.metadata, name, other ) - else: - setattr( data.metadata, name, spec.unwrap( params.get (name, None) ) ) - data.datatype.after_setting_metadata( data ) - # Sanitize annotation before adding it. - if params.annotation: - annotation = sanitize_html( params.annotation, 'utf-8', 'text/html' ) - self.add_item_annotation( trans.sa_session, trans.get_user(), data, annotation ) - # If setting metadata previously failed and all required elements have now been set, clear the failed state. - if data._state == trans.model.Dataset.states.FAILED_METADATA and not data.missing_meta(): - data._state = None - trans.sa_session.flush() - return trans.show_ok_message( "Attributes updated%s" % message, refresh_frames=['history'] ) - else: - trans.sa_session.flush() - return trans.show_warn_message( "Attributes updated, but metadata could not be changed because this dataset is currently being used as input or output. You must cancel or wait for these jobs to complete before changing metadata.", refresh_frames=['history'] ) - elif params.detect: - # The user clicked the Auto-detect button on the 'Edit Attributes' form - #prevent modifying metadata when dataset is queued or running as input/output - if not __ok_to_edit_metadata( data.id ): - return trans.show_error_message( "This dataset is currently being used as input or output. You cannot change metadata until the jobs have completed or you have canceled them." ) - for name, spec in data.metadata.spec.items(): - # We need to be careful about the attributes we are resetting - if name not in [ 'name', 'info', 'dbkey', 'base_name' ]: - if spec.get( 'default' ): - setattr( data.metadata, name, spec.unwrap( spec.get( 'default' ) ) ) - if trans.app.config.set_metadata_externally: - message = 'Attributes have been queued to be updated' - trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, trans, incoming = { 'input1':data } ) - else: - message = 'Attributes updated' - data.set_meta() - data.datatype.after_setting_metadata( data ) - trans.sa_session.flush() - return trans.show_ok_message( message, refresh_frames=['history'] ) - elif params.convert_data: - target_type = kwd.get("target_type", None) - if target_type: - message = data.datatype.convert_dataset(trans, data, target_type) - return trans.show_ok_message( message, refresh_frames=['history'] ) - elif params.update_roles_button: - if not trans.user: - return trans.show_error_message( "You must be logged in if you want to change permissions." ) - if trans.app.security_agent.can_manage_dataset( current_user_roles, data.dataset ): - # The user associated the DATASET_ACCESS permission on the dataset with 1 or more roles. We - # need to ensure that they did not associate roles that would cause accessibility problems. - permissions, in_roles, error, message = \ - trans.app.security_agent.derive_roles_from_access( trans, data.dataset.id, 'root', **kwd ) - a = trans.app.security_agent.get_action( trans.app.security_agent.permitted_actions.DATASET_ACCESS.action ) - if error: - # Keep the original role associations for the DATASET_ACCESS permission on the dataset. - permissions[ a ] = data.dataset.get_access_roles( trans ) - trans.app.security_agent.set_all_dataset_permissions( data.dataset, permissions ) - trans.sa_session.refresh( data.dataset ) - if not message: - message = 'Your changes completed successfully.' - else: - return trans.show_error_message( "You are not authorized to change this dataset's permissions" ) - if "dbkey" in data.datatype.metadata_spec and not data.metadata.dbkey: - # Copy dbkey into metadata, for backwards compatability - # This looks like it does nothing, but getting the dbkey - # returns the metadata dbkey unless it is None, in which - # case it resorts to the old dbkey. Setting the dbkey - # sets it properly in the metadata - #### This is likely no longer required, since the dbkey exists entirely within metadata (the old_dbkey field is gone): REMOVE ME? - data.metadata.dbkey = data.dbkey - # let's not overwrite the imported datatypes module with the variable datatypes? - # the built-in 'id' is overwritten in lots of places as well - ldatatypes = [ dtype_name for dtype_name, dtype_value in trans.app.datatypes_registry.datatypes_by_extension.iteritems() if dtype_value.allow_datatype_change ] - ldatatypes.sort() - all_roles = trans.app.security_agent.get_legitimate_roles( trans, data.dataset, 'root' ) - if error: - status = 'error' - else: - status = 'done' - return trans.fill_template( "/dataset/edit_attributes.mako", - data=data, - data_annotation=self.get_item_annotation_str( trans.sa_session, trans.user, data ), - datatypes=ldatatypes, - current_user_roles=current_user_roles, - all_roles=all_roles, - message=message, - status=status ) - else: - return trans.show_error_message( "You do not have permission to edit this dataset's ( id: %s ) information." % str( id ) ) - - def __delete_dataset( self, trans, id ): - data = trans.sa_session.query( self.app.model.HistoryDatasetAssociation ).get( id ) - if data: - # Walk up parent datasets to find the containing history - topmost_parent = data - while topmost_parent.parent: - topmost_parent = topmost_parent.parent - assert topmost_parent in trans.history.datasets, "Data does not belong to current history" - # Mark deleted and cleanup - data.mark_deleted() - data.clear_associated_files() - trans.log_event( "Dataset id %s marked as deleted" % str(id) ) - if data.parent_id is None and len( data.creating_job_associations ) > 0: - # Mark associated job for deletion - job = data.creating_job_associations[0].job - if job.state in [ self.app.model.Job.states.QUEUED, self.app.model.Job.states.RUNNING, self.app.model.Job.states.NEW ]: - # Are *all* of the job's other output datasets deleted? - if job.check_if_output_datasets_deleted(): - job.mark_deleted( self.app.config.get_bool( 'enable_job_running', True ), - self.app.config.get_bool( 'track_jobs_in_database', False ) ) - self.app.job_manager.job_stop_queue.put( job.id ) - trans.sa_session.flush() - - @web.expose - def delete( self, trans, id = None, show_deleted_on_refresh = False, **kwd): - if id: - if isinstance( id, list ): - dataset_ids = id - else: - dataset_ids = [ id ] - history = trans.get_history() - for id in dataset_ids: - try: - id = int( id ) - except: - continue - self.__delete_dataset( trans, id ) - return self.history( trans, show_deleted = show_deleted_on_refresh ) - - @web.expose - def delete_async( self, trans, id = None, **kwd): - if id: - try: - id = int( id ) - except: - return "Dataset id '%s' is invalid" %str( id ) - self.__delete_dataset( trans, id ) - return "OK" - ## ---- History management ----------------------------------------------- @web.expose @@ -653,49 +483,6 @@ class RootController( BaseController, UsesHistory, UsesAnnotations ): except: return trans.show_error_message( "

    Failed to make secondary dataset primary.

    " ) - # @web.expose - # def masthead( self, trans, active_view=None ): - # brand = trans.app.config.get( "brand", "" ) - # if brand: - # brand ="/%s" % brand - # wiki_url = trans.app.config.get( "wiki_url", "http://g2.trac.bx.psu.edu/" ) - # bugs_email = trans.app.config.get( "bugs_email", "mailto:galaxy-bugs@bx.psu.edu" ) - # blog_url = trans.app.config.get( "blog_url", "http://g2.trac.bx.psu.edu/blog" ) - # screencasts_url = trans.app.config.get( "screencasts_url", "http://g2.trac.bx.psu.edu/wiki/ScreenCasts" ) - # admin_user = "false" - # admin_users = trans.app.config.get( "admin_users", "" ).split( "," ) - # user = trans.get_user() - # if user: - # user_email = trans.get_user().email - # if user_email in admin_users: - # admin_user = "true" - # return trans.fill_template( "/root/masthead.mako", brand=brand, wiki_url=wiki_url, - # blog_url=blog_url,bugs_email=bugs_email, screencasts_url=screencasts_url, admin_user=admin_user, active_view=active_view ) - - # @web.expose - # def dataset_errors( self, trans, id=None, **kwd ): - # """View/fix errors associated with dataset""" - # data = trans.app.model.HistoryDatasetAssociation.get( id ) - # p = kwd - # if p.get("fix_errors", None): - # # launch tool to create new, (hopefully) error free dataset - # tool_params = {} - # tool_params["tool_id"] = 'fix_errors' - # tool_params["runtool_btn"] = 'T' - # tool_params["input"] = id - # tool_params["ext"] = data.ext - # # send methods selected - # repair_methods = data.datatype.repair_methods( data ) - # methods = [] - # for method, description in repair_methods: - # if method in p: methods.append(method) - # tool_params["methods"] = ",".join(methods) - # url = "/tool_runner/index?" + urllib.urlencode(tool_params) - # trans.response.send_redirect(url) - # else: - # history = trans.app.model.History.get( data.history_id ) - # return trans.fill_template('dataset/validation.tmpl', data=data, history=history) - # ---- Debug methods ---------------------------------------------------- @web.expose diff --git a/lib/galaxy/web/controllers/tag.py b/lib/galaxy/web/controllers/tag.py index 5c7af30a0c1..36c6fe4742e 100644 --- a/lib/galaxy/web/controllers/tag.py +++ b/lib/galaxy/web/controllers/tag.py @@ -8,9 +8,9 @@ from sqlalchemy.sql import select log = logging.getLogger( __name__ ) -class TagsController ( BaseController ): +class TagsController ( BaseUIController ): def __init__( self, app ): - BaseController.__init__( self, app ) + BaseUIController.__init__( self, app ) self.tag_handler = app.tag_handler @web.expose @web.require_login( "edit item tags" ) @@ -72,7 +72,7 @@ class TagsController ( BaseController ): if item_id is not None: item = self._get_item( trans, item_class, trans.security.decode_id( item_id ) ) user = trans.user - item_class = self.get_class( trans, item_class ) + item_class = self.get_class( item_class ) q = q.encode( 'utf-8' ) if q.find( ":" ) == -1: return self._get_tag_autocomplete_names( trans, q, limit, timestamp, user, item, item_class ) diff --git a/lib/galaxy/web/controllers/tool_runner.py b/lib/galaxy/web/controllers/tool_runner.py index 941d20f86dc..646c5a3942d 100644 --- a/lib/galaxy/web/controllers/tool_runner.py +++ b/lib/galaxy/web/controllers/tool_runner.py @@ -17,7 +17,7 @@ class AddFrameData: self.debug = None self.from_noframe = None -class ToolRunner( BaseController ): +class ToolRunner( BaseUIController ): #Hack to get biomart to work, ideally, we could pass tool_id to biomart and receive it back @web.expose diff --git a/lib/galaxy/web/controllers/tracks.py b/lib/galaxy/web/controllers/tracks.py index af060d2a3ee..2962554c02b 100644 --- a/lib/galaxy/web/controllers/tracks.py +++ b/lib/galaxy/web/controllers/tracks.py @@ -90,10 +90,14 @@ class LibrarySelectionGrid( LibraryListGrid ): class DbKeyColumn( grids.GridColumn ): """ Column for filtering by and displaying dataset dbkey. """ def filter( self, trans, user, query, dbkey ): - """ Filter by dbkey. """ + """ Filter by dbkey; datasets without a dbkey are returned as well. """ # use raw SQL b/c metadata is a BLOB dbkey = dbkey.replace("'", "\\'") - return query.filter( or_( "metadata like '%%\"dbkey\": [\"%s\"]%%'" % dbkey, "metadata like '%%\"dbkey\": \"%s\"%%'" % dbkey ) ) + return query.filter( or_( \ + or_( "metadata like '%%\"dbkey\": [\"%s\"]%%'" % dbkey, "metadata like '%%\"dbkey\": \"%s\"%%'" % dbkey ), \ + or_( "metadata like '%%\"dbkey\": [\"?\"]%%'", "metadata like '%%\"dbkey\": \"?\"%%'" ) \ + ) + ) class HistoryColumn( grids.GridColumn ): """ Column for filtering by history id. """ @@ -144,21 +148,21 @@ class TracksterSelectionGrid( grids.Grid ): template = "/tracks/add_to_viz.mako" async_template = "/page/select_items_grid_async.mako" model_class = model.Visualization - default_filter = { "deleted" : "False" , "shared" : "All" } - default_sort_key = "title" + default_sort_key = "-update_time" use_async = True use_paging = False columns = [ grids.TextColumn( "Title", key="title", model_class=model.Visualization, filterable="standard" ), - grids.TextColumn( "Dbkey", key="dbkey", model_class=model.Visualization ) + grids.TextColumn( "Dbkey", key="dbkey", model_class=model.Visualization ), + grids.GridColumn( "Last Updated", key="update_time", format=time_ago ) ] def build_initial_query( self, trans, **kwargs ): - return trans.sa_session.query( self.model_class ) + return trans.sa_session.query( self.model_class ).filter( self.model_class.deleted == False ) def apply_query_filter( self, trans, query, **kwargs ): return query.filter( self.model_class.user_id == trans.user.id ) -class TracksController( BaseController, UsesVisualization, UsesHistoryDatasetAssociation ): +class TracksController( BaseUIController, UsesVisualization, UsesHistoryDatasetAssociation ): """ Controller for track browser interface. Handles building a new browser from datasets in the current history, and display of the resulting browser. @@ -168,17 +172,14 @@ class TracksController( BaseController, UsesVisualization, UsesHistoryDatasetAss histories_grid = HistorySelectionGrid() history_datasets_grid = HistoryDatasetsSelectionGrid() tracks_grid = TracksterSelectionGrid() - - # - # TODO: need to encode dataset id and use - # UsesHistoryDatasetAssociation.get_dataset - # for better dataset security. - # - + available_tracks = None available_genomes = None - def _init_references(self, trans): + def _init_references( self, trans ): + """ + Create a list of builds that have reference data specified in twobit.loc file. + """ avail_genomes = {} for line in open( os.path.join( trans.app.config.tool_data_path, "twobit.loc" ) ): if line.startswith("#"): continue @@ -187,6 +188,29 @@ class TracksController( BaseController, UsesVisualization, UsesHistoryDatasetAss key, path = val avail_genomes[key] = path self.available_genomes = avail_genomes + + def _has_reference_data( self, trans, dbkey ): + # Initialize built-in builds if necessary. + if not self.available_genomes: + self._init_references( trans ) + + # Look for key in built-in builds. + if dbkey in self.available_genomes: + # There is built-in reference data. + return True + + # Look for key in user's custom builds. + # TODO: how to make this work for shared visualizations? + user = trans.user + if user and 'dbkeys' in trans.user.preferences: + user_keys = from_json_string( user.preferences['dbkeys'] ) + if dbkey in user_keys: + dbkey_attributes = user_keys[ dbkey ] + if 'fasta' in dbkey_attributes: + # Fasta + converted datasets can provide reference data. + return True + + return False @web.expose @web.require_login() @@ -229,9 +253,7 @@ class TracksController( BaseController, UsesVisualization, UsesHistoryDatasetAss """ Display browser for the datasets listed in `dataset_ids`. """ - decoded_id = trans.security.decode_id( id ) - session = trans.sa_session - vis = session.query( model.Visualization ).get( decoded_id ) + vis = self.get_visualization( trans, id, check_ownership=False, check_accessible=True ) viz_config = self.get_visualization_config( trans, vis ) new_dataset = kwargs.get("dataset_id", None) @@ -295,11 +317,17 @@ class TracksController( BaseController, UsesVisualization, UsesHistoryDatasetAss # len_file = None len_ds = None - # If there is any dataset in the history of extension `len`, this will use it + user_keys = {} if 'dbkeys' in vis_user.preferences: user_keys = from_json_string( vis_user.preferences['dbkeys'] ) - if vis_dbkey in user_keys: - len_file = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( user_keys[ vis_dbkey ][ 'len' ] ).file_name + if vis_dbkey in user_keys: + dbkey_attributes = user_keys[ vis_dbkey ] + if 'fasta' in dbkey_attributes: + build_fasta = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( dbkey_attributes[ 'fasta' ] ) + len_file = build_fasta.get_converted_dataset( trans, 'len' ).file_name + # Backwards compatibility: look for len file directly. + elif 'len' in dbkey_attributes: + len_file = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( user_keys[ vis_dbkey ][ 'len' ] ).file_name if not len_file: len_ds = trans.db_dataset_for( dbkey ) @@ -307,7 +335,7 @@ class TracksController( BaseController, UsesVisualization, UsesHistoryDatasetAss len_file = os.path.join( trans.app.config.len_file_path, "%s.len" % vis_dbkey ) else: len_file = len_ds.file_name - + # # Get chroms data: # (a) chrom name, len; @@ -371,37 +399,56 @@ class TracksController( BaseController, UsesVisualization, UsesHistoryDatasetAss except: # No more chroms to read. pass - - # Check for reference chrom - if self.available_genomes is None: self._init_references(trans) - + to_sort = [{ 'chrom': chrom, 'len': length } for chrom, length in chroms.iteritems()] to_sort.sort(lambda a,b: cmp( split_by_number(a['chrom']), split_by_number(b['chrom']) )) - return { 'reference': vis_dbkey in self.available_genomes, 'chrom_info': to_sort, + return { 'reference': self._has_reference_data( trans, vis_dbkey ), 'chrom_info': to_sort, 'prev_chroms' : prev_chroms, 'next_chroms' : next_chroms, 'start_index' : start_index } @web.json def reference( self, trans, dbkey, chrom, low, high, **kwargs ): - if self.available_genomes is None: self._init_references(trans) - - if dbkey not in self.available_genomes: + """ + Return reference data for a build. + """ + + if not self._has_reference_data( trans, dbkey ): return None + # + # Get twobit file with reference data. + # + twobit_file_name = None + if dbkey in self.available_genomes: + # Built-in twobit. + twobit_file_name = self.available_genomes[dbkey] + else: + # From custom build. + # TODO: how to make this work for shared visualizations? + user = trans.user + user_keys = from_json_string( user.preferences['dbkeys'] ) + dbkey_attributes = user_keys[ dbkey ] + fasta_dataset = trans.app.model.HistoryDatasetAssociation.get( dbkey_attributes[ 'fasta' ] ) + error = self._convert_dataset( trans, fasta_dataset, 'twobit' ) + if error: + return error + else: + twobit_dataset = fasta_dataset.get_converted_dataset( trans, 'twobit' ) + twobit_file_name = twobit_dataset.file_name + + # Read and return reference data. try: - twobit = TwoBitFile( open(self.available_genomes[dbkey]) ) + twobit = TwoBitFile( open( twobit_file_name ) ) + if chrom in twobit: + seq_data = twobit[chrom].get( int(low), int(high) ) + return { 'dataset_type': 'refseq', 'data': seq_data } except IOError: return None - - if chrom in twobit: - return twobit[chrom].get(int(low), int(high)) - - return None @web.json def raw_data( self, trans, dataset_id, chrom, low, high, **kwargs ): """ Uses original (raw) dataset to return data. This method is useful - when the dataset is not yet indexed and hence using /data would + when the dataset is not yet indexed and hence using data would be slow because indexes need to be created. """ @@ -412,10 +459,16 @@ class TracksController( BaseController, UsesVisualization, UsesHistoryDatasetAss return msg # Return data. + data = None + # TODO: for raw data requests, map dataset type to provider using dict in data_providers.py if isinstance( dataset.datatype, Gff ): data = GFFDataProvider( original_dataset=dataset ).get_data( chrom, low, high, **kwargs ) data[ 'dataset_type' ] = 'interval_index' data[ 'extra_info' ] = None + if isinstance( dataset.datatype, Bed ): + data = RawBedDataProvider( original_dataset=dataset ).get_data( chrom, low, high, **kwargs ) + data[ 'dataset_type' ] = 'interval_index' + data[ 'extra_info' ] = None return data @web.json @@ -476,9 +529,9 @@ class TracksController( BaseController, UsesVisualization, UsesHistoryDatasetAss return { "status": messages.DATA, "valid_chroms": valid_chroms } @web.json - def data( self, trans, hda_ldda, dataset_id, chrom, low, high, **kwargs ): + def data( self, trans, hda_ldda, dataset_id, chrom, low, high, start_val=0, max_vals=5000, **kwargs ): """ - Called by the browser to request a block of data + Provides a block of data from a dataset. """ # Parameter check. @@ -533,15 +586,9 @@ class TracksController( BaseController, UsesVisualization, UsesHistoryDatasetAss data_provider = data_provider_class( converted_dataset=converted_dataset, original_dataset=dataset, dependencies=deps ) # Get and return data from data_provider. - data = data_provider.get_data( chrom, low, high, **kwargs ) - message = None - if isinstance(data, dict) and 'message' in data: - message = data['message'] - tracks_dataset_type = data.get( 'data_type', tracks_dataset_type ) - track_data = data['data'] - else: - track_data = data - return { 'dataset_type': tracks_dataset_type, 'extra_info': extra_info, 'data': track_data, 'message': message } + result = data_provider.get_data( chrom, low, high, int(start_val), int(max_vals), **kwargs ) + result.update( { 'dataset_type': tracks_dataset_type, 'extra_info': extra_info } ) + return result @web.json def save( self, trans, **kwargs ): @@ -568,24 +615,51 @@ class TracksController( BaseController, UsesVisualization, UsesHistoryDatasetAss vis_rev.visualization = vis vis_rev.title = vis.title vis_rev.dbkey = dbkey - # Tracks from payload - tracks = [] - for track in decoded_payload['tracks']: - tracks.append( { "dataset_id": track['dataset_id'], - "hda_ldda": track.get('hda_ldda', "hda"), - "name": track['name'], - "track_type": track['track_type'], - "prefs": track['prefs'], - "is_child": track.get('is_child', False) - } ) + + def unpack_track( track_json ): + """ Unpack a track from its json. """ + return { + "dataset_id": track_json['dataset_id'], + "hda_ldda": track_json.get('hda_ldda', "hda"), + "name": track_json['name'], + "track_type": track_json['track_type'], + "prefs": track_json['prefs'], + "mode": track_json['mode'] + } + + def unpack_collection( collection_json ): + """ Unpack a collection from its json. """ + unpacked_drawables = [] + drawables = collection_json[ 'drawables' ] + for drawable_json in drawables: + if 'track_type' in drawable_json: + drawable = unpack_track( drawable_json ) + else: + drawable = unpack_collection( drawable_json ) + unpacked_drawables.append( drawable ) + return { + "name": collection_json.get( 'name', '' ), + "obj_type": collection_json[ 'obj_type' ], + "drawables": unpacked_drawables, + "prefs": collection_json.get( 'prefs' , [] ) + } + + # TODO: unpack and validate bookmarks: + def unpack_bookmarks( bookmarks_json ): + return + + # Unpack and validate view content. + view_content = unpack_collection( decoded_payload[ 'view' ] ) + bookmarks = unpack_bookmarks( decoded_payload[ 'bookmarks' ] ) + vis_rev.config = { "view": view_content, "bookmarks": bookmarks } # Viewport from payload if 'viewport' in decoded_payload: chrom = decoded_payload['viewport']['chrom'] start = decoded_payload['viewport']['start'] end = decoded_payload['viewport']['end'] - vis_rev.config = { "tracks": tracks, "viewport": { 'chrom': chrom, 'start': start, 'end': end } } - else: - vis_rev.config = { "tracks": tracks } + overview = decoded_payload['viewport']['overview'] + vis_rev.config[ "viewport" ] = { 'chrom': chrom, 'start': start, 'end': end, 'overview': overview } + vis.latest_revision = vis_rev session.add( vis_rev ) session.flush() @@ -734,8 +808,7 @@ class TracksController( BaseController, UsesVisualization, UsesHistoryDatasetAss if run_on_region: for jida in original_job.input_datasets: input_dataset = jida.dataset - # TODO: put together more robust way to determine if a dataset can be indexed. - if hasattr( input_dataset, 'get_track_type' ): + if get_data_provider( original_dataset=input_dataset ): # Can index dataset. track_type, data_sources = input_dataset.datatype.get_track_type() # Convert to datasource that provides 'data' because we need to @@ -748,7 +821,7 @@ class TracksController( BaseController, UsesVisualization, UsesHistoryDatasetAss # Return any messages generated during conversions. return_message = _get_highest_priority_msg( messages_list ) if return_message: - return return_message + return to_json_string( return_message ) # # Set target history (the history that tool will use for inputs/outputs). @@ -768,7 +841,9 @@ class TracksController( BaseController, UsesVisualization, UsesHistoryDatasetAss # for jida in original_job.input_datasets: input_dataset = jida.dataset - if run_on_region and hasattr( input_dataset.datatype, 'get_track_type' ): + if input_dataset is None: #optional dataset and dataset wasn't selected + tool_params[ jida.name ] = None + elif run_on_region and hasattr( input_dataset.datatype, 'get_track_type' ): # Dataset is indexed and hence a subset can be extracted and used # as input. track_type, data_sources = input_dataset.datatype.get_track_type() diff --git a/lib/galaxy/web/controllers/ucsc_proxy.py b/lib/galaxy/web/controllers/ucsc_proxy.py index c95f67fdfc6..9245aff3f46 100644 --- a/lib/galaxy/web/controllers/ucsc_proxy.py +++ b/lib/galaxy/web/controllers/ucsc_proxy.py @@ -11,7 +11,7 @@ import re, urllib, logging log = logging.getLogger( __name__ ) -class UCSCProxy( BaseController ): +class UCSCProxy( BaseUIController ): def create_display(self, store): """Creates a more meaningulf display name""" diff --git a/lib/galaxy/web/controllers/user.py b/lib/galaxy/web/controllers/user.py index 05edda0996d..0c6c3ed40b1 100644 --- a/lib/galaxy/web/controllers/user.py +++ b/lib/galaxy/web/controllers/user.py @@ -5,12 +5,12 @@ from galaxy.web.framework.helpers import time_ago, grids from galaxy.web.base.controller import * from galaxy.model.orm import * from galaxy import util, model -import logging, os, string, re, smtplib, socket, glob +import logging, os, string, re, socket, glob from random import choice -from email.MIMEText import MIMEText from galaxy.web.form_builder import * from galaxy.util.json import from_json_string, to_json_string from galaxy.web.framework.helpers import iff +from galaxy.security.validate_user_input import validate_email, validate_username, validate_password log = logging.getLogger( __name__ ) @@ -25,12 +25,9 @@ require_login_template = """ require_login_nocreation_template = require_login_template % "" require_login_creation_template = require_login_template % " If you don't already have an account, you may create one." -VALID_USERNAME_RE = re.compile( "^[a-z0-9\-]+$" ) - OPENID_PROVIDERS = { 'Google' : 'https://www.google.com/accounts/o8/id', 'Yahoo!' : 'http://yahoo.com', 'AOL/AIM' : 'http://openid.aol.com', - 'Flickr' : 'http://flickr.com', 'Launchpad' : 'http://login.launchpad.net', } @@ -51,7 +48,7 @@ class UserOpenIDGrid( grids.Grid ): def build_initial_query( self, trans, **kwd ): return trans.sa_session.query( self.model_class ).filter( self.model_class.user_id == trans.user.id ) -class User( BaseController, UsesFormDefinitions ): +class User( BaseUIController, UsesFormDefinitions ): user_openid_grid = UserOpenIDGrid() installed_len_files = None @@ -111,10 +108,10 @@ class User( BaseController, UsesFormDefinitions ): action = 'login' if auto_associate: action = 'openid_manage' - if trans.app.config.bugs_email is not None: - contact = 'contact support' % trans.app.config.bugs_email + if trans.app.config.support_url is not None: + contact = 'support' % trans.app.config.support_url else: - contact = 'contact support' + contact = 'support' message = 'Verification failed for an unknown reason. Please contact support for assistance.' status = 'error' consumer = trans.app.openid_manager.get_consumer( trans ) @@ -197,7 +194,7 @@ class User( BaseController, UsesFormDefinitions ): message=message, status=status ) ) @web.expose - def openid_associate( self, trans, cntrller, webapp='galaxy', **kwd ): + def openid_associate( self, trans, cntrller='user', webapp='galaxy', **kwd ): if not trans.app.config.enable_openid: return trans.show_error_message( 'OpenID authentication is not enabled in this instance of Galaxy' ) use_panels = util.string_as_bool( kwd.get( 'use_panels', False ) ) @@ -268,7 +265,7 @@ class User( BaseController, UsesFormDefinitions ): if user_type_fd_id == 'none' and user_type_form_definition is not None: user_type_fd_id = trans.security.encode_id( user_type_form_definition.id ) user_type_fd_id_select_field = self.__build_user_type_fd_id_select_field( trans, selected_value=user_type_fd_id ) - widgets = self.__get_widgets( self, trans, user_type_form_definition, user=user, **kwd ) + widgets = self.__get_widgets( trans, user_type_form_definition, user=user, **kwd ) else: user_type_fd_id_select_field = None user_type_form_definition = None @@ -365,7 +362,7 @@ class User( BaseController, UsesFormDefinitions ): else: refresh_frames = [ 'masthead', 'history' ] message, status, user, success = self.__validate_login( trans, webapp, **kwd ) - if success and referer and referer != trans.request.base + url_for( controller='user', action='logout' ): + if success and referer and not referer.startswith( trans.request.base + url_for( controller='user', action='logout' ) ): redirect_url = referer elif success: redirect_url = url_for( '/' ) @@ -395,7 +392,7 @@ class User( BaseController, UsesFormDefinitions ): success = False user = trans.sa_session.query( trans.app.model.User ).filter( trans.app.model.User.table.c.email==email ).first() if not user: - message = "No such user" + message = "No such user (please note that login is case sensitive)" status = 'error' elif user.deleted: message = "This account has been marked deleted, contact your Galaxy administrator to restore the account." @@ -416,7 +413,7 @@ class User( BaseController, UsesFormDefinitions ): success = True return ( message, status, user, success ) @web.expose - def logout( self, trans, webapp='galaxy' ): + def logout( self, trans, webapp='galaxy', logout_all=False ): if webapp == 'galaxy': if trans.app.config.require_login: refresh_frames = [ 'masthead', 'history', 'tools' ] @@ -426,7 +423,7 @@ class User( BaseController, UsesFormDefinitions ): refresh_frames = [ 'masthead' ] # Since logging an event requires a session, we'll log prior to ending the session trans.log_event( "User logged out" ) - trans.handle_user_logout() + trans.handle_user_logout( logout_all=logout_all ) message = 'You have been logged out.
    You can log in again, go back to the page you were visiting or go to the home page.' % \ ( trans.request.referer, url_for( '/' ) ) return trans.fill_template( '/user/logout.mako', @@ -436,7 +433,7 @@ class User( BaseController, UsesFormDefinitions ): status='done', active_view="user" ) @web.expose - def create( self, trans, cntrller, redirect_url='', refresh_frames=[], **kwd ): + def create( self, trans, cntrller='user', redirect_url='', refresh_frames=[], **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) @@ -560,15 +557,12 @@ class User( BaseController, UsesFormDefinitions ): if trans.app.config.smtp_server is None: error = "Now logged in as " + user.email + ". However, subscribing to the mailing list has failed because mail is not configured for this Galaxy instance." else: - msg = MIMEText( 'Join Mailing list.\n' ) - to = msg[ 'To' ] = trans.app.config.mailing_join_addr - frm = msg[ 'From' ] = email - msg[ 'Subject' ] = 'Join Mailing List' + body = 'Join Mailing list.\n' + to = trans.app.config.mailing_join_addr + frm = email + subject = 'Join Mailing List' try: - s = smtplib.SMTP() - s.connect( trans.app.config.smtp_server ) - s.sendmail( frm, [ to ], msg.as_string() ) - s.close() + util.send_mail( frm, to, subject, body, trans.app.config ) except: error = "Now logged in as " + user.email + ". However, subscribing to the mailing list has failed." if not error and not is_admin: @@ -590,57 +584,6 @@ class User( BaseController, UsesFormDefinitions ): message = 'Now logged in as %s.
    Return to the home page.' % ( user.email, url_for( '/' ) ) success = True return ( message, status, user, success ) - def __validate_email( self, trans, email, user=None ): - message = '' - if user and user.email == email: - return message - if len( email ) == 0 or "@" not in email or "." not in email: - message = "Enter a real email address" - elif len( email ) > 255: - message = "Email address exceeds maximum allowable length" - elif trans.sa_session.query( trans.app.model.User ).filter_by( email=email ).first(): - message = "User with that email already exists" - return message - def __validate_username( self, trans, username, user=None ): - # User names must be at least four characters in length and contain only lower-case - # letters, numbers, and the '-' character. - if username in [ 'None', None, '' ]: - return '' - if user and user.username == username: - return '' - if len( username ) < 4: - return "User name must be at least 4 characters in length" - if len( username ) > 255: - return "User name cannot be more than 255 characters in length" - if not( VALID_USERNAME_RE.match( username ) ): - return "User name must contain only lower-case letters, numbers and '-'" - if trans.sa_session.query( trans.app.model.User ).filter_by( username=username ).first(): - return "This user name is not available" - return '' - def __validate_password( self, trans, password, confirm ): - if len( password ) < 6: - return "Use a password of at least 6 characters" - elif password != confirm: - return "Passwords do not match" - return '' - def __validate( self, trans, params, email, password, confirm, username, webapp ): - # If coming from the community webapp, we'll require a public user name - if webapp == 'community' and not username: - return "A public user name is required" - message = self.__validate_email( trans, email ) - if not message: - message = self.__validate_password( trans, password, confirm ) - if not message and username: - message = self.__validate_username( trans, username ) - if not message: - if webapp == 'galaxy': - if self.get_all_forms( trans, - filter=dict( deleted=False ), - form_type=trans.app.model.FormDefinition.types.USER_INFO ): - user_type_fd_id = params.get( 'user_type_fd_id', 'none' ) - if user_type_fd_id in [ 'none' ]: - return "Select the user's type and information" - return message def __get_user_type_form_definition( self, trans, user=None, **kwd ): params = util.Params( kwd ) if user and user.values: @@ -752,7 +695,7 @@ class User( BaseController, UsesFormDefinitions ): if user and params.get( 'change_username_button', False ): username = kwd.get( 'username', '' ) if username: - message = self.__validate_username( trans, username, user ) + message = validate_username( trans, username, user ) if message: status = 'error' else: @@ -788,9 +731,9 @@ class User( BaseController, UsesFormDefinitions ): email = util.restore_text( params.get( 'email', '' ) ) username = util.restore_text( params.get( 'username', '' ) ).lower() # Validate the new values for email and username - message = self.__validate_email( trans, email, user ) + message = validate_email( trans, email, user ) if not message and username: - message = self.__validate_username( trans, username, user ) + message = validate_username( trans, username, user ) if message: status = 'error' else: @@ -819,7 +762,7 @@ class User( BaseController, UsesFormDefinitions ): ok = False if ok: # Validate the new password - message = self.__validate_password( trans, password, confirm ) + message = validate_password( trans, password, confirm ) if message: status = 'error' else: @@ -863,6 +806,8 @@ class User( BaseController, UsesFormDefinitions ): kwd[ 'id' ] = user_id if message: kwd[ 'message' ] = util.sanitize_text( message ) + if status: + kwd[ 'status' ] = status return trans.response.send_redirect( web.url_for( controller='user', action='manage_user_info', cntrller=cntrller, @@ -888,15 +833,12 @@ class User( BaseController, UsesFormDefinitions ): host = trans.request.host.split(':')[0] if host == 'localhost': host = socket.getfqdn() - msg = MIMEText( 'Your password on %s has been reset to:\n\n %s\n' % ( host, new_pass ) ) - to = msg[ 'To' ] = email - frm = msg[ 'From' ] = 'galaxy-no-reply@' + host - msg[ 'Subject' ] = 'Galaxy Password Reset' + body = 'Your password on %s has been reset to:\n\n %s\n' % ( host, new_pass ) + to = email + frm = 'galaxy-no-reply@' + host + subject = 'Galaxy Password Reset' try: - s = smtplib.SMTP() - s.connect( trans.app.config.smtp_server ) - s.sendmail( frm, [ to ], msg.as_string() ) - s.close() + util.send_mail( frm, to, subject, body, trans.app.config ) reset_user.set_password_cleartext( new_pass ) trans.sa_session.add( reset_user ) trans.sa_session.flush() @@ -918,9 +860,30 @@ class User( BaseController, UsesFormDefinitions ): webapp=webapp, message=message, status=status ) + def __validate( self, trans, params, email, password, confirm, username, webapp ): + # If coming from the community webapp, we'll require a public user name + if webapp == 'community' and not username: + return "A public user name is required" + message = validate_email( trans, email ) + if not message: + message = validate_password( trans, password, confirm ) + if not message and username: + message = validate_username( trans, username ) + if not message: + if webapp == 'galaxy': + if self.get_all_forms( trans, + filter=dict( deleted=False ), + form_type=trans.app.model.FormDefinition.types.USER_INFO ): + user_type_fd_id = params.get( 'user_type_fd_id', 'none' ) + if user_type_fd_id in [ 'none' ]: + return "Select the user's type and information" + return message @web.expose def set_default_permissions( self, trans, cntrller, **kwd ): """Sets the user's default permissions for the new histories""" + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) if trans.user: if 'update_roles_button' in kwd: p = util.Params( kwd ) @@ -933,8 +896,11 @@ class User( BaseController, UsesFormDefinitions ): action = trans.app.security_agent.get_action( v.action ).action permissions[ action ] = in_roles trans.app.security_agent.user_set_default_permissions( trans.user, permissions ) - return trans.show_ok_message( 'Default new history permissions have been changed.' ) - return trans.fill_template( 'user/permissions.mako', cntrller=cntrller ) + message = 'Default new history permissions have been changed.' + return trans.fill_template( 'user/permissions.mako', + cntrller=cntrller, + message=message, + status=status ) else: # User not logged in, history group must be only public return trans.show_error_message( "You must be logged in to change your default permitted actions." ) @@ -1198,6 +1164,9 @@ class User( BaseController, UsesFormDefinitions ): @web.expose @web.require_login() def dbkeys( self, trans, **kwds ): + # + # Process arguments and add/delete build. + # user = trans.user message = None lines_skipped = 0 @@ -1211,57 +1180,79 @@ class User( BaseController, UsesFormDefinitions ): else: dbkeys = from_json_string(user.preferences['dbkeys']) if 'delete' in kwds: + # Delete a build. key = kwds.get('key', '') if key and key in dbkeys: del dbkeys[key] elif 'add' in kwds: - name = kwds.get('name', '') - key = kwds.get('key', '') - len_file = kwds.get('len_file', None) - if getattr(len_file, "file", None): # Check if it's a FieldStorage object - len_text = len_file.file.read() - else: - len_text = kwds.get('len_text', '') - if not name or not key or not len_text: + # Add new custom build. + name = kwds.get('name', '') + key = kwds.get('key', '') + dataset_id = kwds.get('dataset_id', '') + if not name or not key or not dataset_id: message = "You must specify values for all the fields." elif key in dbkeys: message = "There is already a custom build with that key. Delete it first if you want to replace it." else: - # Create new len file - new_len = trans.app.model.HistoryDatasetAssociation( extension="len", create_dataset=True, sa_session=trans.sa_session ) - trans.sa_session.add( new_len ) - new_len.name = name - new_len.visible = False - new_len.state = trans.app.model.Job.states.OK - new_len.info = "custom build .len file" - trans.sa_session.flush() - counter = 0 - f = open(new_len.file_name, "w") - # LEN files have format: - # - for line in len_text.split("\n"): - lst = line.strip().rsplit(None, 1) # Splits at the last whitespace in the line - if not lst or len(lst) < 2: - lines_skipped += 1 - continue - chrom, length = lst[0], lst[1] - try: - length = int(length) - except ValueError: - lines_skipped += 1 - continue - counter += 1 - f.write("%s\t%s\n" % (chrom, length)) - f.close() - dbkeys[key] = { "name": name, "len": new_len.id, "count": counter } + dataset_id = trans.security.decode_id( dataset_id ) + dbkeys[key] = { "name": name, "fasta": dataset_id } + # Save builds. + # TODO: use database table to save builds. user.preferences['dbkeys'] = to_json_string(dbkeys) trans.sa_session.flush() + + # + # Display custom builds page. + # + + # Add chrom/contig count to dbkeys dict. + updated = False + for key, attributes in dbkeys.items(): + if 'count' in attributes: + # Already have count, so do nothing. + continue + + # Get len file. + fasta_dataset = trans.app.model.HistoryDatasetAssociation.get( attributes[ 'fasta' ] ) + len_dataset = fasta_dataset.get_converted_dataset( trans, "len" ) + # HACK: need to request dataset again b/c get_converted_dataset() + # doesn't return dataset (as it probably should). + len_dataset = fasta_dataset.get_converted_dataset( trans, "len" ) + if len_dataset.state == trans.app.model.Job.states.ERROR: + # Can't use len dataset. + continue + + # Get chrom count file. + # NOTE: this conversion doesn't work well with set_metadata_externally=False + # because the conversion occurs before metadata can be set; the + # dataset is marked as deleted and a subsequent conversion is run. + chrom_count_dataset = len_dataset.get_converted_dataset( trans, "linecount" ) + if not chrom_count_dataset or chrom_count_dataset.state != trans.app.model.Job.states.OK: + # No valid linecount dataset. + continue + else: + # Set chrom count. + chrom_count = int( open( chrom_count_dataset.file_name ).readline() ) + attributes[ 'count' ] = chrom_count + updated = True + + if updated: + user.preferences['dbkeys'] = to_json_string(dbkeys) + trans.sa_session.flush() + + + # Potential genome data for custom builds is limited to fasta datasets in current history for now. + fasta_hdas = trans.sa_session.query( model.HistoryDatasetAssociation ) \ + .filter_by( history=trans.history, extension="fasta", deleted=False ) \ + .order_by( model.HistoryDatasetAssociation.hid.desc() ) + return trans.fill_template( 'user/dbkeys.mako', user=user, dbkeys=dbkeys, message=message, installed_len_files=self.installed_len_files, lines_skipped=lines_skipped, + fasta_hdas=fasta_hdas, use_panels=kwds.get( 'use_panels', None ) ) @web.expose @web.require_login() diff --git a/lib/galaxy/web/controllers/visualization.py b/lib/galaxy/web/controllers/visualization.py index 1dd06e9e85e..4d5841a181b 100644 --- a/lib/galaxy/web/controllers/visualization.py +++ b/lib/galaxy/web/controllers/visualization.py @@ -68,7 +68,7 @@ class VisualizationAllPublishedGrid( grids.Grid ): return query.filter( self.model_class.deleted==False ).filter( self.model_class.published==True ) -class VisualizationController( BaseController, Sharable, UsesAnnotations, +class VisualizationController( BaseUIController, Sharable, UsesAnnotations, UsesHistoryDatasetAssociation, UsesVisualization, UsesItemRatings ): _user_list_grid = VisualizationListGrid() @@ -87,34 +87,27 @@ class VisualizationController( BaseController, Sharable, UsesAnnotations, @web.require_login( "use Galaxy visualizations", use_panels=True ) def index( self, trans, *args, **kwargs ): """ Lists user's saved visualizations. """ - return self.list( trans, args, kwargs ) + return self.list( trans, *args, **kwargs ) @web.expose @web.require_login() def clone(self, trans, id, *args, **kwargs): - viz = self.get_visualization( trans, id, check_ownership=False ) + visualization = self.get_visualization( trans, id, check_ownership=False ) user = trans.get_user() - if viz.user == user: - owner = True - else: - if trans.sa_session.query( model.VisualizationUserShareAssociation ) \ - .filter_by( user=user, visualization=viz ).count() == 0: - error( "Visualization is not owned by or shared with current user" ) - owner = False - new_viz = model.Visualization() - new_viz.title = "Clone of '%s'" % viz.title - new_viz.dbkey = viz.dbkey - new_viz.type = viz.type - new_viz.latest_revision = viz.latest_revision + owner = ( visualization.user == user ) + new_title = "Copy of '%s'" % visualization.title if not owner: - new_viz.title += " shared by '%s'" % viz.user.email - new_viz.user = user + new_title += " shared by %s" % visualization.user.email + + cloned_visualization = visualization.copy( user=trans.user, title=new_title ) + # Persist session = trans.sa_session - session.add( new_viz ) + session.add( cloned_visualization ) session.flush() + # Display the management page - trans.set_message( 'Clone created with name "%s"' % new_viz.title ) + trans.set_message( 'Copy created with name "%s"' % cloned_visualization.title ) return self.list( trans ) @web.expose @@ -202,18 +195,15 @@ class VisualizationController( BaseController, Sharable, UsesAnnotations, visualization = self.get_visualization( trans, id, check_ownership=False ) if visualization.importable == False: return trans.show_error_message( "The owner of this visualization has disabled imports via this link.
    You can %s" % referer_message, use_panels=True ) - elif visualization.user == trans.user: - return trans.show_error_message( "You can't import this visualization because you own it.
    You can %s" % referer_message, use_panels=True ) elif visualization.deleted: return trans.show_error_message( "You can't import this visualization because it has been deleted.
    You can %s" % referer_message, use_panels=True ) else: - # Create imported visualization via copy. TODO: Visualizations use datasets -- do we need to check to ensure that - # datasets can be imported/viewed and/or copy datasets to user? - imported_visualization = model.Visualization() - imported_visualization.title = "imported: " + visualization.title - imported_visualization.latest_revision = visualization.latest_revision - imported_visualization.user = trans.user - # Save new visualization. + # Create imported visualization via copy. + # TODO: need to handle custom db keys. + + imported_visualization = visualization.copy( user=trans.user, title="imported: " + visualization.title ) + + # Persist session = trans.sa_session session.add( imported_visualization ) session.flush() @@ -310,7 +300,7 @@ class VisualizationController( BaseController, Sharable, UsesAnnotations, raise web.httpexceptions.HTTPNotFound() # Security check raises error if user cannot access visualization. - self.security_check( trans.get_user(), visualization, False, True) + self.security_check( trans, visualization, False, True) # Get rating data. user_item_rating = 0 @@ -462,4 +452,6 @@ class VisualizationController( BaseController, Sharable, UsesAnnotations, help="A description of the visualization; annotation is shown alongside published visualizations."), template="visualization/create.mako" ) - \ No newline at end of file + def get_item( self, trans, id ): + return self.get_visualization( trans, id ) + diff --git a/lib/galaxy/web/controllers/workflow.py b/lib/galaxy/web/controllers/workflow.py index 9333f2d6b68..b1700fe9b7a 100644 --- a/lib/galaxy/web/controllers/workflow.py +++ b/lib/galaxy/web/controllers/workflow.py @@ -103,11 +103,11 @@ class SingleTagContentsParser( sgmllib.SGMLParser ): if self.cur_tag == self.target_tag: self.tag_content += text -class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnnotations, UsesItemRatings ): +class WorkflowController( BaseUIController, Sharable, UsesStoredWorkflow, UsesAnnotations, UsesItemRatings ): stored_list_grid = StoredWorkflowListGrid() published_list_grid = StoredWorkflowAllPublishedGrid() - __myexp_url = "sandbox.myexperiment.org:80" + __myexp_url = "www.myexperiment.org:80" @web.expose def index( self, trans ): @@ -199,7 +199,7 @@ class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnno if stored_workflow is None: raise web.httpexceptions.HTTPNotFound() # Security check raises error if user cannot access workflow. - self.security_check( trans.get_user(), stored_workflow, False, True) + self.security_check( trans, stored_workflow, False, True) # Get data for workflow's steps. self.get_stored_workflow_steps( trans, stored_workflow ) @@ -364,7 +364,7 @@ class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnno # Update workflow attributes if new values submitted. if 'name' in kwargs: # Rename workflow. - stored.name = kwargs[ 'name' ] + stored.name = sanitize_html( kwargs['name'] ) if 'annotation' in kwargs: # Set workflow annotation; sanitize annotation before adding it. annotation = sanitize_html( kwargs[ 'annotation' ], 'utf-8', 'text/html' ) @@ -380,7 +380,9 @@ class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnno def rename( self, trans, id, new_name=None, **kwargs ): stored = self.get_stored_workflow( trans, id ) if new_name is not None: - stored.name = new_name + san_new_name = sanitize_html( new_name ) + stored.name = san_new_name + stored.latest_workflow.name = san_new_name trans.sa_session.flush() # For current workflows grid: trans.set_message ( "Workflow renamed to '%s'." % new_name ) @@ -398,7 +400,9 @@ class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnno def rename_async( self, trans, id, new_name=None, **kwargs ): stored = self.get_stored_workflow( trans, id ) if new_name: - stored.name = new_name + san_new_name = sanitize_html( new_name ) + stored.name = san_new_name + stored.latest_workflow.name = san_new_name trans.sa_session.flush() return stored.name @@ -972,7 +976,7 @@ class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnno # NOTE: blocks web thread. headers = {} if myexp_username and myexp_password: - auth_header = base64.b64encode( '%s:%s' % ( myexp_username, myexp_password ))[:-1] + auth_header = base64.b64encode( '%s:%s' % ( myexp_username, myexp_password )) headers = { "Authorization" : "Basic %s" % auth_header } conn.request( "GET", "/workflow.xml?id=%s&elements=content" % myexp_id, headers=headers ) response = conn.getresponse() @@ -1012,7 +1016,7 @@ class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnno id = trans.security.decode_id( id ) trans.workflow_building_mode = True stored = trans.sa_session.query( model.StoredWorkflow ).get( id ) - self.security_check( trans.get_user(), stored, False, True ) + self.security_check( trans, stored, False, True ) # Convert workflow to dict. workflow_dict = self._workflow_to_dict( trans, stored ) @@ -1037,8 +1041,8 @@ class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnno request = unicode( request_raw.strip(), 'utf-8' ) # Do request and get result. - auth_header = base64.b64encode( '%s:%s' % ( myexp_username, myexp_password ))[:-1] - headers = { "Content-type": "text/xml", "Accept": "text/plain", "Authorization" : "Basic %s" % auth_header } + auth_header = base64.b64encode( '%s:%s' % ( myexp_username, myexp_password )) + headers = { "Content-type": "text/xml", "Accept": "text/xml", "Authorization" : "Basic %s" % auth_header } conn = httplib.HTTPConnection( self.__myexp_url ) # NOTE: blocks web thread. conn.request("POST", "/workflow.xml", request, headers) @@ -1432,7 +1436,10 @@ class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnno has_upgrade_messages=has_upgrade_messages, errors=errors, incoming=kwargs ) - + + def get_item( self, trans, id ): + return self.get_stored_workflow( trans, id ) + @web.expose def tag_outputs( self, trans, id, **kwargs ): stored = self.get_stored_workflow( trans, id, check_ownership=False ) @@ -1592,6 +1599,15 @@ class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnno 'annotation' : annotation_str } + # Add post-job actions to step dict. + if module.type == 'tool': + pja_dict = {} + for pja in step.post_job_actions: + pja_dict[pja.action_type+pja.output_name] = dict( action_type = pja.action_type, + output_name = pja.output_name, + action_arguments = pja.action_arguments ) + step_dict[ 'post_job_actions' ] = pja_dict + # Data inputs step_dict['inputs'] = [] if module.type == "data_input": @@ -1695,6 +1711,12 @@ class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnno if annotation: annotation = sanitize_html( annotation, 'utf-8', 'text/html' ) self.add_item_annotation( trans.sa_session, trans.get_user(), step, annotation ) + # Unpack and add post-job actions. + post_job_actions = step_dict.get( 'post_job_actions', {} ) + for name, pja_dict in post_job_actions.items(): + pja = PostJobAction( pja_dict[ 'action_type' ], + step, pja_dict[ 'output_name' ], + pja_dict[ 'action_arguments' ] ) # Second pass to deal with connections between steps for step in steps: # Input connections diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index d05738a78af..248285fbc8e 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -4,7 +4,7 @@ Classes for generating HTML forms import logging, sys, os, time from cgi import escape -from galaxy.util import restore_text, relpath, nice_size, string_as_bool +from galaxy.util import restore_text, relpath, nice_size from galaxy.web import url_for log = logging.getLogger(__name__) @@ -88,9 +88,17 @@ class CheckboxField(BaseField): >>> print CheckboxField( "bar", checked="yes" ).get_html() """ - def __init__( self, name, checked=None ): + def __init__( self, name, checked=None, refresh_on_change = False, refresh_on_change_values = None ): self.name = name self.checked = ( checked == True ) or ( isinstance( checked, basestring ) and ( checked.lower() in ( "yes", "true", "on" ) ) ) + self.refresh_on_change = refresh_on_change + self.refresh_on_change_values = refresh_on_change_values or [] + if self.refresh_on_change: + self.refresh_on_change_text = ' refresh_on_change="true" ' + if self.refresh_on_change_values: + self.refresh_on_change_text = '%s refresh_on_change_values="%s" ' % ( self.refresh_on_change_text, ",".join( self.refresh_on_change_values ) ) + else: + self.refresh_on_change_text = '' def get_html( self, prefix="", disabled=False ): if self.checked: checked_text = ' checked="checked"' @@ -102,8 +110,8 @@ class CheckboxField(BaseField): # parsing the request, the value 'true' in the hidden field actually means it is NOT checked. # See the is_checked() method below. The prefix is necessary in each case to ensure functional # correctness when the param is inside a conditional. - return '' \ - % ( id_name, id_name, checked_text, self.get_disabled_str( disabled ), prefix, self.name, self.get_disabled_str( disabled ) ) + return '' \ + % ( id_name, id_name, checked_text, self.get_disabled_str( disabled ), self.refresh_on_change_text, prefix, self.name, self.get_disabled_str( disabled ) ) @staticmethod def is_checked( value ): if value == True: @@ -112,8 +120,6 @@ class CheckboxField(BaseField): # above for clarification. Basically, if value is not True, then it will always be a list with # 2 input fields ( a checkbox and a hidden field ) if the checkbox is checked. If it is not # checked, then value will be only the hidden field. - if isinstance( value, basestring ): - return string_as_bool( value ) return isinstance( value, list ) and len( value ) == 2 def set_checked(self, value): if isinstance( value, basestring ): @@ -185,9 +191,9 @@ class FTPFileField(BaseField): def get_html( self, prefix="" ): rval = FTPFileField.thead if self.dir is None: - rval += 'Please create or log in to a Galaxy account to view files uploaded via FTP.' % ( url_for( controller='user', action='create', referer=url_for( controller='root' ) ), url_for( controller='user', action='login', referer=url_for( controller='root' ) ) ) + rval += 'Please create or log in to a Galaxy account to view files uploaded via FTP.' % ( url_for( controller='user', action='create', cntrller='user', referer=url_for( controller='root' ) ), url_for( controller='user', action='login', cntrller='user', referer=url_for( controller='root' ) ) ) elif not os.path.exists( self.dir ): - rval += 'Your FTP upload directory contains no files.' + rval += 'Your FTP upload directory contains no files.' else: uploads = [] for ( dirpath, dirnames, filenames ) in os.walk( self.dir ): @@ -198,7 +204,7 @@ class FTPFileField(BaseField): size=nice_size( statinfo.st_size ), ctime=time.strftime( "%m/%d/%Y %I:%M:%S %p", time.localtime( statinfo.st_ctime ) ) ) ) if not uploads: - rval += 'Your FTP upload directory contains no files.' + rval += 'Your FTP upload directory contains no files.' for upload in uploads: rval += FTPFileField.trow % ( prefix, self.name, upload['path'], upload['path'], upload['size'], upload['ctime'] ) rval += FTPFileField.tfoot @@ -255,7 +261,7 @@ class SelectField(BaseField):
    """ - def __init__( self, name, multiple=None, display=None, refresh_on_change=False, refresh_on_change_values=[], size=None ): + def __init__( self, name, multiple=None, display=None, refresh_on_change=False, refresh_on_change_values=None, size=None ): self.name = name self.multiple = multiple or False self.size = size @@ -268,7 +274,7 @@ class SelectField(BaseField): raise Exception, "Unknown display type: %s" % display self.display = display self.refresh_on_change = refresh_on_change - self.refresh_on_change_values = refresh_on_change_values + self.refresh_on_change_values = refresh_on_change_values or [] if self.refresh_on_change: self.refresh_on_change_text = ' refresh_on_change="true"' if self.refresh_on_change_values: @@ -646,6 +652,27 @@ class HistoryField( BaseField ): return self.value else: return '-' + +class LibraryField( BaseField ): + def __init__( self, name, value=None, trans=None ): + self.name = name + self.lddas = value + self.trans = trans + def get_html( self, prefix="", disabled=False ): + if not self.lddas: + ldda_ids = "" + text = "Select library dataset(s)" + else: + ldda_ids = "||".join( [ self.trans.security.encode_id( ldda.id ) for ldda in self.lddas ] ) + text = "
    ".join( [ "%s. %s" % (i+1, ldda.name) for i, ldda in enumerate(self.lddas)] ) + return '%s \ + ' % ( text, prefix, self.name, escape( str(ldda_ids), quote=True ) ) + + def get_display_text(self): + if self.ldda: + return self.ldda.name + else: + return 'None' def get_suite(): """Get unittest suite for this module""" diff --git a/lib/galaxy/web/framework/__init__.py b/lib/galaxy/web/framework/__init__.py index 08e7b9cf4d6..fbc1283264d 100644 --- a/lib/galaxy/web/framework/__init__.py +++ b/lib/galaxy/web/framework/__init__.py @@ -10,6 +10,7 @@ from Cheetah.Template import Template import base import pickle from galaxy import util +from galaxy.exceptions import MessageException from galaxy.util.json import to_json_string, from_json_string pkg_resources.require( "simplejson" ) @@ -19,6 +20,7 @@ import helpers pkg_resources.require( "PasteDeploy" ) from paste.deploy.converters import asbool +import paste.httpexceptions pkg_resources.require( "Mako" ) import mako.template @@ -103,6 +105,9 @@ def expose_api( func ): except NoResultFound: error_message = 'Provided API key is not valid.' return error + if provided_key.user.deleted: + error_message = 'User account is deactivated, please contact an administrator.' + return error newest_key = provided_key.user.api_keys[0] if newest_key.key != provided_key.key: error_message = 'Provided API key has expired.' @@ -117,10 +122,34 @@ def expose_api( func ): return error trans.response.set_content_type( "application/json" ) trans.set_user( provided_key.user ) - if trans.debug: - return simplejson.dumps( func( self, trans, *args, **kwargs ), indent=4, sort_keys=True ) - else: - return simplejson.dumps( func( self, trans, *args, **kwargs ) ) + # Perform api_run_as processing, possibly changing identity + if 'run_as' in kwargs: + if not trans.user_can_do_run_as(): + error_message = 'User does not have permissions to run jobs as another user' + return error + try: + decoded_user_id = trans.security.decode_id( kwargs['run_as'] ) + except TypeError: + trans.response.status = 400 + return "Malformed user id ( %s ) specified, unable to decode." % str( kwargs['run_as'] ) + try: + user = trans.sa_session.query( trans.app.model.User ).get( decoded_user_id ) + trans.api_inherit_admin = trans.user_is_admin() + trans.set_user(user) + except: + trans.response.status = 400 + return "That user does not exist." + + try: + if trans.debug: + return simplejson.dumps( func( self, trans, *args, **kwargs ), indent=4, sort_keys=True ) + else: + return simplejson.dumps( func( self, trans, *args, **kwargs ) ) + except paste.httpexceptions.HTTPException: + raise # handled + except: + log.exception( 'Uncaught exception in exposed API method:' ) + raise paste.httpexceptions.HTTPServerError() if not hasattr(func, '_orig'): decorator._orig = func decorator.exposed = True @@ -128,27 +157,24 @@ def expose_api( func ): def require_admin( func ): def decorator( self, trans, *args, **kwargs ): - admin_users = trans.app.config.get( "admin_users", "" ).split( "," ) - if not admin_users: - return trans.show_error_message( "You must be logged in as an administrator to access this feature, but no administrators are set in the Galaxy configuration." ) - user = trans.get_user() - if not user: - return trans.show_error_message( "You must be logged in as an administrator to access this feature." ) - if not user.email in admin_users: - return trans.show_error_message( "You must be an administrator to access this feature." ) + if not trans.user_is_admin(): + msg = "You must be an administrator to access this feature." + admin_users = trans.app.config.get( "admin_users", "" ).split( "," ) + user = trans.get_user() + if not admin_users: + msg = "You must be logged in as an administrator to access this feature, but no administrators are set in the Galaxy configuration." + elif not user: + msg = "You must be logged in as an administrator to access this feature." + trans.response.status = 403 + if trans.response.get_content_type() == 'application/json': + return msg + else: + return trans.show_error_message( msg ) return func( self, trans, *args, **kwargs ) return decorator NOT_SET = object() -class MessageException( Exception ): - """ - Exception to make throwing errors from deep in controllers easier - """ - def __init__( self, err_msg, type="info" ): - self.err_msg = err_msg - self.type = type - def error( message ): raise MessageException( message, type='error' ) @@ -197,6 +223,8 @@ class GalaxyWebTransaction( base.DefaultWebTransaction ): # that the current history should not be used for parameter values # and such). self.workflow_building_mode = False + # Flag indicating whether this is an API call and the API key user is an administrator + self.api_inherit_admin = False def setup_i18n( self ): locales = [] if 'HTTP_ACCEPT_LANGUAGE' in self.environ: @@ -471,6 +499,7 @@ class GalaxyWebTransaction( base.DefaultWebTransaction ): - associate new session with user - if old session had a history and it was not associated with a user, associate it with the new session, otherwise associate the current session's history with the user + - add the disk usage of the current session to the user's total disk usage """ # Set the previous session prev_galaxy_session = self.galaxy_session @@ -494,6 +523,10 @@ class GalaxyWebTransaction( base.DefaultWebTransaction ): # If the previous galaxy session had a history, associate it with the new # session, but only if it didn't belong to a different user. history = prev_galaxy_session.current_history + if prev_galaxy_session.user is None: + # Increase the user's disk usage by the amount of the previous history's datasets if they didn't already own it. + for hda in history.datasets: + user.total_disk_usage += hda.quota_amount( user ) elif self.galaxy_session.current_history: history = self.galaxy_session.current_history if not history and \ @@ -518,7 +551,7 @@ class GalaxyWebTransaction( base.DefaultWebTransaction ): self.sa_session.flush() # This method is not called from the Galaxy reports, so the cookie will always be galaxysession self.__update_session_cookie( name=cookie_name ) - def handle_user_logout( self ): + def handle_user_logout( self, logout_all=False ): """ Logout the current user: - invalidate the current session @@ -528,6 +561,14 @@ class GalaxyWebTransaction( base.DefaultWebTransaction ): prev_galaxy_session.is_valid = False self.galaxy_session = self.__create_new_session( prev_galaxy_session ) self.sa_session.add_all( ( prev_galaxy_session, self.galaxy_session ) ) + galaxy_user_id = prev_galaxy_session.user_id + if logout_all and galaxy_user_id is not None: + for other_galaxy_session in self.sa_session.query( self.app.model.GalaxySession ) \ + .filter( and_( self.app.model.GalaxySession.table.c.user_id==galaxy_user_id, + self.app.model.GalaxySession.table.c.is_valid==True, + self.app.model.GalaxySession.table.c.id!=prev_galaxy_session.id ) ): + other_galaxy_session.is_valid = False + self.sa_session.add( other_galaxy_session ) self.sa_session.flush() # This method is not called from the Galaxy reports, so the cookie will always be galaxysession self.__update_session_cookie( name='galaxysession' ) @@ -588,8 +629,13 @@ class GalaxyWebTransaction( base.DefaultWebTransaction ): roles = [] return roles def user_is_admin( self ): + if self.api_inherit_admin: + return True admin_users = self.app.config.get( "admin_users", "" ).split( "," ) return self.user and admin_users and self.user.email in admin_users + def user_can_do_run_as( self ): + run_as_users = self.app.config.get( "api_allow_run_as", "" ).split( "," ) + return self.user and run_as_users and self.user.email in run_as_users def get_toolbox(self): """Returns the application toolbox""" return self.app.toolbox diff --git a/lib/galaxy/web/framework/base.py b/lib/galaxy/web/framework/base.py index a39babdb981..79be1a61fd1 100644 --- a/lib/galaxy/web/framework/base.py +++ b/lib/galaxy/web/framework/base.py @@ -29,6 +29,21 @@ import cgi log = logging.getLogger( __name__ ) +def __resource_with_deleted( self, member_name, collection_name, **kwargs ): + """ + Method to monkeypatch on to routes.mapper.Mapper which does the same thing + as resource() with the addition of standardized routes for handling + elements in Galaxy's "deleted but not really deleted" fashion. + """ + collection_path = kwargs.get( 'path_prefix', '' ) + '/' + collection_name + '/deleted' + member_path = collection_path + '/:id' + self.connect( 'deleted_' + collection_name, collection_path, controller=collection_name, action='index', deleted=True, conditions=dict( method=['GET'] ) ) + self.connect( 'deleted_' + member_name, member_path, controller=collection_name, action='show', deleted=True, conditions=dict( method=['GET'] ) ) + self.connect( 'undelete_deleted_' + member_name, member_path + '/undelete', controller=collection_name, action='undelete', + conditions=dict( method=['POST'] ) ) + self.resource( member_name, collection_name, **kwargs ) +routes.Mapper.resource_with_deleted = __resource_with_deleted + class WebApplication( object ): """ A simple web application which maps requests to objects using routes, @@ -53,7 +68,7 @@ class WebApplication( object ): self.mapper.explicit = False self.api_mapper = routes.Mapper() self.transaction_factory = DefaultWebTransaction - def add_controller( self, controller_name, controller ): + def add_ui_controller( self, controller_name, controller ): """ Add a controller class to this application. A controller class has methods which handle web requests. To connect a URL to a controller's @@ -320,6 +335,8 @@ class Response( object ): Sets the Content-Type header """ self.headers[ "content-type" ] = type + def get_content_type( self ): + return self.headers[ "content-type" ] def send_redirect( self, url ): """ Send an HTTP redirect response to (target `url`) diff --git a/lib/galaxy/web/framework/helpers/__init__.py b/lib/galaxy/web/framework/helpers/__init__.py index 7937406d224..02441e0cbfe 100644 --- a/lib/galaxy/web/framework/helpers/__init__.py +++ b/lib/galaxy/web/framework/helpers/__init__.py @@ -6,9 +6,12 @@ from webhelpers import * from galaxy.util.json import to_json_string from galaxy.util import hash_util from datetime import datetime, timedelta +import time from cgi import escape +server_starttime = int(time.time()) + # If the date is more than one week ago, then display the actual date instead of in words def time_ago( x ): delta = timedelta(weeks=1) @@ -38,20 +41,18 @@ def css( *args ): Take a list of stylesheet names (no extension) and return appropriate string of link tags. - TODO: This has a hardcoded "?v=X" to defeat caching. This should be done - in a better way. + Cache-bust with time that server started running on """ - return "\n".join( [ stylesheet_link_tag( "/static/style/" + name + ".css?v=3" ) for name in args ] ) + return "\n".join( [ stylesheet_link_tag( "/static/style/" + name + ".css?v=%s" % server_starttime ) for name in args ] ) def js( *args ): """ Take a list of javascript names (no extension) and return appropriate string of script tags. - TODO: This has a hardcoded "?v=X" to defeat caching. This should be done - in a better way. + Cache-bust with time that server started running on """ - return "\n".join( [ javascript_include_tag( "/static/scripts/" + name + ".js?v=8" ) for name in args ] ) + return "\n".join( [ javascript_include_tag( "/static/scripts/" + name + ".js?v=%s" % server_starttime ) for name in args ] ) # Hashes diff --git a/lib/galaxy/web/framework/helpers/grids.py b/lib/galaxy/web/framework/helpers/grids.py index e5844a71e48..e295495f1c7 100644 --- a/lib/galaxy/web/framework/helpers/grids.py +++ b/lib/galaxy/web/framework/helpers/grids.py @@ -626,6 +626,13 @@ class DeletedColumn( GridColumn ): args = { self.key: val } accepted_filters.append( GridColumnFilter( label, args) ) return accepted_filters + def filter( self, trans, user, query, column_filter ): + """Modify query to filter self.model_class by state.""" + if column_filter == "All": + pass + elif column_filter in [ "True", "False" ]: + query = query.filter( self.model_class.deleted == ( column_filter == "True" ) ) + return query class StateColumn( GridColumn ): """ @@ -702,7 +709,8 @@ class SharingStatusColumn( GridColumn ): class GridOperation( object ): def __init__( self, label, key=None, condition=None, allow_multiple=True, allow_popup=True, - target=None, url_args=None, async_compatible=False, confirm=None ): + target=None, url_args=None, async_compatible=False, confirm=None, + global_operation=None ): self.label = label self.key = key self.allow_multiple = allow_multiple @@ -713,6 +721,11 @@ class GridOperation( object ): self.async_compatible = async_compatible # if 'confirm' is set, then ask before completing the operation self.confirm = confirm + # specify a general operation that acts on the full grid + # this should be a function returning a dictionary with parameters + # to pass to the URL, similar to GridColumn links: + # global_operation=(lambda: dict(operation="download") + self.global_operation = global_operation def get_url_args( self, item ): if self.url_args: temp = dict( self.url_args ) diff --git a/lib/galaxy/web/params.py b/lib/galaxy/web/params.py new file mode 100644 index 00000000000..a9a4ea0ac31 --- /dev/null +++ b/lib/galaxy/web/params.py @@ -0,0 +1,29 @@ +""" +Mixins for parsing web form and API parameters +""" +from galaxy import util + +class BaseParamParser( object ): + def get_params( self, kwargs ): + params = util.Params( kwargs ) + # set defaults if unset + updates = dict( webapp = params.get( 'webapp', 'galaxy' ), + message = util.restore_text( params.get( 'message', '' ) ), + status = util.restore_text( params.get( 'status', 'done' ) ) ) + params.update( updates ) + return params + +class QuotaParamParser( BaseParamParser ): + def get_quota_params( self, kwargs ): + params = self.get_params( kwargs ) + updates = dict( name = util.restore_text( params.get( 'name', '' ) ), + description = util.restore_text( params.get( 'description', '' ) ), + amount = util.restore_text( params.get( 'amount', '' ).strip() ), + operation = params.get( 'operation', '' ), + default = params.get( 'default', '' ), + in_users = util.listify( params.get( 'in_users', [] ) ), + out_users = util.listify( params.get( 'out_users', [] ) ), + in_groups = util.listify( params.get( 'in_groups', [] ) ), + out_groups = util.listify( params.get( 'out_groups', [] ) ) ) + params.update( updates ) + return params diff --git a/lib/galaxy/web/security/__init__.py b/lib/galaxy/web/security/__init__.py index bfcc03d0436..a1a292a6779 100644 --- a/lib/galaxy/web/security/__init__.py +++ b/lib/galaxy/web/security/__init__.py @@ -43,8 +43,6 @@ class SecurityHelper( object ): return self.id_cipher.encrypt( s ).encode( 'hex' ) def decode_id( self, obj_id ): return int( self.id_cipher.decrypt( obj_id.decode( 'hex' ) ).lstrip( "!" ) ) - def decode_string_id( self, obj_id ): - return self.id_cipher.decrypt( obj_id.decode( 'hex' ) ).lstrip( "!" ) def encode_guid( self, session_key ): # Session keys are strings # Pad to a multiple of 8 with leading "!" @@ -57,4 +55,3 @@ class SecurityHelper( object ): def get_new_guid( self ): # Generate a unique, high entropy 128 bit random number return get_random_bytes( 16 ) - diff --git a/lib/galaxy/webapps/community/app.py b/lib/galaxy/webapps/community/app.py index 68cbf54fef0..3f622002751 100644 --- a/lib/galaxy/webapps/community/app.py +++ b/lib/galaxy/webapps/community/app.py @@ -1,4 +1,7 @@ import sys, config +import galaxy.tools.data +import galaxy.quota +import galaxy.datatypes.registry import galaxy.webapps.community.model from galaxy.web import security from galaxy.tags.tag_handler import CommunityTagHandler @@ -11,14 +14,8 @@ class UniverseApplication( object ): self.config = config.Configuration( **kwargs ) self.config.check() config.configure_logging( self.config ) - if self.config.enable_next_gen_tool_shed: - # We don't need a datatypes_registry since we have no datatypes - pass - else: - import galaxy.webapps.community.datatypes - # Set up datatypes registry - self.datatypes_registry = galaxy.webapps.community.datatypes.Registry( self.config.root, self.config.datatypes_config ) - galaxy.model.set_datatypes_registry( self.datatypes_registry ) + # Set up datatypes registry + self.datatypes_registry = galaxy.datatypes.registry.Registry( self.config.root, self.config.datatypes_config ) # Determine the database url if self.config.database_connection: db_url = self.config.database_connection @@ -29,15 +26,17 @@ class UniverseApplication( object ): create_or_verify_database( db_url, self.config.database_engine_options ) # Setup the database engine and ORM from galaxy.webapps.community.model import mapping - self.model = mapping.init( self.config.enable_next_gen_tool_shed, - self.config.file_path, + self.model = mapping.init( self.config.file_path, db_url, self.config.database_engine_options ) # Security helper self.security = security.SecurityHelper( id_secret=self.config.id_secret ) # Tag handler self.tag_handler = CommunityTagHandler() + # Tool data tables + self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( self.config.tool_data_table_config_path ) # Load security policy self.security_agent = self.model.security_agent + self.quota_agent = galaxy.quota.NoQuotaAgent( self.model ) def shutdown( self ): pass diff --git a/lib/galaxy/webapps/community/buildapp.py b/lib/galaxy/webapps/community/buildapp.py index 8e6bca2d295..354509d326f 100644 --- a/lib/galaxy/webapps/community/buildapp.py +++ b/lib/galaxy/webapps/community/buildapp.py @@ -20,23 +20,17 @@ from galaxy.webapps.community.framework.middleware import hg log = logging.getLogger( __name__ ) -def add_controllers( webapp, app ): +def add_ui_controllers( webapp, app ): """ Search for controllers in the 'galaxy.webapps.controllers' module and add them to the webapp. """ - from galaxy.web.base.controller import BaseController + from galaxy.web.base.controller import BaseUIController from galaxy.web.base.controller import ControllerUnavailable import galaxy.webapps.community.controllers controller_dir = galaxy.webapps.community.controllers.__path__[0] for fname in os.listdir( controller_dir ): if not fname.startswith( "_" ) and fname.endswith( ".py" ): - if app.config.enable_next_gen_tool_shed and fname.startswith( 'tool_upload' ): - # The tool_upload controller is for the old version of the tool shed - continue - if not app.config.enable_next_gen_tool_shed and fname.startswith( 'upload' ): - # The upload controller is for the next gen tool shed - continue name = fname[:-3] module_name = "galaxy.webapps.community.controllers." + name module = __import__( module_name ) @@ -45,8 +39,8 @@ def add_controllers( webapp, app ): # Look for a controller inside the modules for key in dir( module ): T = getattr( module, key ) - if isclass( T ) and T is not BaseController and issubclass( T, BaseController ): - webapp.add_controller( name, T( app ) ) + if isclass( T ) and T is not BaseUIController and issubclass( T, BaseUIController ): + webapp.add_ui_controller( name, T( app ) ) import galaxy.web.controllers controller_dir = galaxy.web.controllers.__path__[0] for fname in os.listdir( controller_dir ): @@ -60,8 +54,8 @@ def add_controllers( webapp, app ): # Look for a controller inside the modules for key in dir( module ): T = getattr( module, key ) - if isclass( T ) and T is not BaseController and issubclass( T, BaseController ): - webapp.add_controller( name, T( app ) ) + if isclass( T ) and T is not BaseUIController and issubclass( T, BaseUIController ): + webapp.add_ui_controller( name, T( app ) ) def app_factory( global_conf, **kwargs ): """Return a wsgi application serving the root object""" @@ -79,13 +73,10 @@ def app_factory( global_conf, **kwargs ): atexit.register( app.shutdown ) # Create the universe WSGI application webapp = galaxy.web.framework.WebApplication( app, session_cookie='galaxycommunitysession' ) - add_controllers( webapp, app ) + add_ui_controllers( webapp, app ) webapp.add_route( '/:controller/:action', action='index' ) - if app.config.enable_next_gen_tool_shed: - webapp.add_route( '/:action', controller='repository', action='index' ) - webapp.add_route( '/repos/*path_info', controller='hg', action='handle_request', path_info='/' ) - else: - webapp.add_route( '/:action', controller='tool', action='index' ) + webapp.add_route( '/:action', controller='repository', action='index' ) + webapp.add_route( '/repos/*path_info', controller='hg', action='handle_request', path_info='/' ) webapp.finalize_config() # Wrap the webapp in some useful middleware if kwargs.get( 'middleware', True ): diff --git a/lib/galaxy/webapps/community/config.py b/lib/galaxy/webapps/community/config.py index bd059d2108d..574aad0cf2c 100644 --- a/lib/galaxy/webapps/community/config.py +++ b/lib/galaxy/webapps/community/config.py @@ -39,15 +39,29 @@ class Configuration( object ): self.file_path = resolve_path( kwargs.get( "file_path", "database/files" ), self.root ) self.new_file_path = resolve_path( kwargs.get( "new_file_path", "database/tmp" ), self.root ) self.cookie_path = kwargs.get( "cookie_path", "/" ) + # web API + self.enable_api = string_as_bool( kwargs.get( 'enable_api', False ) ) + self.datatypes_config = kwargs.get( 'datatypes_config_file', 'datatypes_conf.xml' ) self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root ) self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!" ) + # Tool stuff + self.tool_secret = kwargs.get( "tool_secret", "" ) + self.tool_data_path = resolve_path( kwargs.get( "tool_data_path", "tool-data" ), os.getcwd() ) + self.tool_data_table_config_path = resolve_path( kwargs.get( 'tool_data_table_config_path', 'tool_data_table_conf.xml' ), self.root ) + self.ftp_upload_dir = kwargs.get( 'ftp_upload_dir', None ) + # Location for dependencies + if 'tool_dependency_dir' in kwargs: + self.tool_dependency_dir = resolve_path( kwargs.get( "tool_dependency_dir" ), self.root ) + self.use_tool_dependencies = True + else: + self.tool_dependency_dir = None + self.use_tool_dependencies = False self.use_remote_user = string_as_bool( kwargs.get( "use_remote_user", "False" ) ) self.remote_user_maildomain = kwargs.get( "remote_user_maildomain", None ) self.remote_user_logout_href = kwargs.get( "remote_user_logout_href", None ) self.require_login = string_as_bool( kwargs.get( "require_login", "False" ) ) self.allow_user_creation = string_as_bool( kwargs.get( "allow_user_creation", "True" ) ) self.enable_openid = string_as_bool( kwargs.get( 'enable_openid', False ) ) - self.enable_next_gen_tool_shed = string_as_bool( kwargs.get( 'enable_next_gen_tool_shed', False ) ) self.template_path = resolve_path( kwargs.get( "template_path", "templates" ), self.root ) self.template_cache = resolve_path( kwargs.get( "template_cache_path", "database/compiled_templates/community" ), self.root ) self.admin_users = kwargs.get( "admin_users", "" ) @@ -55,15 +69,19 @@ class Configuration( object ): self.mailing_join_addr = kwargs.get('mailing_join_addr',"galaxy-user-join@bx.psu.edu") self.error_email_to = kwargs.get( 'error_email_to', None ) self.smtp_server = kwargs.get( 'smtp_server', None ) + self.smtp_username = kwargs.get( 'smtp_username', None ) + self.smtp_password = kwargs.get( 'smtp_password', None ) + self.start_job_runners = kwargs.get( 'start_job_runners', None ) + self.email_from = kwargs.get( 'email_from', None ) + self.nginx_upload_path = kwargs.get( 'nginx_upload_path', False ) self.log_actions = string_as_bool( kwargs.get( 'log_actions', 'False' ) ) self.brand = kwargs.get( 'brand', None ) - self.wiki_url = kwargs.get( 'wiki_url', 'http://bitbucket.org/galaxy/galaxy-central/wiki/Home' ) - self.bugs_email = kwargs.get( 'bugs_email', None ) + self.support_url = kwargs.get( 'support_url', 'http://wiki.g2.bx.psu.edu/Support' ) + self.wiki_url = kwargs.get( 'wiki_url', 'http://wiki.g2.bx.psu.edu/FrontPage' ) self.blog_url = kwargs.get( 'blog_url', None ) self.screencasts_url = kwargs.get( 'screencasts_url', None ) self.log_events = False self.cloud_controller_instance = False - self.datatypes_config = kwargs.get( 'datatypes_config_file', 'community_datatypes_conf.xml' ) # Proxy features self.apache_xsendfile = kwargs.get( 'apache_xsendfile', False ) self.nginx_x_accel_redirect_base = kwargs.get( 'nginx_x_accel_redirect_base', False ) diff --git a/lib/galaxy/webapps/community/controllers/admin.py b/lib/galaxy/webapps/community/controllers/admin.py index 069ae5ef15b..040f596eaf1 100644 --- a/lib/galaxy/webapps/community/controllers/admin.py +++ b/lib/galaxy/webapps/community/controllers/admin.py @@ -2,14 +2,17 @@ from galaxy.web.base.controller import * from galaxy.webapps.community import model from galaxy.model.orm import * from galaxy.web.framework.helpers import time_ago, iff, grids -from common import ToolListGrid, CategoryListGrid, get_category, get_event, get_tool, get_versions -from repository import RepositoryListGrid, RepositoryCategoryListGrid +from galaxy.util import inflector +from common import * +from repository import RepositoryListGrid, CategoryListGrid +from mercurial import hg import logging + log = logging.getLogger( __name__ ) class UserListGrid( grids.Grid ): # TODO: move this to an admin_common controller since it is virtually the same - # in the galaxy webapp. NOTE the additional ToolsColumn in this grid though... + # in the galaxy webapp. class UserLoginColumn( grids.TextColumn ): def get_value( self, trans, grid, user ): return user.email @@ -282,164 +285,155 @@ class GroupListGrid( grids.Grid ): preserve_state = False use_paging = True -class AdminToolListGrid( ToolListGrid ): - class StateColumn( grids.TextColumn ): - def get_value( self, trans, grid, tool ): - state = tool.state - if state == 'approved': - state_color = 'ok' - elif state == 'rejected': - state_color = 'error' - elif state == 'archived': - state_color = 'upload' - else: - state_color = state - return '
    %s
    ' % ( state_color, state ) - class ToolStateColumn( grids.StateColumn ): - def filter( self, trans, user, query, column_filter ): - """Modify query to filter by state.""" - if column_filter == "All": - pass - elif column_filter in [ v for k, v in self.model_class.states.items() ]: - # Get all of the latest ToolEventAssociation ids - tea_ids = [ tea_id_tup[0] for tea_id_tup in trans.sa_session.query( func.max( model.ToolEventAssociation.table.c.id ) ) \ - .group_by( model.ToolEventAssociation.table.c.tool_id ) ] - # Get all of the Event ids associated with the latest ToolEventAssociation ids - event_ids = [ event_id_tup[0] for event_id_tup in trans.sa_session.query( model.ToolEventAssociation.table.c.event_id ) \ - .filter( model.ToolEventAssociation.table.c.id.in_( tea_ids ) ) ] - # Filter our query by state and event ids - return query.filter( and_( model.Event.table.c.state == column_filter, - model.Event.table.c.id.in_( event_ids ) ) ) - return query - - columns = [ col for col in ToolListGrid.columns ] - columns.append( - StateColumn( "Status", - model_class=model.Tool, - link=( lambda item: dict( operation="tools_by_state", id=item.id, webapp="community" ) ), - attach_popup=False ), - ) - columns.append( - # Columns that are valid for filtering but are not visible. - ToolStateColumn( "State", - key="state", - model_class=model.Tool, - visible=False, - filterable="advanced" ) - ) - operations = [ - grids.GridOperation( "Edit information", - condition=( lambda item: not item.deleted ), - allow_multiple=False, - url_args=dict( controller="common", action="edit_tool", cntrller="admin", webapp="community" ) ) - ] - -class AdminCategoryListGrid( CategoryListGrid ): - # Override standard filters - standard_filters = [ - grids.GridColumnFilter( "Active", args=dict( deleted=False ) ), - grids.GridColumnFilter( "Deleted", args=dict( deleted=True ) ), - grids.GridColumnFilter( "All", args=dict( deleted='All' ) ) - ] - class ManageCategoryListGrid( CategoryListGrid ): columns = [ col for col in CategoryListGrid.columns ] # Override the NameColumn to include an Edit link columns[ 0 ] = CategoryListGrid.NameColumn( "Name", - key="name", + key="Category.name", link=( lambda item: dict( operation="Edit", id=item.id, webapp="community" ) ), model_class=model.Category, - attach_popup=False, - filterable="advanced" ) + attach_popup=False ) global_actions = [ grids.GridAction( "Add new category", dict( controller='admin', action='manage_categories', operation='create', webapp="community" ) ) ] - operations = [ grids.GridOperation( "Delete", - condition=( lambda item: not item.deleted ), - allow_multiple=True, - url_args=dict( webapp="community", action="mark_category_deleted" ) ), - grids.GridOperation( "Undelete", - condition=( lambda item: item.deleted ), - allow_multiple=True, - url_args=dict( webapp="community", action="undelete_category" ) ), - grids.GridOperation( "Purge", - condition=( lambda item: item.deleted ), - allow_multiple=True, - url_args=dict( webapp="community", action="purge_category" ) ) ] -class AdminController( BaseController, Admin ): +class AdminRepositoryListGrid( RepositoryListGrid ): + operations = [ operation for operation in RepositoryListGrid.operations ] + operations.append( grids.GridOperation( "Delete", + allow_multiple=False, + condition=( lambda item: not item.deleted ), + async_compatible=False ) ) + operations.append( grids.GridOperation( "Undelete", + allow_multiple=False, + condition=( lambda item: item.deleted ), + async_compatible=False ) ) + standard_filters = [] + +class RepositoryMetadataListGrid( grids.Grid ): + class IdColumn( grids.IntegerColumn ): + def get_value( self, trans, grid, repository_metadata ): + return repository_metadata.id + class NameColumn( grids.TextColumn ): + def get_value( self, trans, grid, repository_metadata ): + return repository_metadata.repository.name + class RevisionColumn( grids.TextColumn ): + def get_value( self, trans, grid, repository_metadata ): + repository = repository_metadata.repository + repo = hg.repository( get_configured_ui(), repository.repo_path ) + ctx = get_changectx_for_changeset( trans, repo, repository_metadata.changeset_revision ) + return "%s:%s" % ( str( ctx.rev() ), repository_metadata.changeset_revision ) + class MetadataColumn( grids.TextColumn ): + def get_value( self, trans, grid, repository_metadata ): + metadata_str = '' + if repository_metadata: + metadata = repository_metadata.metadata + if metadata: + if 'tools' in metadata: + metadata_str += 'Tools:
    ' + for tool_metadata_dict in metadata[ 'tools' ]: + metadata_str += '%s %s
    ' % \ + ( tool_metadata_dict[ 'id' ], tool_metadata_dict[ 'version' ] ) + if 'workflows' in metadata: + metadata_str += 'Workflows:
    ' + for workflow_metadata_dict in metadata[ 'workflows' ]: + metadata_str += '%s %s
    ' % \ + ( workflow_metadata_dict[ 'name' ], workflow_metadata_dict[ 'format-version' ] ) + return metadata_str + class MaliciousColumn( grids.BooleanColumn ): + def get_value( self, trans, grid, repository_metadata ): + return repository_metadata.malicious + # Grid definition + title = "Repository Metadata" + model_class = model.RepositoryMetadata + template='/webapps/community/repository/grid.mako' + default_sort_key = "name" + columns = [ + IdColumn( "Id", + visible=False, + attach_popup=False ), + NameColumn( "Name", + key="name", + model_class=model.Repository, + link=( lambda item: dict( operation="view_or_manage_repository_revision", + id=item.id, + webapp="community" ) ), + attach_popup=True ), + RevisionColumn( "Revision", + attach_popup=False ), + MetadataColumn( "Metadata", + attach_popup=False ), + MaliciousColumn( "Malicious", + attach_popup=False ) + ] + operations = [ grids.GridOperation( "Delete", + allow_multiple=False, + allow_popup=True, + async_compatible=False, + confirm="Repository metadata records cannot be recovered after they are deleted. Click OK to delete the selected items." ) ] + standard_filters = [] + default_filter = {} + num_rows_per_page = 50 + preserve_state = False + use_paging = True + def build_initial_query( self, trans, **kwd ): + return trans.sa_session.query( self.model_class ) \ + .join( model.Repository.table ) + +class AdminController( BaseUIController, Admin ): user_list_grid = UserListGrid() role_list_grid = RoleListGrid() group_list_grid = GroupListGrid() manage_category_list_grid = ManageCategoryListGrid() - tool_category_list_grid = AdminCategoryListGrid() - tool_list_grid = AdminToolListGrid() - repository_list_grid = RepositoryListGrid() - repository_category_list_grid = RepositoryCategoryListGrid() + repository_list_grid = AdminRepositoryListGrid() + repository_metadata_list_grid = RepositoryMetadataListGrid() @web.expose @web.require_admin - def browse_tools( self, trans, **kwd ): - # We add params to the keyword dict in this method in order to rename the param - # with an "f-" prefix, simulating filtering by clicking a search link. We have - # to take this approach because the "-" character is illegal in HTTP requests. + def browse_repository_metadata( self, trans, **kwd ): if 'operation' in kwd: - operation = kwd['operation'].lower() - if operation == "edit_tool": - return trans.response.send_redirect( web.url_for( controller='common', - action='edit_tool', - cntrller='admin', + operation = kwd[ 'operation' ].lower() + if operation == "delete": + return self.delete_repository_metadata( trans, **kwd ) + if operation == "view_or_manage_repository_revision": + # The received id is a RepositoryMetadata object id, so we need to get the + # associated Repository and redirect to view_or_manage_repository with the + # changeset_revision. + repository_metadata = get_repository_metadata_by_id( trans, kwd[ 'id' ] ) + repository = repository_metadata.repository + kwd[ 'id' ] = trans.security.encode_id( repository.id ) + kwd[ 'changeset_revision' ] = repository_metadata.changeset_revision + kwd[ 'operation' ] = 'view_or_manage_repository' + return trans.response.send_redirect( web.url_for( controller='repository', + action='browse_repositories', **kwd ) ) - elif operation == "view_tool": - return trans.response.send_redirect( web.url_for( controller='common', - action='view_tool', - cntrller='admin', - **kwd ) ) - elif operation == 'tool_history': - return trans.response.send_redirect( web.url_for( controller='common', - cntrller='admin', - action='events', - **kwd ) ) - elif operation == "tools_by_user": - # Eliminate the current filters if any exist. - for k, v in kwd.items(): - if k.startswith( 'f-' ): - del kwd[ k ] - if 'user_id' in kwd: - user = get_user( trans, kwd[ 'user_id' ] ) - kwd[ 'f-email' ] = user.email - del kwd[ 'user_id' ] - else: - # The received id is the tool id, so we need to get the id of the user - # that uploaded the tool. - tool_id = kwd.get( 'id', None ) - tool = get_tool( trans, tool_id ) - kwd[ 'f-email' ] = tool.user.email - elif operation == "tools_by_state": - # Eliminate the current filters if any exist. - for k, v in kwd.items(): - if k.startswith( 'f-' ): - del kwd[ k ] - if 'state' in kwd: - # Called from the Admin menu - kwd[ 'f-state' ] = kwd[ 'state' ] - else: - # Called from the ToolStateColumn link - tool_id = kwd.get( 'id', None ) - tool = get_tool( trans, tool_id ) - kwd[ 'f-state' ] = tool.state - elif operation == "tools_by_category": - # Eliminate the current filters if any exist. - for k, v in kwd.items(): - if k.startswith( 'f-' ): - del kwd[ k ] - category_id = kwd.get( 'id', None ) - category = get_category( trans, category_id ) - kwd[ 'f-Category.name' ] = category.name # Render the list view - return self.tool_list_grid( trans, **kwd ) + return self.repository_metadata_list_grid( trans, **kwd ) + @web.expose + @web.require_admin + def delete_repository_metadata( self, trans, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + id = kwd.get( 'id', None ) + if id: + ids = util.listify( id ) + count = 0 + for repository_metadata_id in ids: + repository_metadata = get_repository_metadata_by_id( trans, repository_metadata_id ) + trans.sa_session.delete( repository_metadata ) + trans.sa_session.flush() + count += 1 + if count: + message = "Deleted %d repository metadata %s" % ( count, inflector.cond_plural( len( ids ), "record" ) ) + else: + message = "No repository metadata ids received for deleting." + status = 'error' + trans.response.send_redirect( web.url_for( controller='admin', + action='browse_repository_metadata', + message=util.sanitize_text( message ), + status=status ) ) @web.expose @web.require_admin def browse_repositories( self, trans, **kwd ): @@ -448,9 +442,9 @@ class AdminController( BaseController, Admin ): # to take this approach because the "-" character is illegal in HTTP requests. if 'operation' in kwd: operation = kwd['operation'].lower() - if operation == "view_repository": + if operation == "view_or_manage_repository": return trans.response.send_redirect( web.url_for( controller='repository', - action='view_repository', + action='browse_repositories', **kwd ) ) elif operation == "edit_repository": return trans.response.send_redirect( web.url_for( controller='repository', @@ -479,38 +473,105 @@ class AdminController( BaseController, Admin ): category_id = kwd.get( 'id', None ) category = get_category( trans, category_id ) kwd[ 'f-Category.name' ] = category.name + elif operation == "receive email alerts": + if kwd[ 'id' ]: + return trans.response.send_redirect( web.url_for( controller='repository', + action='set_email_alerts', + **kwd ) ) + else: + del kwd[ 'operation' ] + elif operation == 'delete': + return self.mark_repository_deleted( trans, **kwd ) + elif operation == "undelete": + return self.undelete_repository( trans, **kwd ) + # The changeset_revision_select_field in the RepositoryListGrid performs a refresh_on_change + # which sends in request parameters like changeset_revison_1, changeset_revision_2, etc. One + # of the many select fields on the grid performed the refresh_on_change, so we loop through + # all of the received values to see which value is not the repository tip. If we find it, we + # know the refresh_on_change occurred, and we have the necessary repository id and change set + # revision to pass on. + for k, v in kwd.items(): + changset_revision_str = 'changeset_revision_' + if k.startswith( changset_revision_str ): + repository_id = trans.security.encode_id( int( k.lstrip( changset_revision_str ) ) ) + repository = get_repository( trans, repository_id ) + if repository.tip != v: + return trans.response.send_redirect( web.url_for( controller='repository', + action='browse_repositories', + operation='view_or_manage_repository', + id=trans.security.encode_id( repository.id ), + changeset_revision=v ) ) # Render the list view return self.repository_list_grid( trans, **kwd ) @web.expose @web.require_admin - def browse_categories( self, trans, **kwd ): - if 'operation' in kwd: - operation = kwd['operation'].lower() - if trans.app.config.enable_next_gen_tool_shed: - if operation in [ "repositories_by_category", "repositories_by_user" ]: - # Eliminate the current filters if any exist. - for k, v in kwd.items(): - if k.startswith( 'f-' ): - del kwd[ k ] - return trans.response.send_redirect( web.url_for( controller='admin', - action='browse_repositories', - **kwd ) ) + def mark_repository_deleted( self, trans, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + id = kwd.get( 'id', None ) + if id: + ids = util.listify( id ) + count = 0 + deleted_repositories = "" + for repository_id in ids: + repository = get_repository( trans, repository_id ) + if not repository.deleted: + repository.deleted = True + trans.sa_session.add( repository ) + trans.sa_session.flush() + count += 1 + deleted_repositories += " %s " % repository.name + if count: + message = "Deleted %d %s: %s" % ( count, inflector.cond_plural( len( ids ), "repository" ), deleted_repositories ) else: - if operation in [ "tools_by_category", "tools_by_state", "tools_by_user" ]: - # Eliminate the current filters if any exist. - for k, v in kwd.items(): - if k.startswith( 'f-' ): - del kwd[ k ] - return trans.response.send_redirect( web.url_for( controller='admin', - action='browse_tools', - **kwd ) ) - if trans.app.config.enable_next_gen_tool_shed: - return self.repository_category_list_grid( trans, **kwd ) + message = "All selected repositories were already marked deleted." else: - return self.tool_category_list_grid( trans, **kwd ) + message = "No repository ids received for deleting." + status = 'error' + trans.response.send_redirect( web.url_for( controller='admin', + action='browse_repositories', + message=util.sanitize_text( message ), + status=status ) ) + @web.expose + @web.require_admin + def undelete_repository( self, trans, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + id = kwd.get( 'id', None ) + if id: + ids = util.listify( id ) + count = 0 + undeleted_repositories = "" + for repository_id in ids: + repository = get_repository( trans, repository_id ) + if repository.deleted: + repository.deleted = False + trans.sa_session.add( repository ) + trans.sa_session.flush() + count += 1 + undeleted_repositories += " %s" % repository.name + if count: + message = "Undeleted %d %s: %s" % ( count, inflector.cond_plural( count, "repository" ), undeleted_repositories ) + else: + message = "No selected repositories were marked deleted, so they could not be undeleted." + else: + message = "No repository ids received for undeleting." + status = 'error' + trans.response.send_redirect( web.url_for( controller='admin', + action='browse_repositories', + message=util.sanitize_text( message ), + status='done' ) ) @web.expose @web.require_admin def manage_categories( self, trans, **kwd ): + if 'f-free-text-search' in kwd: + # Trick to enable searching repository name, description from the CategoryListGrid. + # What we've done is rendered the search box for the RepositoryListGrid on the grid.mako + # template for the CategoryListGrid. See ~/templates/webapps/community/category/grid.mako. + # Since we are searching repositories and not categories, redirect to browse_repositories(). + return self.browse_repositories( trans, **kwd ) if 'operation' in kwd: operation = kwd['operation'].lower() if operation == "create": @@ -573,140 +634,6 @@ class AdminController( BaseController, Admin ): status=status ) @web.expose @web.require_admin - def set_tool_state( self, trans, state, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - comments = util.restore_text( params.get( 'comments', '' ) ) - id = params.get( 'id', None ) - if not id: - message = "No tool id received for setting status" - status = 'error' - else: - tool = get_tool( trans, id ) - if state == trans.app.model.Tool.states.APPROVED: - # If we're approving a tool, all previously approved versions must be set to archived - for version in get_versions( tool ): - # TODO: get latest approved version instead of all versions - if version != tool and version.is_approved: - # Create an event with state ARCHIVED for the previously approved version of this tool - self.__create_tool_event( trans, - version, - trans.app.model.Tool.states.ARCHIVED ) - # Create an event with state APPROVED for this tool - self.__create_tool_event( trans, tool, state, comments ) - elif state == trans.app.model.Tool.states.REJECTED: - # If we're rejecting a tool, comments about why are necessary. - return trans.fill_template( '/webapps/community/admin/reject_tool.mako', - tool=tool, - cntrller='admin' ) - message = "State of tool '%s' is now %s" % ( tool.name, state ) - trans.response.send_redirect( web.url_for( controller='admin', - action='browse_tools', - message=message, - status=status ) ) - @web.expose - @web.require_admin - def reject_tool( self, trans, **kwd ): - params = util.Params( kwd ) - if params.get( 'cancel_reject_button', False ): - # Fix up the keyword dict to include params to view the current tool - # since that is the page from which we originated. - del kwd[ 'cancel_reject_button' ] - del kwd[ 'comments' ] - kwd[ 'webapp' ] = 'community' - kwd[ 'operation' ] = 'view_tool' - message = 'Tool rejection cancelled' - status = 'done' - return trans.response.send_redirect( web.url_for( controller='admin', - action='browse_tools', - message=message, - status=status, - **kwd ) ) - id = params.get( 'id', None ) - if not id: - return trans.response.send_redirect( web.url_for( controller=cntrller, - action='browse_tools', - message='No tool id received for rejecting', - status='error' ) ) - tool = get_tool( trans, id ) - if not trans.app.security_agent.can_approve_or_reject( trans.user, trans.user_is_admin(), 'admin', tool ): - return trans.response.send_redirect( web.url_for( controller='admin', - action='browse_tools', - message='You are not allowed to reject this tool', - status='error' ) ) - # Comments are required when rejecting a tool. - comments = util.restore_text( params.get( 'comments', '' ) ) - if not comments: - message = 'The reason for rejection is required when rejecting a tool.' - return trans.fill_template( '/webapps/community/admin/reject_tool.mako', - tool=tool, - cntrller='admin', - message=message, - status='error' ) - # Create an event with state REJECTED for this tool - self.__create_tool_event( trans, tool, trans.app.model.Tool.states.REJECTED, comments ) - message = 'The tool "%s" has been rejected.' % tool.name - return trans.response.send_redirect( web.url_for( controller='admin', - action='browse_tools', - operation='tools_by_state', - state='rejected', - message=message, - status='done' ) ) - def __create_tool_event( self, trans, tool, state, comments='' ): - event = trans.model.Event( state, comments ) - # Flush so we can get an id - trans.sa_session.add( event ) - trans.sa_session.flush() - tea = trans.model.ToolEventAssociation( tool, event ) - trans.sa_session.add( tea ) - trans.sa_session.flush() - @web.expose - @web.require_admin - def purge_tool( self, trans, **kwd ): - # This method completely removes a tool record and all associated foreign key rows - # from the database, so it must be used carefully. - # This method should only be called for a tool that has previously been deleted. - # Purging a deleted tool deletes all of the following from the database: - # - ToolCategoryAssociations - # - ToolEventAssociations and associated Events - # TODO: when we add tagging for tools, we'll have to purge them as well - params = util.Params( kwd ) - id = kwd.get( 'id', None ) - if not id: - message = "No tool ids received for purging" - trans.response.send_redirect( web.url_for( controller='admin', - action='browse_tools', - message=util.sanitize_text( message ), - status='error' ) ) - ids = util.listify( id ) - message = "Purged %d tools: " % len( ids ) - for tool_id in ids: - tool = get_tool( trans, tool_id ) - message += " %s " % tool.name - if not tool.deleted: - message = "Tool '%s' has not been deleted, so it cannot be purged." % tool.name - trans.response.send_redirect( web.url_for( controller='admin', - action='browse_tools', - message=util.sanitize_text( message ), - status='error' ) ) - # Delete ToolCategoryAssociations - for tca in tool.categories: - trans.sa_session.delete( tca ) - # Delete ToolEventAssociations and associated events - for tea in tool.events: - event = tea.event - trans.sa_session.delete( event ) - trans.sa_session.delete( tea ) - # Delete the tool - trans.sa_session.delete( tool ) - trans.sa_session.flush() - trans.response.send_redirect( web.url_for( controller='admin', - action='browse_tools', - message=util.sanitize_text( message ), - status='done' ) ) - @web.expose - @web.require_admin def edit_category( self, trans, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) @@ -747,22 +674,25 @@ class AdminController( BaseController, Admin ): @web.expose @web.require_admin def mark_category_deleted( self, trans, **kwd ): + # TODO: We should probably eliminate the Category.deleted column since it really makes no + # sense to mark a category as deleted (category names and descriptions can be changed instead). + # If we do this, and the following 2 methods can be eliminated. params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) id = kwd.get( 'id', None ) - if not id: - message = "No category ids received for deleting" - trans.response.send_redirect( web.url_for( controller='admin', - action='manage_categories', - message=message, - status='error' ) ) - ids = util.listify( id ) - message = "Deleted %d categories: " % len( ids ) - for category_id in ids: - category = get_category( trans, category_id ) - category.deleted = True - trans.sa_session.add( category ) - trans.sa_session.flush() - message += " %s " % category.name + if id: + ids = util.listify( id ) + message = "Deleted %d categories: " % len( ids ) + for category_id in ids: + category = get_category( trans, category_id ) + category.deleted = True + trans.sa_session.add( category ) + trans.sa_session.flush() + message += " %s " % category.name + else: + message = "No category ids received for deleting." + status = 'error' trans.response.send_redirect( web.url_for( controller='admin', action='manage_categories', message=util.sanitize_text( message ), @@ -771,30 +701,25 @@ class AdminController( BaseController, Admin ): @web.require_admin def undelete_category( self, trans, **kwd ): params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) id = kwd.get( 'id', None ) - if not id: - message = "No category ids received for undeleting" - trans.response.send_redirect( web.url_for( controller='admin', - action='manage_categories', - message=message, - status='error' ) ) - ids = util.listify( id ) - count = 0 - undeleted_categories = "" - for category_id in ids: - category = get_category( trans, category_id ) - if not category.deleted: - message = "Category '%s' has not been deleted, so it cannot be undeleted." % category.name - trans.response.send_redirect( web.url_for( controller='admin', - action='manage_categories', - message=util.sanitize_text( message ), - status='error' ) ) - category.deleted = False - trans.sa_session.add( category ) - trans.sa_session.flush() - count += 1 - undeleted_categories += " %s" % category.name - message = "Undeleted %d categories: %s" % ( count, undeleted_categories ) + if id: + ids = util.listify( id ) + count = 0 + undeleted_categories = "" + for category_id in ids: + category = get_category( trans, category_id ) + if category.deleted: + category.deleted = False + trans.sa_session.add( category ) + trans.sa_session.flush() + count += 1 + undeleted_categories += " %s" % category.name + message = "Undeleted %d categories: %s" % ( count, undeleted_categories ) + else: + message = "No category ids received for undeleting." + status = 'error' trans.response.send_redirect( web.url_for( controller='admin', action='manage_categories', message=util.sanitize_text( message ), @@ -804,38 +729,28 @@ class AdminController( BaseController, Admin ): def purge_category( self, trans, **kwd ): # This method should only be called for a Category that has previously been deleted. # Purging a deleted Category deletes all of the following from the database: - # If trans.app.config.enable_next_gen_tool_shed: # - RepoitoryCategoryAssociations where category_id == Category.id - # Otherwise: - # - ToolCategoryAssociations where category_id == Category.id params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) id = kwd.get( 'id', None ) - if not id: - message = "No category ids received for purging" - trans.response.send_redirect( web.url_for( controller='admin', - action='manage_categories', - message=util.sanitize_text( message ), - status='error' ) ) - ids = util.listify( id ) - message = "Purged %d categories: " % len( ids ) - for category_id in ids: - category = get_category( trans, category_id ) - if not category.deleted: - message = "Category '%s' has not been deleted, so it cannot be purged." % category.name - trans.response.send_redirect( web.url_for( controller='admin', - action='manage_categories', - message=util.sanitize_text( message ), - status='error' ) ) - if trans.app.config.enable_next_gen_tool_shed: - # Delete RepositoryCategoryAssociations - for rca in category.repositories: - trans.sa_session.delete( rca ) - else: - # Delete ToolCategoryAssociations - for tca in category.tools: - trans.sa_session.delete( tca ) - trans.sa_session.flush() - message += " %s " % category.name + if id: + ids = util.listify( id ) + count = 0 + purged_categories = "" + message = "Purged %d categories: " % len( ids ) + for category_id in ids: + category = get_category( trans, category_id ) + if category.deleted: + # Delete RepositoryCategoryAssociations + for rca in category.repositories: + trans.sa_session.delete( rca ) + trans.sa_session.flush() + purged_categories += " %s " % category.name + message = "Purged %d categories: %s" % ( count, purged_categories ) + else: + message = "No category ids received for purging." + status = 'error' trans.response.send_redirect( web.url_for( controller='admin', action='manage_categories', message=util.sanitize_text( message ), diff --git a/lib/galaxy/webapps/community/controllers/common.py b/lib/galaxy/webapps/community/controllers/common.py index 8691531b63d..0cbda2eed7d 100644 --- a/lib/galaxy/webapps/community/controllers/common.py +++ b/lib/galaxy/webapps/community/controllers/common.py @@ -1,16 +1,55 @@ -import tarfile +import os, string, socket, logging +from time import strftime +from datetime import * +from galaxy.tools import * +from galaxy.util.json import from_json_string, to_json_string from galaxy.web.base.controller import * from galaxy.webapps.community import model from galaxy.model.orm import * -from galaxy.web.framework.helpers import time_ago, iff, grids -from galaxy.web.form_builder import SelectField from galaxy.model.item_attrs import UsesItemRatings -import logging +from mercurial import hg, ui, commands + log = logging.getLogger( __name__ ) +email_alert_template = """ +GALAXY TOOL SHED REPOSITORY UPDATE ALERT +----------------------------------------------------------------------------- +You received this alert because you registered to receive email whenever +changes were made to the repository named "${repository_name}". +----------------------------------------------------------------------------- + +Date of change: ${display_date} +Changed by: ${username} + +Revision: ${revision} +Change description: +${description} + +----------------------------------------------------------------------------- +This change alert was sent from the Galaxy tool shed hosted on the server +"${host}" +""" + +contact_owner_template = """ +GALAXY TOOL SHED REPOSITORY MESSAGE +------------------------ + +The user '${username}' sent you the following message regarding your tool shed +repository named '${repository_name}'. You can respond by sending a reply to +the user's email address: ${email}. +----------------------------------------------------------------------------- +${message} +----------------------------------------------------------------------------- +This message was sent from the Galaxy Tool Shed instance hosted on the server +'${host}' +""" + # States for passing messages SUCCESS, INFO, WARNING, ERROR = "done", "info", "warning", "error" +malicious_error = " This changeset cannot be downloaded because it potentially produces malicious behavior or contains inappropriate content." +malicious_error_can_push = " Correct this changeset as soon as possible, it potentially produces malicious behavior or contains inappropriate content." + class ItemRatings( UsesItemRatings ): """Overrides rate_item method since we also allow for comments""" def rate_item( self, trans, user, item, rating, comment='' ): @@ -34,508 +73,8 @@ class ItemRatings( UsesItemRatings ): trans.sa_session.flush() return item_rating -class ToolListGrid( grids.Grid ): - class NameColumn( grids.TextColumn ): - def get_value( self, trans, grid, tool ): - return tool.name - class TypeColumn( grids.BooleanColumn ): - def get_value( self, trans, grid, tool ): - if tool.is_suite: - return 'Suite' - return 'Tool' - class VersionColumn( grids.TextColumn ): - def get_value( self, trans, grid, tool ): - return tool.version - class DescriptionColumn( grids.TextColumn ): - def get_value( self, trans, grid, tool ): - return tool.description - class CategoryColumn( grids.TextColumn ): - def get_value( self, trans, grid, tool ): - rval = '
      ' - if tool.categories: - for tca in tool.categories: - rval += '
    • %s
      • ' \ - % ( trans.security.encode_id( tca.category.id ), tca.category.name ) - else: - rval += '
    • not set
      • ' - rval += '
    ' - return rval - class ToolCategoryColumn( grids.GridColumn ): - def filter( self, trans, user, query, column_filter ): - """Modify query to filter by category.""" - if column_filter == "All": - pass - return query.filter( model.Category.name == column_filter ) - class UserColumn( grids.TextColumn ): - def get_value( self, trans, grid, tool ): - if tool.user: - return tool.user.username - return 'no user' - class EmailColumn( grids.TextColumn ): - def filter( self, trans, user, query, column_filter ): - if column_filter == 'All': - return query - return query.filter( and_( model.Tool.table.c.user_id == model.User.table.c.id, - model.User.table.c.email == column_filter ) ) - # Grid definition - title = "Tools" - model_class = model.Tool - template='/webapps/community/tool/grid.mako' - default_sort_key = "name" - columns = [ - NameColumn( "Name", - key="Tool.name", - link=( lambda item: dict( operation="view_tool", id=item.id, webapp="community" ) ), - attach_popup=False ), - DescriptionColumn( "Description", - key="description", - attach_popup=False ), - VersionColumn( "Version", - key="version", - attach_popup=False, - filterable="advanced" ), - CategoryColumn( "Category", - model_class=model.Category, - key="Category.name", - attach_popup=False ), - UserColumn( "Uploaded By", - model_class=model.User, - link=( lambda item: dict( operation="tools_by_user", id=item.id, webapp="community" ) ), - attach_popup=False, - key="username" ), - TypeColumn( "Type", - key="suite", - attach_popup=False ), - grids.CommunityRatingColumn( "Average Rating", - key="rating" ), - # Columns that are valid for filtering but are not visible. - EmailColumn( "Email", - model_class=model.User, - key="email", - visible=False ), - ToolCategoryColumn( "Category", - model_class=model.Category, - key="Category.name", - visible=False ) - ] - columns.append( grids.MulticolFilterColumn( "Search tool name, description, version", - cols_to_filter=[ columns[0], columns[1], columns[2] ], - key="free-text-search", - visible=False, - filterable="standard" ) ) - operations = [] - standard_filters = [] - default_filter = {} - num_rows_per_page = 50 - preserve_state = False - use_paging = True - def build_initial_query( self, trans, **kwd ): - return trans.sa_session.query( self.model_class ) \ - .join( model.User.table ) \ - .join( model.ToolEventAssociation.table ) \ - .join( model.Event.table ) \ - .outerjoin( model.ToolCategoryAssociation.table ) \ - .outerjoin( model.Category.table ) - -class CategoryListGrid( grids.Grid ): - class NameColumn( grids.TextColumn ): - def get_value( self, trans, grid, category ): - return category.name - class DescriptionColumn( grids.TextColumn ): - def get_value( self, trans, grid, category ): - return category.description - class ToolsColumn( grids.TextColumn ): - def get_value( self, trans, grid, category ): - if category.tools: - viewable_tools = 0 - for tca in category.tools: - viewable_tools += 1 - return viewable_tools - return 0 - - # Grid definition - webapp = "community" - title = "Categories" - model_class = model.Category - template='/webapps/community/category/grid.mako' - default_sort_key = "name" - columns = [ - NameColumn( "Name", - key="name", - link=( lambda item: dict( operation="tools_by_category", id=item.id, webapp="community" ) ), - attach_popup=False, - filterable="advanced" ), - DescriptionColumn( "Description", - key="description", - attach_popup=False, - filterable="advanced" ), - # Columns that are valid for filtering but are not visible. - grids.DeletedColumn( "Deleted", - key="deleted", - visible=False, - filterable="advanced" ), - ToolsColumn( "Tools", - model_class=model.Tool, - attach_popup=False ) - ] - columns.append( grids.MulticolFilterColumn( "Search category name, description", - cols_to_filter=[ columns[0], columns[1] ], - key="free-text-search", - visible=False, - filterable="standard" ) ) - - # Override these - global_actions = [] - operations = [] - standard_filters = [] - num_rows_per_page = 50 - preserve_state = False - use_paging = True - -class CommonController( BaseController, ItemRatings ): - @web.expose - def edit_tool( self, trans, cntrller, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - id = params.get( 'id', None ) - if not id: - return trans.response.send_redirect( web.url_for( controller=cntrller, - action='browse_tools', - cntrller=cntrller, - message='Select a tool to edit', - status='error' ) ) - tool = get_tool( trans, id ) - can_edit = trans.app.security_agent.can_edit( trans.user, trans.user_is_admin(), cntrller, tool ) - if not can_edit: - return trans.response.send_redirect( web.url_for( controller=cntrller, - action='browse_tools', - cntrller=cntrller, - message='You are not allowed to edit this tool', - status='error' ) ) - if params.get( 'edit_tool_button', False ): - if params.get( 'in_categories', False ): - in_categories = [ trans.sa_session.query( trans.app.model.Category ).get( x ) for x in util.listify( params.in_categories ) ] - trans.app.security_agent.set_entity_category_associations( tools=[ tool ], categories=in_categories ) - else: - # There must not be any categories associated with the tool - trans.app.security_agent.set_entity_category_associations( tools=[ tool ], categories=[] ) - user_description = util.restore_text( params.get( 'user_description', '' ) ) - if user_description: - tool.user_description = user_description - else: - tool.user_description = '' - trans.sa_session.add( tool ) - trans.sa_session.flush() - message = "Tool '%s' description and category associations have been saved" % tool.name - return trans.response.send_redirect( web.url_for( controller='common', - action='edit_tool', - cntrller=cntrller, - id=id, - message=message, - status='done' ) ) - elif params.get( 'approval_button', False ): - user_description = util.restore_text( params.get( 'user_description', '' ) ) - if user_description: - tool.user_description = user_description - if params.get( 'in_categories', False ): - in_categories = [ trans.sa_session.query( trans.app.model.Category ).get( x ) for x in util.listify( params.in_categories ) ] - trans.app.security_agent.set_entity_category_associations( tools=[ tool ], categories=in_categories ) - else: - # There must not be any categories associated with the tool - trans.app.security_agent.set_entity_category_associations( tools=[ tool ], categories=[] ) - trans.sa_session.add( tool ) - trans.sa_session.flush() - # Move the state from NEW to WAITING - event = trans.app.model.Event( state=trans.app.model.Tool.states.WAITING ) - tea = trans.app.model.ToolEventAssociation( tool, event ) - trans.sa_session.add_all( ( event, tea ) ) - trans.sa_session.flush() - message = "Tool '%s' has been submitted for approval and can no longer be modified" % ( tool.name ) - return trans.response.send_redirect( web.url_for( controller='common', - action='view_tool', - cntrller=cntrller, - id=id, - message=message, - status='done' ) ) - else: - # The user_description field is required when submitting for approval - message = 'A user description is required prior to approval.' - status = 'error' - in_categories = [] - out_categories = [] - for category in get_categories( trans ): - if category in [ x.category for x in tool.categories ]: - in_categories.append( ( category.id, category.name ) ) - else: - out_categories.append( ( category.id, category.name ) ) - if tool.is_rejected: - # Include the comments regarding the reason for rejection - reason_for_rejection = tool.latest_event.comment - else: - reason_for_rejection = '' - can_approve_or_reject = trans.app.security_agent.can_approve_or_reject( trans.user, trans.user_is_admin(), cntrller, tool ) - can_delete = trans.app.security_agent.can_delete( trans.user, trans.user_is_admin(), cntrller, tool ) - can_download = trans.app.security_agent.can_download( trans.user, trans.user_is_admin(), cntrller, tool ) - can_purge = trans.app.security_agent.can_purge( trans.user, trans.user_is_admin(), cntrller ) - can_upload_new_version = trans.app.security_agent.can_upload_new_version( trans.user, tool ) - can_view = trans.app.security_agent.can_view( trans.user, trans.user_is_admin(), cntrller, tool ) - return trans.fill_template( '/webapps/community/tool/edit_tool.mako', - cntrller=cntrller, - tool=tool, - id=id, - in_categories=in_categories, - out_categories=out_categories, - can_approve_or_reject=can_approve_or_reject, - can_delete=can_delete, - can_download=can_download, - can_edit=can_edit, - can_purge=can_purge, - can_upload_new_version=can_upload_new_version, - can_view=can_view, - reason_for_rejection=reason_for_rejection, - message=message, - status=status ) - @web.expose - def view_tool( self, trans, cntrller, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - id = params.get( 'id', None ) - if not id: - return trans.response.send_redirect( web.url_for( controller=cntrller, - action='browse_tools', - cntrller=cntrller, - message='Select a tool to view', - status='error' ) ) - tool = get_tool( trans, id ) - can_view = trans.app.security_agent.can_view( trans.user, trans.user_is_admin(), cntrller, tool ) - if not can_view: - return trans.response.send_redirect( web.url_for( controller=cntrller, - action='browse_tools', - cntrller=cntrller, - message='You are not allowed to view this tool', - status='error' ) ) - avg_rating, num_ratings = self.get_ave_item_rating_data( trans.sa_session, tool, webapp_model=trans.model ) - can_approve_or_reject = trans.app.security_agent.can_approve_or_reject( trans.user, trans.user_is_admin(), cntrller, tool ) - can_delete = trans.app.security_agent.can_delete( trans.user, trans.user_is_admin(), cntrller, tool ) - can_download = trans.app.security_agent.can_download( trans.user, trans.user_is_admin(), cntrller, tool ) - can_edit = trans.app.security_agent.can_edit( trans.user, trans.user_is_admin(), cntrller, tool ) - can_purge = trans.app.security_agent.can_purge( trans.user, trans.user_is_admin(), cntrller ) - can_rate = trans.app.security_agent.can_rate( trans.user, trans.user_is_admin(), cntrller, tool ) - can_upload_new_version = trans.app.security_agent.can_upload_new_version( trans.user, tool ) - categories = [ tca.category for tca in tool.categories ] - display_reviews = util.string_as_bool( params.get( 'display_reviews', False ) ) - tool_file_contents = tarfile.open( tool.file_name, 'r' ).getnames() - tra = self.get_user_item_rating( trans.sa_session, trans.user, tool, webapp_model=trans.model ) - visible_versions = trans.app.security_agent.get_visible_versions( trans.user, trans.user_is_admin(), cntrller, tool ) - if tool.is_rejected: - # Include the comments regarding the reason for rejection - reason_for_rejection = tool.latest_event.comment - else: - reason_for_rejection = '' - return trans.fill_template( '/webapps/community/tool/view_tool.mako', - avg_rating=avg_rating, - categories=categories, - can_approve_or_reject=can_approve_or_reject, - can_delete=can_delete, - can_download=can_download, - can_edit=can_edit, - can_purge=can_purge, - can_rate=can_rate, - can_upload_new_version=can_upload_new_version, - can_view=can_view, - cntrller=cntrller, - display_reviews=display_reviews, - num_ratings=num_ratings, - reason_for_rejection=reason_for_rejection, - tool=tool, - tool_file_contents=tool_file_contents, - tra=tra, - visible_versions=visible_versions, - message=message, - status=status ) - @web.expose - def delete_tool( self, trans, cntrller, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - id = params.get( 'id', None ) - if not id: - message='Select a tool to delete' - status='error' - else: - tool = get_tool( trans, id ) - if not trans.app.security_agent.can_delete( trans.user, trans.user_is_admin(), cntrller, tool ): - return trans.response.send_redirect( web.url_for( controller=cntrller, - action='browse_tools', - cntrller=cntrller, - message='You are not allowed to delete this tool', - status='error' ) ) - # Create a new event - event = trans.model.Event( state=trans.model.Tool.states.DELETED ) - # Flush so we can get an event id - trans.sa_session.add( event ) - trans.sa_session.flush() - # Associate the tool with the event - tea = trans.model.ToolEventAssociation( tool=tool, event=event ) - # Delete the tool, keeping state for categories, events and versions - tool.deleted = True - trans.sa_session.add_all( ( tool, tea ) ) - trans.sa_session.flush() - # TODO: What if the tool has versions, should they all be deleted? - message = "Tool '%s' version %s has been marked deleted" % ( tool.name, tool.version ) - status = 'done' - return trans.response.send_redirect( web.url_for( controller=cntrller, - action='browse_tools', - cntrller=cntrller, - message=message, - status=status ) ) - @web.expose - def download_tool( self, trans, cntrller, **kwd ): - params = util.Params( kwd ) - id = params.get( 'id', None ) - if not id: - return trans.response.send_redirect( web.url_for( controller='tool', - action='browse_tools', - cntrller=cntrller, - message='Select a tool to download', - status='error' ) ) - tool = get_tool( trans, id ) - if not trans.app.security_agent.can_download( trans.user, trans.user_is_admin(), cntrller, tool ): - return trans.response.send_redirect( web.url_for( controller=cntrller, - action='browse_tools', - cntrller=cntrller, - message='You are not allowed to download this tool', - status='error' ) ) - trans.response.set_content_type( tool.mimetype ) - trans.response.headers['Content-Length'] = int( os.stat( tool.file_name ).st_size ) - trans.response.headers['Content-Disposition'] = 'attachment; filename=%s' % tool.download_file_name - return open( tool.file_name ) - @web.expose - def upload_new_tool_version( self, trans, cntrller, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - id = params.get( 'id', None ) - if not id: - return trans.response.send_redirect( web.url_for( controller=cntrller, - action='browse_tools', - cntrller=cntrller, - message='Select a tool to upload a new version', - status='error' ) ) - tool = get_tool( trans, id ) - if not trans.app.security_agent.can_upload_new_version( trans.user, tool ): - return trans.response.send_redirect( web.url_for( controller=cntrller, - action='browse_tools', - cntrller=cntrller, - message='You are not allowed to upload a new version of this tool', - status='error' ) ) - return trans.response.send_redirect( web.url_for( controller='tool_upload', - action='upload', - message=message, - status=status, - replace_id=id ) ) - @web.expose - @web.require_login( "view tool history" ) - def view_tool_history( self, trans, cntrller, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - id = params.get( 'id', None ) - if not id: - return trans.response.send_redirect( web.url_for( controller=cntrller, - action='browse_tools', - cntrller=cntrller, - message='Select a tool to view its history', - status='error' ) ) - tool = get_tool( trans, id ) - can_view = trans.app.security_agent.can_view( trans.user, trans.user_is_admin(), cntrller, tool ) - if not can_view: - return trans.response.send_redirect( web.url_for( controller=cntrller, - action='browse_tools', - cntrller=cntrller, - message="You are not allowed to view this tool's history", - status='error' ) ) - can_approve_or_reject = trans.app.security_agent.can_approve_or_reject( trans.user, trans.user_is_admin(), cntrller, tool ) - can_edit = trans.app.security_agent.can_edit( trans.user, trans.user_is_admin(), cntrller, tool ) - can_delete = trans.app.security_agent.can_delete( trans.user, trans.user_is_admin(), cntrller, tool ) - can_download = trans.app.security_agent.can_download( trans.user, trans.user_is_admin(), cntrller, tool ) - events = [ tea.event for tea in tool.events ] - events = [ ( event.state, time_ago( event.update_time ), event.comment ) for event in events ] - return trans.fill_template( '/webapps/community/common/view_tool_history.mako', - cntrller=cntrller, - events=events, - tool=tool, - can_approve_or_reject=can_approve_or_reject, - can_edit=can_edit, - can_delete=can_delete, - can_download=can_download, - can_view=can_view, - message=message, - status=status ) - @web.expose - @web.require_login( "rate tools" ) - def rate_tool( self, trans, cntrller, **kwd ): - """ Rate a tool and return updated rating data. """ - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - id = params.get( 'id', None ) - if not id: - return trans.response.send_redirect( web.url_for( controller=cntrller, - action='browse_tools', - cntrller=cntrller, - message='Select a tool to rate', - status='error' ) ) - tool = get_tool( trans, id ) - can_rate = trans.app.security_agent.can_rate( trans.user, trans.user_is_admin(), cntrller, tool ) - if not can_rate: - return trans.response.send_redirect( web.url_for( controller=cntrller, - action='browse_tools', - cntrller=cntrller, - message="You are not allowed to rate this tool", - status='error' ) ) - if params.get( 'rate_button', False ): - rating = int( params.get( 'rating', '0' ) ) - comment = util.restore_text( params.get( 'comment', '' ) ) - rating = self.rate_item( trans, trans.user, tool, rating, comment ) - avg_rating, num_ratings = self.get_ave_item_rating_data( trans.sa_session, tool, webapp_model=trans.model ) - can_approve_or_reject = trans.app.security_agent.can_approve_or_reject( trans.user, trans.user_is_admin(), cntrller, tool ) - can_edit = trans.app.security_agent.can_edit( trans.user, trans.user_is_admin(), cntrller, tool ) - can_delete = trans.app.security_agent.can_delete( trans.user, trans.user_is_admin(), cntrller, tool ) - can_download = trans.app.security_agent.can_download( trans.user, trans.user_is_admin(), cntrller, tool ) - display_reviews = util.string_as_bool( params.get( 'display_reviews', False ) ) - tra = self.get_user_item_rating( trans.sa_session, trans.user, tool, webapp_model=trans.model ) - return trans.fill_template( '/webapps/community/common/rate_tool.mako', - cntrller=cntrller, - tool=tool, - avg_rating=avg_rating, - can_approve_or_reject=can_approve_or_reject, - can_edit=can_edit, - can_delete=can_delete, - can_download=can_download, - can_rate=can_rate, - display_reviews=display_reviews, - num_ratings=num_ratings, - tra=tra, - message=message, - status=status ) - ## ---- Utility methods ------------------------------------------------------- -def get_versions( item ): - """Get all versions of item""" - versions = [ item ] - this_item = item - while item.newer_version: - versions.insert( 0, item.newer_version ) - item = item.newer_version - item = this_item - while item.older_version: - versions.append( item.older_version[ 0 ] ) - item = item.older_version[ 0 ] - return versions def get_categories( trans ): """Get all categories from the database""" return trans.sa_session.query( trans.model.Category ) \ @@ -544,36 +83,539 @@ def get_categories( trans ): def get_category( trans, id ): """Get a category from the database""" return trans.sa_session.query( trans.model.Category ).get( trans.security.decode_id( id ) ) -def get_tool( trans, id ): - """Get a tool from the database""" - return trans.sa_session.query( trans.model.Tool ).get( trans.security.decode_id( id ) ) -def get_latest_versions_of_tools( trans ): - """Get only the latest version of each tool from the database""" - return trans.sa_session.query( trans.model.Tool ) \ - .filter( trans.model.Tool.table.c.newer_version_id == None ) \ - .order_by( trans.model.Tool.table.c.name ) -def get_latest_versions_of_tools_by_state( trans, state ): - """Get only the latest version of each tool whose state is the received state from the database""" - tools = [] - for tool in trans.sa_session.query( trans.model.Tool ) \ - .order_by( trans.model.Tool.table.c.name ): - if tool.state == state: - tools.append( tool ) - return tools -def get_event( trans, id ): - """Get an event from the databse""" - return trans.sa_session.query( trans.model.Event ).get( trans.security.decode_id( id ) ) -def get_user( trans, id ): - """Get a user from the database""" - return trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( id ) ) def get_repository( trans, id ): """Get a repository from the database via id""" return trans.sa_session.query( trans.model.Repository ).get( trans.security.decode_id( id ) ) +def get_repository_by_name_and_owner( trans, name, owner ): + """Get a repository from the database via name and owner""" + user = get_user_by_username( trans, owner ) + return trans.sa_session.query( trans.model.Repository ) \ + .filter( and_( trans.model.Repository.table.c.name == name, + trans.model.Repository.table.c.user_id == user.id ) ) \ + .first() +def get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ): + """Get metadata for a specified repository change set from the database""" + return trans.sa_session.query( trans.model.RepositoryMetadata ) \ + .filter( and_( trans.model.RepositoryMetadata.table.c.repository_id == trans.security.decode_id( id ), + trans.model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) ) \ + .first() +def get_repository_metadata_by_id( trans, id ): + """Get repository metadata from the database""" + return trans.sa_session.query( trans.model.RepositoryMetadata ).get( trans.security.decode_id( id ) ) +def get_repository_metadata_by_repository_id( trans, id ): + """Get all metadata records for a specified repository.""" + return trans.sa_session.query( trans.model.RepositoryMetadata ) \ + .filter( trans.model.RepositoryMetadata.table.c.repository_id == trans.security.decode_id( id ) ) +def get_revision_label( trans, repository, changeset_revision ): + """ + Return a string consisting of the human read-able + changeset rev and the changeset revision string. + """ + repo = hg.repository( get_configured_ui(), repository.repo_path ) + ctx = get_changectx_for_changeset( trans, repo, changeset_revision ) + if ctx: + return "%s:%s" % ( str( ctx.rev() ), changeset_revision ) + else: + return "-1:%s" % changeset_revision +def get_latest_repository_metadata( trans, id ): + """Get last metadata defined for a specified repository from the database""" + return trans.sa_session.query( trans.model.RepositoryMetadata ) \ + .filter( trans.model.RepositoryMetadata.table.c.repository_id == trans.security.decode_id( id ) ) \ + .order_by( trans.model.RepositoryMetadata.table.c.id.desc() ) \ + .first() +def generate_workflow_metadata( trans, id, changeset_revision, exported_workflow_dict, metadata_dict ): + """ + Update the received metadata_dict with changes that have been applied + to the received exported_workflow_dict. Store everything except the + workflow steps in the database. + """ + workflow_dict = { 'a_galaxy_workflow' : exported_workflow_dict[ 'a_galaxy_workflow' ], + 'name' :exported_workflow_dict[ 'name' ], + 'annotation' : exported_workflow_dict[ 'annotation' ], + 'format-version' : exported_workflow_dict[ 'format-version' ] } + if 'workflows' in metadata_dict: + metadata_dict[ 'workflows' ].append( workflow_dict ) + else: + metadata_dict[ 'workflows' ] = [ workflow_dict ] + return metadata_dict +def new_workflow_metadata_required( trans, id, metadata_dict ): + """ + TODO: Currently everything about an exported workflow except the name is hard-coded, so + there's no real way to differentiate versions of exported workflows. If this changes at + some future time, this method should be enhanced accordingly... + """ + if 'workflows' in metadata_dict: + repository_metadata = get_latest_repository_metadata( trans, id ) + if repository_metadata: + if repository_metadata.metadata: + # The repository has metadata, so update the workflows value - no new record is needed. + return False + else: + # There is no saved repository metadata, so we need to create a new repository_metadata table record. + return True + # The received metadata_dict includes no metadata for workflows, so a new repository_metadata table + # record is not needed. + return False +def generate_clone_url( trans, repository_id ): + repository = get_repository( trans, repository_id ) + protocol, base = trans.request.base.split( '://' ) + if trans.user: + username = '%s@' % trans.user.username + else: + username = '' + return '%s://%s%s/repos/%s/%s' % ( protocol, username, base, repository.user.username, repository.name ) +def generate_tool_guid( trans, repository, tool ): + """ + Generate a guid for the received tool. The form of the guid is + /repos//// + """ + return '%s/repos/%s/%s/%s/%s' % ( trans.request.host, + repository.user.username, + repository.name, + tool.id, + tool.version ) +def check_tool_input_params( trans, name, tool, sample_files, invalid_files ): + """ + Check all of the tool's input parameters, looking for any that are dynamically generated + using external data files to make sure the files exist. + """ + can_set_metadata = True + correction_msg = '' + for input_param in tool.input_params: + if isinstance( input_param, galaxy.tools.parameters.basic.SelectToolParameter ) and input_param.is_dynamic: + # If the tool refers to .loc files or requires an entry in the + # tool_data_table_conf.xml, make sure all requirements exist. + options = input_param.dynamic_options or input_param.options + if options: + if options.tool_data_table or options.missing_tool_data_table_name: + # Make sure the repository contains a tool_data_table_conf.xml.sample file. + sample_found = False + for sample_file in sample_files: + head, tail = os.path.split( sample_file ) + if tail == 'tool_data_table_conf.xml.sample': + sample_found = True + error, correction_msg = handle_sample_tool_data_table_conf_file( trans, sample_file ) + if error: + can_set_metadata = False + invalid_files.append( ( tail, correction_msg ) ) + else: + options.missing_tool_data_table_name = None + break + if not sample_found: + can_set_metadata = False + correction_msg = "This file requires an entry in the tool_data_table_conf.xml file. " + correction_msg += "Upload a file named tool_data_table_conf.xml.sample to the repository " + correction_msg += "that includes the required entry to resolve this issue.
    " + invalid_files.append( ( name, correction_msg ) ) + if options.index_file or options.missing_index_file: + # Make sure the repository contains the required xxx.loc.sample file. + index_file = options.index_file or options.missing_index_file + index_head, index_tail = os.path.split( index_file ) + sample_found = False + for sample_file in sample_files: + sample_head, sample_tail = os.path.split( sample_file ) + if sample_tail == '%s.sample' % index_tail: + copy_sample_loc_file( trans, sample_file ) + options.index_file = index_tail + options.missing_index_file = None + options.tool_data_table.missing_index_file = None + sample_found = True + break + if not sample_found: + can_set_metadata = False + correction_msg = "This file refers to a file named %s. " % str( index_file ) + correction_msg += "Upload a file named %s.sample to the repository to correct this error." % str( index_tail ) + invalid_files.append( ( name, correction_msg ) ) + return can_set_metadata, invalid_files +def generate_tool_metadata( trans, id, changeset_revision, tool_config, tool, metadata_dict ): + """ + Update the received metadata_dict with changes that have been + applied to the received tool. + """ + repository = get_repository( trans, id ) + # Handle tool.requirements. + tool_requirements = [] + for tr in tool.requirements: + name=tr.name + type=tr.type + if type == 'fabfile': + version = None + fabfile = tr.fabfile + method = tr.method + else: + version = tr.version + fabfile = None + method = None + requirement_dict = dict( name=name, + type=type, + version=version, + fabfile=fabfile, + method=method ) + tool_requirements.append( requirement_dict ) + # Handle tool.tests. + tool_tests = [] + if tool.tests: + for ttb in tool.tests: + test_dict = dict( name=ttb.name, + required_files=ttb.required_files, + inputs=ttb.inputs, + outputs=ttb.outputs ) + tool_tests.append( test_dict ) + tool_dict = dict( id=tool.id, + guid = generate_tool_guid( trans, repository, tool ), + name=tool.name, + version=tool.version, + description=tool.description, + version_string_cmd = tool.version_string_cmd, + tool_config=tool_config, + requirements=tool_requirements, + tests=tool_tests ) + if 'tools' in metadata_dict: + metadata_dict[ 'tools' ].append( tool_dict ) + else: + metadata_dict[ 'tools' ] = [ tool_dict ] + return metadata_dict +def new_tool_metadata_required( trans, id, metadata_dict ): + """ + Compare the last saved metadata for each tool in the repository with the new metadata + in metadata_dict to determine if a new repository_metadata table record is required, or + if the last saved metadata record can updated instead. + """ + if 'tools' in metadata_dict: + repository_metadata = get_latest_repository_metadata( trans, id ) + if repository_metadata: + metadata = repository_metadata.metadata + if metadata and 'tools' in metadata: + saved_tool_ids = [] + # The metadata for one or more tools was successfully generated in the past + # for this repository, so we first compare the version string for each tool id + # in metadata_dict with what was previously saved to see if we need to create + # a new table record or if we can simply update the existing record. + for new_tool_metadata_dict in metadata_dict[ 'tools' ]: + for saved_tool_metadata_dict in metadata[ 'tools' ]: + if saved_tool_metadata_dict[ 'id' ] not in saved_tool_ids: + saved_tool_ids.append( saved_tool_metadata_dict[ 'id' ] ) + if new_tool_metadata_dict[ 'id' ] == saved_tool_metadata_dict[ 'id' ]: + if new_tool_metadata_dict[ 'version' ] != saved_tool_metadata_dict[ 'version' ]: + return True + # So far, a new metadata record is not required, but we still have to check to see if + # any new tool ids exist in metadata_dict that are not in the saved metadata. We do + # this because if a new tarball was uploaded to a repository that included tools, it + # may have removed existing tool files if they were not included in the uploaded tarball. + for new_tool_metadata_dict in metadata_dict[ 'tools' ]: + if new_tool_metadata_dict[ 'id' ] not in saved_tool_ids: + return True + else: + # We have repository metadata that does not include metadata for any tools in the + # repository, so we can update the existing repository metadata. + return False + else: + # There is no saved repository metadata, so we need to create a new repository_metadata + # table record. + return True + # The received metadata_dict includes no metadata for tools, so a new repository_metadata table + # record is not needed. + return False +def set_repository_metadata( trans, id, changeset_revision, **kwd ): + """Set repository metadata""" + message = '' + status = 'done' + repository = get_repository( trans, id ) + repo_dir = repository.repo_path + repo = hg.repository( get_configured_ui(), repo_dir ) + invalid_files = [] + sample_files = [] + ctx = get_changectx_for_changeset( trans, repo, changeset_revision ) + if ctx is not None: + metadata_dict = {} + if changeset_revision == repository.tip: + for root, dirs, files in os.walk( repo_dir ): + if not root.find( '.hg' ) >= 0 and not root.find( 'hgrc' ) >= 0: + if '.hg' in dirs: + # Don't visit .hg directories - should be impossible since we don't + # allow uploaded archives that contain .hg dirs, but just in case... + dirs.remove( '.hg' ) + if 'hgrc' in files: + # Don't include hgrc files in commit. + files.remove( 'hgrc' ) + # Find all special .sample files first. + for name in files: + if name.endswith( '.sample' ): + sample_files.append( os.path.abspath( os.path.join( root, name ) ) ) + for name in files: + # Find all tool configs. + if name.endswith( '.xml' ): + try: + full_path = os.path.abspath( os.path.join( root, name ) ) + tool = load_tool( trans, full_path ) + if tool is not None: + can_set_metadata, invalid_files = check_tool_input_params( trans, name, tool, sample_files, invalid_files ) + if can_set_metadata: + # Update the list of metadata dictionaries for tools in metadata_dict. + tool_config = os.path.join( root, name ) + metadata_dict = generate_tool_metadata( trans, id, changeset_revision, tool_config, tool, metadata_dict ) + except Exception, e: + invalid_files.append( ( name, str( e ) ) ) + # Find all exported workflows + elif name.endswith( '.ga' ): + try: + full_path = os.path.abspath( os.path.join( root, name ) ) + # Convert workflow data from json + fp = open( full_path, 'rb' ) + workflow_text = fp.read() + fp.close() + exported_workflow_dict = from_json_string( workflow_text ) + # Update the list of metadata dictionaries for workflows in metadata_dict. + metadata_dict = generate_workflow_metadata( trans, id, changeset_revision, exported_workflow_dict, metadata_dict ) + except Exception, e: + invalid_files.append( ( name, str( e ) ) ) + else: + # Find all special .sample files first. + for filename in ctx: + if filename.endswith( '.sample' ): + sample_files.append( os.path.abspath( os.path.join( root, filename ) ) ) + # Get all tool config file names from the hgweb url, something like: + # /repos/test/convert_chars1/file/e58dcf0026c7/convert_characters.xml + for filename in ctx: + # Find all tool configs - should not have to update metadata for workflows for now. + if filename.endswith( '.xml' ): + fctx = ctx[ filename ] + # Write the contents of the old tool config to a temporary file. + fh = tempfile.NamedTemporaryFile( 'w' ) + tmp_filename = fh.name + fh.close() + fh = open( tmp_filename, 'w' ) + fh.write( fctx.data() ) + fh.close() + try: + tool = load_tool( trans, tmp_filename ) + if tool is not None: + can_set_metadata, invalid_files = check_tool_input_params( trans, filename, tool, sample_files, invalid_files ) + if can_set_metadata: + # Update the list of metadata dictionaries for tools in metadata_dict. Note that filename + # here is the relative path to the config file within the change set context, something + # like filtering.xml, but when the change set was the repository tip, the value was + # something like database/community_files/000/repo_1/filtering.xml. This shouldn't break + # anything, but may result in a bit of confusion when maintaining the code / data over time. + metadata_dict = generate_tool_metadata( trans, id, changeset_revision, filename, tool, metadata_dict ) + except Exception, e: + invalid_files.append( ( name, str( e ) ) ) + try: + os.unlink( tmp_filename ) + except: + pass + if metadata_dict: + if changeset_revision == repository.tip: + if new_tool_metadata_required( trans, id, metadata_dict ) or new_workflow_metadata_required( trans, id, metadata_dict ): + # Create a new repository_metadata table row. + repository_metadata = trans.model.RepositoryMetadata( repository.id, changeset_revision, metadata_dict ) + trans.sa_session.add( repository_metadata ) + trans.sa_session.flush() + else: + # Update the last saved repository_metadata table row. + repository_metadata = get_latest_repository_metadata( trans, id ) + repository_metadata.changeset_revision = changeset_revision + repository_metadata.metadata = metadata_dict + trans.sa_session.add( repository_metadata ) + trans.sa_session.flush() + else: + # We're re-generating metadata for an old repository revision. + repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + repository_metadata.metadata = metadata_dict + trans.sa_session.add( repository_metadata ) + trans.sa_session.flush() + else: + message = "Change set revision '%s' includes no tools or exported workflows for which metadata can be set." % str( changeset_revision ) + status = "error" + else: + # change_set is None + message = "Repository does not include change set revision '%s'." % str( changeset_revision ) + status = 'error' + if invalid_files: + if metadata_dict: + message = "Metadata was defined for some items in change set revision '%s'. " % str( changeset_revision ) + message += "Correct the following problems if necessary and reset metadata.
    " + else: + message = "Metadata cannot be defined for change set revision '%s'. Correct the following problems and reset metadata.
    " % str( changeset_revision ) + for itc_tup in invalid_files: + tool_file, exception_msg = itc_tup + if exception_msg.find( 'No such file or directory' ) >= 0: + exception_items = exception_msg.split() + missing_file_items = exception_items[7].split( '/' ) + missing_file = missing_file_items[-1].rstrip( '\'' ) + if missing_file.endswith( '.loc' ): + sample_ext = '%s.sample' % missing_file + else: + sample_ext = missing_file + correction_msg = "This file refers to a missing file %s. " % str( missing_file ) + correction_msg += "Upload a file named %s to the repository to correct this error." % sample_ext + else: + correction_msg = exception_msg + message += "%s - %s
    " % ( tool_file, correction_msg ) + status = 'error' + return message, status def get_repository_by_name( trans, name ): """Get a repository from the database via name""" - return trans.sa_session.query( app.model.Repository ).filter_by( name=name ).one() -def get_repository_tip( repository ): - # The received repository must be a mercurial repository, not a db record. - tip_changeset = repository.changelog.tip() - tip_ctx = repository.changectx( tip_changeset ) - return "%s:%s" % ( str( tip_ctx.rev() ), tip_ctx.parents()[0] ) + return trans.sa_session.query( trans.model.Repository ).filter_by( name=name ).one() +def get_changectx_for_changeset( trans, repo, changeset_revision, **kwd ): + """Retrieve a specified changectx from a repository""" + for changeset in repo.changelog: + ctx = repo.changectx( changeset ) + if str( ctx ) == changeset_revision: + return ctx + return None +def change_set_is_malicious( trans, id, changeset_revision, **kwd ): + """Check the malicious flag in repository metadata for a specified change set""" + repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + if repository_metadata: + return repository_metadata.malicious + return False +def get_configured_ui(): + # Configure any desired ui settings. + _ui = ui.ui() + # The following will suppress all messages. This is + # the same as adding the following setting to the repo + # hgrc file' [ui] section: + # quiet = True + _ui.setconfig( 'ui', 'quiet', True ) + return _ui +def get_user( trans, id ): + """Get a user from the database by id""" + return trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( id ) ) +def handle_email_alerts( trans, repository ): + repo_dir = repository.repo_path + repo = hg.repository( get_configured_ui(), repo_dir ) + smtp_server = trans.app.config.smtp_server + if smtp_server and repository.email_alerts: + # Send email alert to users that want them. + if trans.app.config.email_from is not None: + email_from = trans.app.config.email_from + elif trans.request.host.split( ':' )[0] == 'localhost': + email_from = 'galaxy-no-reply@' + socket.getfqdn() + else: + email_from = 'galaxy-no-reply@' + trans.request.host.split( ':' )[0] + tip_changeset = repo.changelog.tip() + ctx = repo.changectx( tip_changeset ) + t, tz = ctx.date() + date = datetime( *time.gmtime( float( t ) - tz )[:6] ) + display_date = date.strftime( "%Y-%m-%d" ) + try: + username = ctx.user().split()[0] + except: + username = ctx.user() + # Build the email message + body = string.Template( email_alert_template ) \ + .safe_substitute( host=trans.request.host, + repository_name=repository.name, + revision='%s:%s' %( str( ctx.rev() ), ctx ), + display_date=display_date, + description=ctx.description(), + username=username ) + frm = email_from + subject = "Galaxy tool shed repository update alert" + email_alerts = from_json_string( repository.email_alerts ) + for email in email_alerts: + to = email.strip() + # Send it + try: + util.send_mail( frm, to, subject, body, trans.app.config ) + except Exception, e: + log.exception( "An error occurred sending a tool shed repository update alert by email." ) +def update_for_browsing( trans, repository, current_working_dir, commit_message='' ): + # Make a copy of a repository's files for browsing, remove from disk all files that + # are not tracked, and commit all added, modified or removed files that have not yet + # been committed. + repo_dir = repository.repo_path + repo = hg.repository( get_configured_ui(), repo_dir ) + # The following will delete the disk copy of only the files in the repository. + #os.system( 'hg update -r null > /dev/null 2>&1' ) + repo.ui.pushbuffer() + commands.status( repo.ui, repo, all=True ) + status_and_file_names = repo.ui.popbuffer().strip().split( "\n" ) + # status_and_file_names looks something like: + # ['? README', '? tmap_tool/tmap-0.0.9.tar.gz', '? dna_filtering.py', 'C filtering.py', 'C filtering.xml'] + # The codes used to show the status of files are: + # M = modified + # A = added + # R = removed + # C = clean + # ! = deleted, but still tracked + # ? = not tracked + # I = ignored + files_to_remove_from_disk = [] + files_to_commit = [] + for status_and_file_name in status_and_file_names: + if status_and_file_name.startswith( '?' ) or status_and_file_name.startswith( 'I' ): + files_to_remove_from_disk.append( os.path.abspath( os.path.join( repo_dir, status_and_file_name.split()[1] ) ) ) + elif status_and_file_name.startswith( 'M' ) or status_and_file_name.startswith( 'A' ) or status_and_file_name.startswith( 'R' ): + files_to_commit.append( os.path.abspath( os.path.join( repo_dir, status_and_file_name.split()[1] ) ) ) + for full_path in files_to_remove_from_disk: + # We'll remove all files that are not tracked or ignored. + if os.path.isdir( full_path ): + try: + os.rmdir( full_path ) + except OSError, e: + # The directory is not empty + pass + elif os.path.isfile( full_path ): + os.remove( full_path ) + dir = os.path.split( full_path )[0] + try: + os.rmdir( dir ) + except OSError, e: + # The directory is not empty + pass + if files_to_commit: + if not commit_message: + commit_message = 'Committed changes to: %s' % ', '.join( files_to_commit ) + repo.dirstate.write() + repo.commit( user=trans.user.username, text=commit_message ) + os.chdir( repo_dir ) + os.system( 'hg update > /dev/null 2>&1' ) + os.chdir( current_working_dir ) +def load_tool( trans, config_file ): + """ + Load a single tool from the file named by `config_file` and return + an instance of `Tool`. + """ + # Parse XML configuration file and get the root element + tree = util.parse_xml( config_file ) + root = tree.getroot() + if root.tag == 'tool': + # Allow specifying a different tool subclass to instantiate + if root.find( "type" ) is not None: + type_elem = root.find( "type" ) + module = type_elem.get( 'module', 'galaxy.tools' ) + cls = type_elem.get( 'class' ) + mod = __import__( module, globals(), locals(), [cls]) + ToolClass = getattr( mod, cls ) + elif root.get( 'tool_type', None ) is not None: + ToolClass = tool_types.get( root.get( 'tool_type' ) ) + else: + ToolClass = Tool + return ToolClass( config_file, root, trans.app ) + return None +def build_changeset_revision_select_field( trans, repository, selected_value=None, add_id_to_name=True ): + """ + Build a SelectField whose options are the changeset_revision + strings of all downloadable_revisions of the received repository. + """ + repo = hg.repository( get_configured_ui(), repository.repo_path ) + options = [] + refresh_on_change_values = [] + for repository_metadata in repository.downloadable_revisions: + changeset_revision = repository_metadata.changeset_revision + revision_label = get_revision_label( trans, repository, changeset_revision ) + options.append( ( revision_label, changeset_revision ) ) + refresh_on_change_values.append( changeset_revision ) + if add_id_to_name: + name = 'changeset_revision_%d' % repository.id + else: + name = 'changeset_revision' + select_field = SelectField( name=name, + refresh_on_change=True, + refresh_on_change_values=refresh_on_change_values ) + for option_tup in options: + selected = selected_value and option_tup[1] == selected_value + select_field.add_option( option_tup[0], option_tup[1], selected=selected ) + return select_field diff --git a/lib/galaxy/webapps/community/controllers/hg.py b/lib/galaxy/webapps/community/controllers/hg.py index 85e56423875..fe37a275125 100644 --- a/lib/galaxy/webapps/community/controllers/hg.py +++ b/lib/galaxy/webapps/community/controllers/hg.py @@ -5,7 +5,7 @@ from mercurial.hgweb.request import wsgiapplication log = logging.getLogger(__name__) -class HgController( BaseController ): +class HgController( BaseUIController ): @web.expose def handle_request( self, trans, **kwd ): # The os command that results in this method being called will look something like diff --git a/lib/galaxy/webapps/community/controllers/repository.py b/lib/galaxy/webapps/community/controllers/repository.py index 01417921369..1a377dcc450 100644 --- a/lib/galaxy/webapps/community/controllers/repository.py +++ b/lib/galaxy/webapps/community/controllers/repository.py @@ -1,22 +1,31 @@ -import os, logging, urllib, ConfigParser, tempfile, shutil, pexpect +import os, logging, urllib, ConfigParser, tempfile, shutil from time import strftime -from datetime import * - +from datetime import date, datetime from galaxy import util +from galaxy.datatypes.checkers import * from galaxy.web.base.controller import * +from galaxy.web.form_builder import CheckboxField from galaxy.webapps.community import model from galaxy.webapps.community.model import directory_hash_id from galaxy.web.framework.helpers import time_ago, iff, grids +from galaxy.util.json import from_json_string, to_json_string from galaxy.model.orm import * from common import * -from mercurial import hg, ui, patch +from mercurial import hg, ui, patch, commands log = logging.getLogger( __name__ ) +# Characters that must be html escaped +MAPPED_CHARS = { '>' :'>', + '<' :'<', + '"' : '"', + '&' : '&', + '\'' : ''' } +MAX_CONTENT_SIZE = 32768 +VALID_CHARS = set( string.letters + string.digits + "'\"-=_.()/+*^,:?!#[]%\\$@;{}" ) VALID_REPOSITORYNAME_RE = re.compile( "^[a-z0-9\_]+$" ) - -class RepositoryCategoryListGrid( grids.Grid ): - # TODO rename this class to be categoryListGrid when we eliminate all the tools stuff. + +class CategoryListGrid( grids.Grid ): class NameColumn( grids.TextColumn ): def get_value( self, trans, grid, category ): return category.name @@ -40,30 +49,19 @@ class RepositoryCategoryListGrid( grids.Grid ): default_sort_key = "name" columns = [ NameColumn( "Name", - key="name", + key="Category.name", link=( lambda item: dict( operation="repositories_by_category", id=item.id, webapp="community" ) ), - attach_popup=False, - filterable="advanced" ), + attach_popup=False ), DescriptionColumn( "Description", - key="description", - attach_popup=False, - filterable="advanced" ), + key="Category.description", + attach_popup=False ), # Columns that are valid for filtering but are not visible. - grids.DeletedColumn( "Deleted", - key="deleted", - visible=False, - filterable="advanced" ), RepositoriesColumn( "Repositories", model_class=model.Repository, attach_popup=False ) ] - columns.append( grids.MulticolFilterColumn( "Search category name, description", - cols_to_filter=[ columns[0], columns[1] ], - key="free-text-search", - visible=False, - filterable="standard" ) ) - # Override these + default_filter = {} global_actions = [] operations = [] standard_filters = [] @@ -75,10 +73,18 @@ class RepositoryListGrid( grids.Grid ): class NameColumn( grids.TextColumn ): def get_value( self, trans, grid, repository ): return repository.name - class VersionColumn( grids.TextColumn ): + class RevisionColumn( grids.GridColumn ): + def __init__( self, col_name ): + grids.GridColumn.__init__( self, col_name ) def get_value( self, trans, grid, repository ): - repo = hg.repository( ui.ui(), repository.repo_path ) - return get_repository_tip( repo ) + """ + Display a SelectField whose options are the changeset_revision + strings of all downloadable_revisions of this repository. + """ + select_field = build_changeset_revision_select_field( trans, repository ) + if len( select_field.options ) > 1: + return select_field.get_html() + return repository.revision class DescriptionColumn( grids.TextColumn ): def get_value( self, trans, grid, repository ): return repository.description @@ -97,7 +103,7 @@ class RepositoryListGrid( grids.Grid ): def filter( self, trans, user, query, column_filter ): """Modify query to filter by category.""" if column_filter == "All": - pass + return query return query.filter( model.Category.name == column_filter ) class UserColumn( grids.TextColumn ): def get_value( self, trans, grid, repository ): @@ -110,6 +116,11 @@ class RepositoryListGrid( grids.Grid ): return query return query.filter( and_( model.Repository.table.c.user_id == model.User.table.c.id, model.User.table.c.email == column_filter ) ) + class EmailAlertsColumn( grids.TextColumn ): + def get_value( self, trans, grid, repository ): + if trans.user and repository.email_alerts and trans.user.email in from_json_string( repository.email_alerts ): + return 'yes' + return '' # Grid definition title = "Repositories" model_class = model.Repository @@ -117,14 +128,15 @@ class RepositoryListGrid( grids.Grid ): default_sort_key = "name" columns = [ NameColumn( "Name", - key="Repository.name", - link=( lambda item: dict( operation="view_or_manage_repository", id=item.id, webapp="community" ) ), - attach_popup=False ), - DescriptionColumn( "Description", + key="name", + link=( lambda item: dict( operation="view_or_manage_repository", + id=item.id, + webapp="community" ) ), + attach_popup=True ), + DescriptionColumn( "Synopsis", key="description", attach_popup=False ), - VersionColumn( "Version", - attach_popup=False ), + RevisionColumn( "Revision" ), CategoryColumn( "Category", model_class=model.Category, key="Category.name", @@ -133,9 +145,9 @@ class RepositoryListGrid( grids.Grid ): model_class=model.User, link=( lambda item: dict( operation="repositories_by_user", id=item.id, webapp="community" ) ), attach_popup=False, - key="username" ), - grids.CommunityRatingColumn( "Average Rating", - key="rating" ), + key="User.username" ), + grids.CommunityRatingColumn( "Average Rating", key="rating" ), + EmailAlertsColumn( "Alert", attach_popup=False ), # Columns that are valid for filtering but are not visible. EmailColumn( "Email", model_class=model.User, @@ -144,16 +156,23 @@ class RepositoryListGrid( grids.Grid ): RepositoryCategoryColumn( "Category", model_class=model.Category, key="Category.name", - visible=False ) + visible=False ), + grids.DeletedColumn( "Deleted", + key="deleted", + visible=False, + filterable="advanced" ) ] columns.append( grids.MulticolFilterColumn( "Search repository name, description", cols_to_filter=[ columns[0], columns[1] ], key="free-text-search", visible=False, filterable="standard" ) ) - operations = [] + operations = [ grids.GridOperation( "Receive email alerts", + allow_multiple=False, + condition=( lambda item: not item.deleted ), + async_compatible=False ) ] standard_filters = [] - default_filter = {} + default_filter = dict( deleted="False" ) num_rows_per_page = 50 preserve_state = False use_paging = True @@ -163,11 +182,50 @@ class RepositoryListGrid( grids.Grid ): .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) -class RepositoryController( BaseController, ItemRatings ): +class DownloadableRepositoryListGrid( RepositoryListGrid ): + class RevisionColumn( grids.GridColumn ): + def __init__( self, col_name ): + grids.GridColumn.__init__( self, col_name ) + def get_value( self, trans, grid, repository ): + """ + Display a SelectField whose options are the changeset_revision + strings of all downloadable_revisions of this repository. + """ + select_field = build_changeset_revision_select_field( trans, repository ) + if len( select_field.options ) > 1: + return select_field.get_html() + return repository.revision + title = "Downloadable repositories" + columns = [ + RepositoryListGrid.NameColumn( "Name", + key="name", + attach_popup=True ), + RepositoryListGrid.DescriptionColumn( "Synopsis", + key="description", + attach_popup=False ), + RevisionColumn( "Revision" ), + RepositoryListGrid.UserColumn( "Owner", + model_class=model.User, + attach_popup=False, + key="User.username" ) + ] + columns.append( grids.MulticolFilterColumn( "Search repository name, description", + cols_to_filter=[ columns[0], columns[1] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) + operations = [] + def build_initial_query( self, trans, **kwd ): + return trans.sa_session.query( self.model_class ) \ + .join( model.RepositoryMetadata.table ) \ + .join( model.User.table ) +class RepositoryController( BaseUIController, ItemRatings ): + + downloadable_repository_list_grid = DownloadableRepositoryListGrid() repository_list_grid = RepositoryListGrid() - category_list_grid = RepositoryCategoryListGrid() - + category_list_grid = CategoryListGrid() + @web.expose def index( self, trans, **kwd ): params = util.Params( kwd ) @@ -176,6 +234,20 @@ class RepositoryController( BaseController, ItemRatings ): return trans.fill_template( '/webapps/community/index.mako', message=message, status=status ) @web.expose def browse_categories( self, trans, **kwd ): + if 'f-free-text-search' in kwd: + # Trick to enable searching repository name, description from the CategoryListGrid. + # What we've done is rendered the search box for the RepositoryListGrid on the grid.mako + # template for the CategoryListGrid. See ~/templates/webapps/community/category/grid.mako. + # Since we are searching repositories and not categories, redirect to browse_repositories(). + if 'id' in kwd and 'f-free-text-search' in kwd and kwd[ 'id' ] == kwd[ 'f-free-text-search' ]: + # The value of 'id' has been set to the search string, which is a repository name. + # We'll try to get the desired encoded repository id to pass on. + try: + repository = get_repository_by_name( trans, kwd[ 'id' ] ) + kwd[ 'id' ] = trans.security.encode_id( repository.id ) + except: + pass + return self.browse_repositories( trans, **kwd ) if 'operation' in kwd: operation = kwd['operation'].lower() if operation in [ "repositories_by_category", "repositories_by_user" ]: @@ -189,6 +261,182 @@ class RepositoryController( BaseController, ItemRatings ): # Render the list view return self.category_list_grid( trans, **kwd ) @web.expose + def browse_downloadable_repositories( self, trans, **kwd ): + # Set the toolshedgalaxyurl cookie so we can get back + # to the calling local Galaxy instance. + galaxy_url = kwd.get( 'galaxy_url', None ) + if galaxy_url: + trans.set_cookie( galaxy_url, name='toolshedgalaxyurl' ) + repository_id = kwd.get( 'id', None ) + if 'operation' in kwd: + operation = kwd[ 'operation' ].lower() + if operation == "preview_tools_in_changeset": + repository = get_repository( trans, repository_id ) + return trans.response.send_redirect( web.url_for( controller='repository', + action='preview_tools_in_changeset', + repository_id=repository_id, + changeset_revision=repository.tip ) ) + + # The changeset_revision_select_field in the RepositoryListGrid performs a refresh_on_change + # which sends in request parameters like changeset_revison_1, changeset_revision_2, etc. One + # of the many select fields on the grid performed the refresh_on_change, so we loop through + # all of the received values to see which value is not the repository tip. If we find it, we + # know the refresh_on_change occurred, and we have the necessary repository id and change set + # revision to pass on. + for k, v in kwd.items(): + changset_revision_str = 'changeset_revision_' + if k.startswith( changset_revision_str ): + repository_id = trans.security.encode_id( int( k.lstrip( changset_revision_str ) ) ) + repository = get_repository( trans, repository_id ) + if repository.tip != v: + return trans.response.send_redirect( web.url_for( controller='repository', + action='preview_tools_in_changeset', + repository_id=trans.security.encode_id( repository.id ), + changeset_revision=v ) ) + url_args = dict( action='browse_downloadable_repositories', + operation='preview_tools_in_changeset', + repository_id=repository_id ) + self.downloadable_repository_list_grid.operations = [ grids.GridOperation( "Preview and install tools", + url_args=url_args, + allow_multiple=False, + async_compatible=False ) ] + + # Render the list view + return self.downloadable_repository_list_grid( trans, **kwd ) + @web.expose + def preview_tools_in_changeset( self, trans, repository_id, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + repository = get_repository( trans, repository_id ) + changeset_revision = util.restore_text( params.get( 'changeset_revision', repository.tip ) ) + repository_metadata = get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) + if repository_metadata: + metadata = repository_metadata.metadata + else: + metadata = None + revision_label = get_revision_label( trans, repository, changeset_revision ) + changeset_revision_select_field = build_changeset_revision_select_field( trans, + repository, + selected_value=changeset_revision, + add_id_to_name=False ) + return trans.fill_template( '/webapps/community/repository/preview_tools_in_changeset.mako', + repository=repository, + changeset_revision=changeset_revision, + revision_label=revision_label, + changeset_revision_select_field=changeset_revision_select_field, + metadata=metadata, + display_for_install=True, + message=message, + status=status ) + @web.expose + def install_repository_revision( self, trans, repository_id, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + galaxy_url = trans.get_cookie( name='toolshedgalaxyurl' ) + repository = get_repository( trans, repository_id ) + changeset_revision = util.restore_text( params.get( 'changeset_revision', repository.tip ) ) + # Redirect back to local Galaxy to perform install. + tool_shed_url = trans.request.host + repository_clone_url = generate_clone_url( trans, repository_id ) + # TODO: support https in the following url. + url = 'http://%s/admin/install_tool_shed_repository?tool_shed_url=%s&name=%s&description=%s&repository_clone_url=%s&changeset_revision=%s' % \ + ( galaxy_url, tool_shed_url, repository.name, repository.description, repository_clone_url, changeset_revision ) + return trans.response.send_redirect( url ) + @web.expose + def check_for_updates( self, trans, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + # The sender didn't store galaxy_url in a cookie since + # this method immediately redirects back to the caller. + galaxy_url = kwd[ 'galaxy_url' ] + name = params.get( 'name', None ) + owner = params.get( 'owner', None ) + changeset_revision = params.get( 'changeset_revision', None ) + webapp = params.get( 'webapp', None ) + tool_shed_url = trans.request.host + # Start building up the url to redirect back to the calling Galaxy instance. + # TODO: support https in the following url. + url = 'http://%s/admin/update_to_changeset_revision?tool_shed_url=%s' % ( galaxy_url, tool_shed_url ) + repository = get_repository_by_name_and_owner( trans, name, owner ) + #if error: + # url += '&message=%s&status=error' % message + #else: + url += '&name=%s&owner=%s&changeset_revision=%s&latest_changeset_revision=' % \ + ( repository.name, repository.user.username, changeset_revision ) + if changeset_revision == repository.tip: + # If changeset_revision is the repository tip, then + # we know there are no additional updates for the tools. + url += repository.tip + else: + repository_metadata = get_repository_metadata_by_changeset_revision( trans, + trans.security.encode_id( repository.id ), + changeset_revision ) + if repository_metadata: + # If changeset_revision is in the repository_metadata table for this + # repository, then we know there are no additional updates for the tools. + url += changeset_revision + else: + # The changeset_revision column in the repository_metadata table has been + # updated with a new changeset_revision value since the repository was cloned. + repo_dir = repository.repo_path + repo = hg.repository( get_configured_ui(), repo_dir ) + # Load each tool in the repository's changeset_revision to generate a list of + # tool guids, since guids differentiate tools by id and version. + ctx = get_changectx_for_changeset( trans, repo, changeset_revision ) + if ctx is not None: + tool_guids = [] + for filename in ctx: + # Find all tool configs in this repository changeset_revision. + if filename.endswith( '.xml' ): + fctx = ctx[ filename ] + # Write the contents of the old tool config to a temporary file. + fh = tempfile.NamedTemporaryFile( 'w' ) + tmp_filename = fh.name + fh.close() + fh = open( tmp_filename, 'w' ) + fh.write( fctx.data() ) + fh.close() + try: + tool = load_tool( trans, tmp_filename ) + if tool is not None: + tool_guids.append( generate_tool_guid( trans, repository, tool ) ) + except: + # File must not be a valid tool config even though it has a .xml extension. + pass + try: + os.unlink( tmp_filename ) + except: + pass + tool_guids.sort() + if tool_guids: + # Compare our list of tool guids against those in each repository_metadata record + # for the repository to find the repository_metadata record with the changeset_revision + # value we want to pass back to the caller. + found = False + for repository_metadata in get_repository_metadata_by_repository_id( trans, trans.security.encode_id( repository.id ) ): + metadata = repository_metadata.metadata + metadata_tool_guids = [] + for tool_dict in metadata[ 'tools' ]: + metadata_tool_guids.append( tool_dict[ 'guid' ] ) + metadata_tool_guids.sort() + if tool_guids == metadata_tool_guids: + # We've found the repository_metadata record whose changeset_revision + # value has been updated. + url += repository_metadata.changeset_revision + found = True + break + if not found: + # There must be a problem in the data, so we'll just send back the received changeset_revision. + log.debug( "Possible data corruption - updated repository_metadata cannot be found for repository id %d." % repository.id ) + url += changeset_revision + else: + # There are not tools in the changeset_revision, so no tool updates are possible. + url += changeset_revision + return trans.response.send_redirect( url ) + @web.expose def browse_repositories( self, trans, **kwd ): # We add params to the keyword dict in this method in order to rename the param # with an "f-" prefix, simulating filtering by clicking a search link. We have @@ -196,9 +444,10 @@ class RepositoryController( BaseController, ItemRatings ): if 'operation' in kwd: operation = kwd['operation'].lower() if operation == "view_or_manage_repository": - repository_id = kwd.get( 'id', None ) + repository_id = kwd[ 'id' ] repository = get_repository( trans, repository_id ) - if repository.user == trans.user: + is_admin = trans.user_is_admin() + if is_admin or repository.user == trans.user: return trans.response.send_redirect( web.url_for( controller='repository', action='manage_repository', **kwd ) ) @@ -239,6 +488,33 @@ class RepositoryController( BaseController, ItemRatings ): category_id = kwd.get( 'id', None ) category = get_category( trans, category_id ) kwd[ 'f-Category.name' ] = category.name + elif operation == "receive email alerts": + if trans.user: + if kwd[ 'id' ]: + return trans.response.send_redirect( web.url_for( controller='repository', + action='set_email_alerts', + **kwd ) ) + else: + kwd[ 'message' ] = 'You must be logged in to set email alerts.' + kwd[ 'status' ] = 'error' + del kwd[ 'operation' ] + # The changeset_revision_select_field in the RepositoryListGrid performs a refresh_on_change + # which sends in request parameters like changeset_revison_1, changeset_revision_2, etc. One + # of the many select fields on the grid performed the refresh_on_change, so we loop through + # all of the received values to see which value is not the repository tip. If we find it, we + # know the refresh_on_change occurred, and we have the necessary repository id and change set + # revision to pass on. + for k, v in kwd.items(): + changset_revision_str = 'changeset_revision_' + if k.startswith( changset_revision_str ): + repository_id = trans.security.encode_id( int( k.lstrip( changset_revision_str ) ) ) + repository = get_repository( trans, repository_id ) + if repository.tip != v: + return trans.response.send_redirect( web.url_for( controller='repository', + action='browse_repositories', + operation='view_or_manage_repository', + id=trans.security.encode_id( repository.id ), + changeset_revision=v ) ) # Render the list view return self.repository_list_grid( trans, **kwd ) @web.expose @@ -257,12 +533,10 @@ class RepositoryController( BaseController, ItemRatings ): status=status ) ) name = util.restore_text( params.get( 'name', '' ) ) description = util.restore_text( params.get( 'description', '' ) ) + long_description = util.restore_text( params.get( 'long_description', '' ) ) category_ids = util.listify( params.get( 'category_id', '' ) ) selected_categories = [ trans.security.decode_id( id ) for id in category_ids ] if params.get( 'create_repository_button', False ): - # TODOS: - # 1. Make sure we can update the version column in the repository table when new change set are pushed. - # If it's triclky, eliminate the column. error = False message = self.__validate_repository_name( name, trans.user ) if message: @@ -272,7 +546,10 @@ class RepositoryController( BaseController, ItemRatings ): error = True if not error: # Add the repository record to the db - repository = trans.app.model.Repository( name=name, description=description, user_id=trans.user.id ) + repository = trans.app.model.Repository( name=name, + description=description, + long_description=long_description, + user_id=trans.user.id ) # Flush to get the id trans.sa_session.add( repository ) trans.sa_session.flush() @@ -287,7 +564,7 @@ class RepositoryController( BaseController, ItemRatings ): if not os.path.exists( repository_path ): os.makedirs( repository_path ) # Create the local repository - repo = hg.repository( ui.ui(), repository_path, create=True ) + repo = hg.repository( get_configured_ui(), repository_path, create=True ) # Add an entry in the hgweb.config file for the local repository # This enables calls to repository.repo_path self.__add_hgweb_config_entry( trans, repository, repository_path ) @@ -311,6 +588,7 @@ class RepositoryController( BaseController, ItemRatings ): return trans.fill_template( '/webapps/community/repository/create_repository.mako', name=name, description=description, + long_description=long_description, selected_categories=selected_categories, categories=categories, message=message, @@ -330,11 +608,19 @@ class RepositoryController( BaseController, ItemRatings ): if not( VALID_REPOSITORYNAME_RE.match( name ) ): return "Repository names must contain only lower-case letters, numbers and underscore '_'." return '' + def __make_hgweb_config_copy( self, trans, hgweb_config ): + # Make a backup of the hgweb.config file + today = date.today() + backup_date = today.strftime( "%Y_%m_%d" ) + hgweb_config_copy = '%s/hgweb.config_%s_backup' % ( trans.app.config.root, backup_date ) + shutil.copy( os.path.abspath( hgweb_config ), os.path.abspath( hgweb_config_copy ) ) def __add_hgweb_config_entry( self, trans, repository, repository_path ): - # Add an entry in the hgweb.config file for a new repository. This enables calls to repository.repo_path. - # An entry looks something like: repos/test/mira_assembler = database/community_files/000/repo_123. - # TODO: I believe this can be done via ui.updateconfig(), but I haven't confirmed this... + # Add an entry in the hgweb.config file for a new repository. + # An entry looks something like: + # repos/test/mira_assembler = database/community_files/000/repo_123. hgweb_config = "%s/hgweb.config" % trans.app.config.root + # Make a backup of the hgweb.config file since we're going to be changing it. + self.__make_hgweb_config_copy( trans, hgweb_config ) entry = "repos/%s/%s = %s" % ( repository.user.username, repository.name, repository_path.lstrip( './' ) ) if os.path.exists( hgweb_config ): output = open( hgweb_config, 'a' ) @@ -343,6 +629,25 @@ class RepositoryController( BaseController, ItemRatings ): output.write( '[paths]\n' ) output.write( "%s\n" % entry ) output.close() + def __change_hgweb_config_entry( self, trans, repository, old_repository_name, new_repository_name ): + # Change an entry in the hgweb.config file for a repository. This only happens when + # the owner changes the name of the repository. An entry looks something like: + # repos/test/mira_assembler = database/community_files/000/repo_123. + hgweb_config = "%s/hgweb.config" % trans.app.config.root + # Make a backup of the hgweb.config file since we're going to be changing it. + self.__make_hgweb_config_copy( trans, hgweb_config ) + repo_dir = repository.repo_path + old_lhs = "repos/%s/%s" % ( repository.user.username, old_repository_name ) + old_entry = "%s = %s" % ( old_lhs, repo_dir ) + new_entry = "repos/%s/%s = %s\n" % ( repository.user.username, new_repository_name, repo_dir ) + tmp_fd, tmp_fname = tempfile.mkstemp() + new_hgweb_config = open( tmp_fname, 'wb' ) + for i, line in enumerate( open( hgweb_config ) ): + if line.startswith( old_lhs ): + new_hgweb_config.write( new_entry ) + else: + new_hgweb_config.write( line ) + shutil.move( tmp_fname, os.path.abspath( hgweb_config ) ) def __create_hgrc_file( self, repository ): # At this point, an entry for the repository is required to be in the hgweb.config # file so we can call repository.repo_path. @@ -351,65 +656,168 @@ class RepositoryController( BaseController, ItemRatings ): # allow_push = test # name = convert_characters1 # push_ssl = False - # Upon repository creation, only the owner can push to it ( allow_push setting ), - # and since we support both http and https, we set push_ssl to False to override + # Since we support both http and https, we set push_ssl to False to override # the default (which is True) in the mercurial api. - hgrc_file = os.path.abspath( os.path.join( repository.repo_path, ".hg", "hgrc" ) ) - output = open( hgrc_file, 'w' ) - output.write( '[web]\n' ) - output.write( 'allow_push = %s\n' % repository.user.username ) - output.write( 'name = %s\n' % repository.name ) - output.write( 'push_ssl = false\n' ) - output.flush() - output.close() - def __get_allow_push( self, repository ): - # TODO: Use the mercurial api to handle this - hgrc_file = os.path.abspath( os.path.join( repository.repo_path, ".hg", "hgrc" ) ) - config = ConfigParser.ConfigParser() - config.read( hgrc_file ) - for option in config.options( "web" ): - if option == 'allow_push': - return config.get( "web", option ) - raise Exception( "Repository %s missing allow_push entry under the [web] option in it's hgrc file." % repository.name ) - def __set_allow_push( self, repository, usernames, remove_auth='' ): - # TODO: Use the mercurial api to handle this - hgrc_file = os.path.abspath( os.path.join( repository.repo_path, ".hg", "hgrc" ) ) - fh, fn = tempfile.mkstemp() - for i, line in enumerate( open( hgrc_file ) ): - if line.startswith( 'allow_push' ): - value = line.split( ' = ' )[1].rstrip( '\n' ) - if remove_auth: - current_usernames = value.split( ',' ) - new_usernames = [] - for current_username in current_usernames: - if current_username != remove_auth: - new_usernames.append( current_username ) - new_usernames = ','.join( new_usernames ) - line = 'allow_push = %s\n' % new_usernames - else: - value = '%s,%s\n' % ( value, usernames ) - line = 'allow_push = %s' % value - os.write( fh, line ) - os.close( fh ) - shutil.move( fn, hgrc_file ) + repo = hg.repository( get_configured_ui(), path=repository.repo_path ) + fp = repo.opener( 'hgrc', 'wb' ) + fp.write( '[paths]\n' ) + fp.write( 'default = .\n' ) + fp.write( 'default-push = .\n' ) + fp.write( '[web]\n' ) + fp.write( 'allow_push = %s\n' % repository.user.username ) + fp.write( 'name = %s\n' % repository.name ) + fp.write( 'push_ssl = false\n' ) + fp.close() @web.expose def browse_repository( self, trans, id, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) + commit_message = util.restore_text( params.get( 'commit_message', 'Deleted selected files' ) ) repository = get_repository( trans, id ) - repo = hg.repository( ui.ui(), repository.repo_path ) - # TODO: Our current support for browsing a repository requires copies of the - # repository files to be in the repository root directory. We do the following - # to ensure the latest files are being browsed. + repo = hg.repository( get_configured_ui(), repository.repo_path ) current_working_dir = os.getcwd() - repo_dir = repository.repo_path - os.chdir( repo_dir ) - os.system( 'hg update > /dev/null 2>&1' ) - os.chdir( current_working_dir ) + # Update repository files for browsing. + update_for_browsing( trans, repository, current_working_dir, commit_message=commit_message ) + is_malicious = change_set_is_malicious( trans, id, repository.tip ) return trans.fill_template( '/webapps/community/repository/browse_repository.mako', repo=repo, repository=repository, + commit_message=commit_message, + is_malicious=is_malicious, + message=message, + status=status ) + @web.expose + def contact_owner( self, trans, id, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + repository = get_repository( trans, id ) + if trans.user and trans.user.email: + return trans.fill_template( "/webapps/community/repository/contact_owner.mako", + repository=repository, + message=message, + status=status ) + else: + # Do all we can to eliminate spam. + return trans.show_error_message( "You must be logged in to contact the owner of a repository." ) + @web.expose + def send_to_owner( self, trans, id, message='' ): + repository = get_repository( trans, id ) + if not message: + message = 'Enter a message' + status = 'error' + elif trans.user and trans.user.email: + smtp_server = trans.app.config.smtp_server + from_address = trans.app.config.email_from + if smtp_server is None or from_address is None: + return trans.show_error_message( "Mail is not configured for this Galaxy tool shed instance" ) + to_address = repository.user.email + # Get the name of the server hosting the tool shed instance. + host = trans.request.host + # Build the email message + body = string.Template( contact_owner_template ) \ + .safe_substitute( username=trans.user.username, + repository_name=repository.name, + email=trans.user.email, + message=message, + host=host ) + subject = "Regarding your tool shed repository named %s" % repository.name + # Send it + try: + util.send_mail( from_address, to_address, subject, body, trans.app.config ) + message = "Your message has been sent" + status = "done" + except Exception, e: + message = "An error occurred sending your message by email: %s" % str( e ) + status = "error" + else: + # Do all we can to eliminate spam. + return trans.show_error_message( "You must be logged in to contact the owner of a repository." ) + return trans.response.send_redirect( web.url_for( controller='repository', + action='contact_owner', + id=id, + message=message, + status=status ) ) + @web.expose + def select_files_to_delete( self, trans, id, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + commit_message = util.restore_text( params.get( 'commit_message', 'Deleted selected files' ) ) + repository = get_repository( trans, id ) + repo_dir = repository.repo_path + repo = hg.repository( get_configured_ui(), repo_dir ) + selected_files_to_delete = util.restore_text( params.get( 'selected_files_to_delete', '' ) ) + if params.get( 'select_files_to_delete_button', False ): + if selected_files_to_delete: + selected_files_to_delete = selected_files_to_delete.split( ',' ) + current_working_dir = os.getcwd() + # Get the current repository tip. + tip = repository.tip + for selected_file in selected_files_to_delete: + try: + commands.remove( repo.ui, repo, repo_file, force=True ) + except Exception, e: + # I never have a problem with commands.remove on a Mac, but in the test/production + # tool shed environment, it throws an exception whenever I delete all files from a + # repository. If this happens, we'll try the following. + relative_selected_file = selected_file.split( 'repo_%d' % repository.id )[1].lstrip( '/' ) + repo.dirstate.remove( relative_selected_file ) + repo.dirstate.write() + absolute_selected_file = os.path.abspath( selected_file ) + if os.path.isdir( absolute_selected_file ): + try: + os.rmdir( absolute_selected_file ) + except OSError, e: + # The directory is not empty + pass + elif os.path.isfile( absolute_selected_file ): + os.remove( absolute_selected_file ) + dir = os.path.split( absolute_selected_file )[0] + try: + os.rmdir( dir ) + except OSError, e: + # The directory is not empty + pass + # Commit the change set. + if not commit_message: + commit_message = 'Deleted selected files' + try: + commands.commit( repo.ui, repo, repo_dir, user=trans.user.username, message=commit_message ) + except Exception, e: + # I never have a problem with commands.commit on a Mac, but in the test/production + # tool shed environment, it occasionally throws a "TypeError: array item must be char" + # exception. If this happens, we'll try the following. + repo.dirstate.write() + repo.commit( user=trans.user.username, text=commit_message ) + handle_email_alerts( trans, repository ) + # Update the repository files for browsing. + update_for_browsing( trans, repository, current_working_dir, commit_message=commit_message ) + # Get the new repository tip. + repo = hg.repository( get_configured_ui(), repo_dir ) + if tip != repository.tip: + message = "The selected files were deleted from the repository." + else: + message = 'No changes to repository.' + # Set metadata on the repository tip + error_message, status = set_repository_metadata( trans, id, repository.tip, **kwd ) + if error_message: + message = '%s
    %s' % ( message, error_message ) + return trans.response.send_redirect( web.url_for( controller='repository', + action='manage_repository', + id=id, + message=message, + status=status ) ) + else: + message = "Select at least 1 file to delete from the repository before clicking Delete selected files." + status = "error" + is_malicious = change_set_is_malicious( trans, id, repository.tip ) + return trans.fill_template( '/webapps/community/repository/browse_repository.mako', + repo=repo, + repository=repository, + commit_message=commit_message, + is_malicious=is_malicious, message=message, status=status ) @web.expose @@ -418,17 +826,63 @@ class RepositoryController( BaseController, ItemRatings ): message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) repository = get_repository( trans, id ) - repo = hg.repository( ui.ui(), repository.repo_path ) - tip = get_repository_tip( repo ) + repo = hg.repository( get_configured_ui(), repository.repo_path ) avg_rating, num_ratings = self.get_ave_item_rating_data( trans.sa_session, repository, webapp_model=trans.model ) + changeset_revision = util.restore_text( params.get( 'changeset_revision', repository.tip ) ) display_reviews = util.string_as_bool( params.get( 'display_reviews', False ) ) + alerts = params.get( 'alerts', '' ) + alerts_checked = CheckboxField.is_checked( alerts ) + if repository.email_alerts: + email_alerts = from_json_string( repository.email_alerts ) + else: + email_alerts = [] + user = trans.user + if user and params.get( 'receive_email_alerts_button', False ): + flush_needed = False + if alerts_checked: + if user.email not in email_alerts: + email_alerts.append( user.email ) + repository.email_alerts = to_json_string( email_alerts ) + flush_needed = True + else: + if user.email in email_alerts: + email_alerts.remove( user.email ) + repository.email_alerts = to_json_string( email_alerts ) + flush_needed = True + if flush_needed: + trans.sa_session.add( repository ) + trans.sa_session.flush() + checked = alerts_checked or ( user and user.email in email_alerts ) + alerts_check_box = CheckboxField( 'alerts', checked=checked ) + changeset_revision_select_field = build_changeset_revision_select_field( trans, + repository, + selected_value=changeset_revision, + add_id_to_name=False ) + revision_label = get_revision_label( trans, repository, changeset_revision ) + repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + if repository_metadata: + metadata = repository_metadata.metadata + else: + metadata = None + is_malicious = change_set_is_malicious( trans, id, repository.tip ) + if is_malicious: + if trans.app.security_agent.can_push( trans.user, repository ): + message += malicious_error_can_push + else: + message += malicious_error + status = 'error' return trans.fill_template( '/webapps/community/repository/view_repository.mako', repo=repo, repository=repository, - tip=tip, + metadata=metadata, avg_rating=avg_rating, display_reviews=display_reviews, num_ratings=num_ratings, + alerts_check_box=alerts_check_box, + changeset_revision=changeset_revision, + changeset_revision_select_field=changeset_revision_select_field, + revision_label=revision_label, + is_malicious=is_malicious, message=message, status=status ) @web.expose @@ -438,17 +892,28 @@ class RepositoryController( BaseController, ItemRatings ): message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) repository = get_repository( trans, id ) - repo = hg.repository( ui.ui(), repository.repo_path ) - tip = get_repository_tip( repo ) + repo_dir = repository.repo_path + repo = hg.repository( get_configured_ui(), repo_dir ) repo_name = util.restore_text( params.get( 'repo_name', repository.name ) ) + changeset_revision = util.restore_text( params.get( 'changeset_revision', repository.tip ) ) description = util.restore_text( params.get( 'description', repository.description ) ) + long_description = util.restore_text( params.get( 'long_description', repository.long_description ) ) avg_rating, num_ratings = self.get_ave_item_rating_data( trans.sa_session, repository, webapp_model=trans.model ) display_reviews = util.string_as_bool( params.get( 'display_reviews', False ) ) + alerts = params.get( 'alerts', '' ) + alerts_checked = CheckboxField.is_checked( alerts ) + category_ids = util.listify( params.get( 'category_id', '' ) ) + if repository.email_alerts: + email_alerts = from_json_string( repository.email_alerts ) + else: + email_alerts = [] allow_push = params.get( 'allow_push', '' ) error = False + user = trans.user if params.get( 'edit_repository_button', False ): flush_needed = False - if trans.user != repository.user: + # TODO: add a can_manage in the security agent. + if user != repository.user: message = "You are not the owner of this repository, so you cannot manage it." status = error return trans.response.send_redirect( web.url_for( controller='repository', @@ -457,18 +922,37 @@ class RepositoryController( BaseController, ItemRatings ): message=message, status=status ) ) if repo_name != repository.name: - message = self.__validate_repository_name( repo_name, trans.user ) + message = self.__validate_repository_name( repo_name, user ) if message: error = True else: + self.__change_hgweb_config_entry( trans, repository, repository.name, repo_name ) repository.name = repo_name flush_needed = True if description != repository.description: repository.description = description flush_needed = True + if long_description != repository.long_description: + repository.long_description = long_description + flush_needed = True if flush_needed: trans.sa_session.add( repository ) trans.sa_session.flush() + message = "The repository information has been updated." + elif params.get( 'manage_categories_button', False ): + flush_needed = False + # Delete all currently existing categories. + for rca in repository.categories: + trans.sa_session.delete( rca ) + trans.sa_session.flush() + if category_ids: + # Create category associations + for category_id in category_ids: + category = trans.app.model.Category.get( trans.security.decode_id( category_id ) ) + rca = trans.app.model.RepositoryCategoryAssociation( repository, category ) + trans.sa_session.add( rca ) + trans.sa_session.flush() + message = "The repository information has been updated." elif params.get( 'user_access_button', False ): if allow_push not in [ 'none' ]: remove_auth = params.get( 'remove_auth', '' ) @@ -481,22 +965,74 @@ class RepositoryController( BaseController, ItemRatings ): user = trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( user_id ) ) usernames.append( user.username ) usernames = ','.join( usernames ) - self.__set_allow_push( repository, usernames, remove_auth=remove_auth ) + repository.set_allow_push( usernames, remove_auth=remove_auth ) + message = "The repository information has been updated." + elif params.get( 'receive_email_alerts_button', False ): + flush_needed = False + if alerts_checked: + if user.email not in email_alerts: + email_alerts.append( user.email ) + repository.email_alerts = to_json_string( email_alerts ) + flush_needed = True + else: + if user.email in email_alerts: + email_alerts.remove( user.email ) + repository.email_alerts = to_json_string( email_alerts ) + flush_needed = True + if flush_needed: + trans.sa_session.add( repository ) + trans.sa_session.flush() + message = "The repository information has been updated." if error: status = 'error' - current_allow_push_list = self.__get_allow_push( repository ).split( ',' ) + if repository.allow_push: + current_allow_push_list = repository.allow_push.split( ',' ) + else: + current_allow_push_list = [] allow_push_select_field = self.__build_allow_push_select_field( trans, current_allow_push_list ) + checked = alerts_checked or user.email in email_alerts + alerts_check_box = CheckboxField( 'alerts', checked=checked ) + changeset_revision_select_field = build_changeset_revision_select_field( trans, + repository, + selected_value=changeset_revision, + add_id_to_name=False ) + revision_label = get_revision_label( trans, repository, changeset_revision ) + repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + if repository_metadata: + metadata = repository_metadata.metadata + is_malicious = repository_metadata.malicious + else: + metadata = None + is_malicious = False + if is_malicious: + if trans.app.security_agent.can_push( trans.user, repository ): + message += malicious_error_can_push + else: + message += malicious_error + status = 'error' + malicious_check_box = CheckboxField( 'malicious', checked=is_malicious ) + categories = get_categories( trans ) + selected_categories = [ rca.category_id for rca in repository.categories ] return trans.fill_template( '/webapps/community/repository/manage_repository.mako', repo_name=repo_name, description=description, + long_description=long_description, current_allow_push_list=current_allow_push_list, allow_push_select_field=allow_push_select_field, repo=repo, repository=repository, - tip=tip, + changeset_revision=changeset_revision, + changeset_revision_select_field=changeset_revision_select_field, + revision_label=revision_label, + selected_categories=selected_categories, + categories=categories, + metadata=metadata, avg_rating=avg_rating, display_reviews=display_reviews, num_ratings=num_ratings, + alerts_check_box=alerts_check_box, + malicious_check_box=malicious_check_box, + is_malicious=is_malicious, message=message, status=status ) @web.expose @@ -505,7 +1041,7 @@ class RepositoryController( BaseController, ItemRatings ): message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) repository = get_repository( trans, id ) - repo = hg.repository( ui.ui(), repository.repo_path ) + repo = hg.repository( get_configured_ui(), repository.repo_path ) changesets = [] for changeset in repo.changelog: ctx = repo.changectx( changeset ) @@ -522,9 +1058,11 @@ class RepositoryController( BaseController, ItemRatings ): 'parent' : ctx.parents()[0] } # Make sure we'll view latest changeset first. changesets.insert( 0, change_dict ) + is_malicious = change_set_is_malicious( trans, id, repository.tip ) return trans.fill_template( '/webapps/community/repository/view_changelog.mako', repository=repository, changesets=changesets, + is_malicious=is_malicious, message=message, status=status ) @web.expose @@ -533,14 +1071,9 @@ class RepositoryController( BaseController, ItemRatings ): message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) repository = get_repository( trans, id ) - repo = hg.repository( ui.ui(), repository.repo_path ) - found = False - for changeset in repo.changelog: - ctx = repo.changectx( changeset ) - if str( ctx ) == ctx_str: - found = True - break - if not found: + repo = hg.repository( get_configured_ui(), repository.repo_path ) + ctx = get_changectx_for_changeset( trans, repo, ctx_str ) + if ctx is None: message = "Repository does not include changeset revision '%s'." % str( ctx_str ) status = 'error' return trans.response.send_redirect( web.url_for( controller='repository', @@ -551,23 +1084,10 @@ class RepositoryController( BaseController, ItemRatings ): ctx_parent = ctx.parents()[0] modified, added, removed, deleted, unknown, ignored, clean = repo.status( node1=ctx_parent.node(), node2=ctx.node() ) anchors = modified + added + removed + deleted + unknown + ignored + clean - def is_binary( chars ): - is_binary = False - chars_read = 0 - for char in chars: - chars_read += 1 - if ord( char ) > 128: - is_binary = True - break - return is_binary diffs = [] for diff in patch.diff( repo, node1=ctx_parent.node(), node2=ctx.node() ): - if not util.is_multi_byte( diff ) and not is_binary( diff ): - # TODO: is there a better way? - diffs.append( diff ) - else: - fixed_diff = diff.split( '\n' )[0] + '\nFile contains non-ascii characters that cannot be displayed\n' - diffs.append( fixed_diff ) + diffs.append( self.to_html_escaped( diff ) ) + is_malicious = change_set_is_malicious( trans, id, repository.tip ) return trans.fill_template( '/webapps/community/repository/view_changeset.mako', repository=repository, ctx=ctx, @@ -580,6 +1100,7 @@ class RepositoryController( BaseController, ItemRatings ): ignored=ignored, clean=clean, diffs=diffs, + is_malicious=is_malicious, message=message, status=status ) @web.expose @@ -596,6 +1117,7 @@ class RepositoryController( BaseController, ItemRatings ): message='Select a repository to rate', status='error' ) ) repository = get_repository( trans, id ) + repo = hg.repository( get_configured_ui(), repository.repo_path ) if repository.user == trans.user: return trans.response.send_redirect( web.url_for( controller='repository', action='browse_repositories', @@ -608,26 +1130,229 @@ class RepositoryController( BaseController, ItemRatings ): avg_rating, num_ratings = self.get_ave_item_rating_data( trans.sa_session, repository, webapp_model=trans.model ) display_reviews = util.string_as_bool( params.get( 'display_reviews', False ) ) rra = self.get_user_item_rating( trans.sa_session, trans.user, repository, webapp_model=trans.model ) + is_malicious = change_set_is_malicious( trans, id, repository.tip ) return trans.fill_template( '/webapps/community/repository/rate_repository.mako', repository=repository, avg_rating=avg_rating, display_reviews=display_reviews, num_ratings=num_ratings, rra=rra, + is_malicious=is_malicious, message=message, status=status ) + @web.expose + @web.require_login( "set email alerts" ) + def set_email_alerts( self, trans, **kwd ): + # Set email alerts for selected repositories + params = util.Params( kwd ) + user = trans.user + if user: + repository_ids = util.listify( kwd.get( 'id', '' ) ) + total_alerts_added = 0 + total_alerts_removed = 0 + flush_needed = False + for repository_id in repository_ids: + repository = get_repository( trans, repository_id ) + if repository.email_alerts: + email_alerts = from_json_string( repository.email_alerts ) + else: + email_alerts = [] + if user.email in email_alerts: + email_alerts.remove( user.email ) + repository.email_alerts = to_json_string( email_alerts ) + trans.sa_session.add( repository ) + flush_needed = True + total_alerts_removed += 1 + else: + email_alerts.append( user.email ) + repository.email_alerts = to_json_string( email_alerts ) + trans.sa_session.add( repository ) + flush_needed = True + total_alerts_added += 1 + if flush_needed: + trans.sa_session.flush() + message = 'Total alerts added: %d, total alerts removed: %d' % ( total_alerts_added, total_alerts_removed ) + kwd[ 'message' ] = message + kwd[ 'status' ] = 'done' + del kwd[ 'operation' ] + return trans.response.send_redirect( web.url_for( controller='repository', + action='browse_repositories', + **kwd ) ) + @web.expose + @web.require_login( "set repository metadata" ) + def set_metadata( self, trans, id, ctx_str, **kwd ): + malicious = kwd.get( 'malicious', '' ) + if kwd.get( 'malicious_button', False ): + repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, ctx_str ) + malicious_checked = CheckboxField.is_checked( malicious ) + repository_metadata.malicious = malicious_checked + trans.sa_session.add( repository_metadata ) + trans.sa_session.flush() + if malicious_checked: + message = "The repository tip has been defined as malicious." + else: + message = "The repository tip has been defined as not malicious." + status = 'done' + else: + # The set_metadata_button was clicked + message, status = set_repository_metadata( trans, id, ctx_str, **kwd ) + if not message: + message = "Metadata for change set revision '%s' has been reset." % str( ctx_str ) + return trans.response.send_redirect( web.url_for( controller='repository', + action='manage_repository', + id=id, + changeset_revision=ctx_str, + malicious=malicious, + message=message, + status=status ) ) + @web.expose + def display_tool( self, trans, repository_id, tool_config, changeset_revision, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + display_for_install = util.string_as_bool( params.get( 'display_for_install', False ) ) + repository = get_repository( trans, repository_id ) + repo = hg.repository( get_configured_ui(), repository.repo_path ) + try: + if changeset_revision == repository.tip: + # Get the tool config from the file system we use for browsing. + tool = load_tool( trans, os.path.abspath( tool_config ) ) + else: + # Get the tool config file name from the hgweb url, something like: + # /repos/test/convert_chars1/file/e58dcf0026c7/convert_characters.xml + old_tool_config_file_name = tool_config.split( '/' )[ -1 ] + ctx = get_changectx_for_changeset( trans, repo, changeset_revision ) + fctx = None + for filename in ctx: + filename_head, filename_tail = os.path.split( filename ) + if filename_tail == old_tool_config_file_name: + fctx = ctx[ filename ] + break + if fctx: + # Write the contents of the old tool config to a temporary file. + fh = tempfile.NamedTemporaryFile( 'w' ) + tmp_filename = fh.name + fh.close() + fh = open( tmp_filename, 'w' ) + fh.write( fctx.data() ) + fh.close() + tool = load_tool( trans, tmp_filename ) + try: + os.unlink( tmp_filename ) + except: + pass + else: + tool = None + tool_state = self.__new_state( trans ) + is_malicious = change_set_is_malicious( trans, repository_id, repository.tip ) + return trans.fill_template( "/webapps/community/repository/tool_form.mako", + repository=repository, + changeset_revision=changeset_revision, + tool=tool, + tool_state=tool_state, + is_malicious=is_malicious, + display_for_install=display_for_install, + message=message, + status=status ) + except Exception, e: + message = "Error loading tool: %s. Click Reset metadata to correct this error." % str( e ) + if display_for_install: + return trans.response.send_redirect( web.url_for( controller='repository', + action='preview_tools_in_changeset', + repository_id=repository_id, + changeset_revision=changeset_revision, + message=message, + status='error' ) ) + return trans.response.send_redirect( web.url_for( controller='repository', + action='browse_repositories', + operation='view_or_manage_repository', + id=repository_id, + changeset_revision=changeset_revision, + message=message, + status='error' ) ) + def __new_state( self, trans, all_pages=False ): + """ + Create a new `DefaultToolState` for this tool. It will not be initialized + with default values for inputs. + + Only inputs on the first page will be initialized unless `all_pages` is + True, in which case all inputs regardless of page are initialized. + """ + state = DefaultToolState() + state.inputs = {} + return state + @web.expose + def view_tool_metadata( self, trans, repository_id, changeset_revision, tool_id, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + display_for_install = util.string_as_bool( params.get( 'display_for_install', False ) ) + repository = get_repository( trans, repository_id ) + metadata = {} + tool = None + revision_label = get_revision_label( trans, repository, changeset_revision ) + repository_metadata = get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ).metadata + if 'tools' in repository_metadata: + for tool_metadata_dict in repository_metadata[ 'tools' ]: + if tool_metadata_dict[ 'id' ] == tool_id: + metadata = tool_metadata_dict + try: + # We may be attempting to load a tool that no longer exists in the repository tip. + tool = load_tool( trans, os.path.abspath( metadata[ 'tool_config' ] ) ) + except: + tool = None + break + is_malicious = change_set_is_malicious( trans, repository_id, repository.tip ) + changeset_revision_select_field = build_changeset_revision_select_field( trans, + repository, + selected_value=changeset_revision, + add_id_to_name=False ) + return trans.fill_template( "/webapps/community/repository/view_tool_metadata.mako", + repository=repository, + tool=tool, + metadata=metadata, + changeset_revision=changeset_revision, + revision_label=revision_label, + changeset_revision_select_field=changeset_revision_select_field, + is_malicious=is_malicious, + display_for_install=display_for_install, + message=message, + status=status ) + @web.expose + def download( self, trans, repository_id, changeset_revision, file_type, **kwd ): + # Download an archive of the repository files compressed as zip, gz or bz2. + params = util.Params( kwd ) + repository = get_repository( trans, repository_id ) + # Allow hgweb to handle the download. This requires the tool shed + # server account's .hgrc file to include the following setting: + # [web] + # allow_archive = bz2, gz, zip + if file_type == 'zip': + file_type_str = '%s.zip' % changeset_revision + elif file_type == 'bz2': + file_type_str = '%s.tar.bz2' % changeset_revision + elif file_type == 'gz': + file_type_str = '%s.tar.gz' % changeset_revision + repository.times_downloaded += 1 + trans.sa_session.add( repository ) + trans.sa_session.flush() + download_url = '/repos/%s/%s/archive/%s' % ( repository.user.username, repository.name, file_type_str ) + return trans.response.send_redirect( download_url ) @web.json def open_folder( self, trans, repository_id, key ): - # TODO: The tool shed includes a repository source file browser, which currently depends upon - # copies of the hg repository file store in the repo_path for browsing. We need to figure - # out how to use the mercurial api to browse repository contents so we don't need these copied - # files ( not bad now, but hwen the tools shed includes data indexes, not good ). + # The tool shed includes a repository source file browser, which currently depends upon + # copies of the hg repository file store in the repo_path for browsing. # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' repository = trans.sa_session.query( trans.model.Repository ).get( trans.security.decode_id( repository_id ) ) folder_path = key - files_list = self.__get_files( trans, repository, folder_path ) + try: + files_list = self.__get_files( trans, folder_path ) + except OSError, e: + if str( e ).find( 'No such file or directory' ) >= 0: + # We have a repository with no contents. + return [] folder_contents = [] for filename in files_list: is_folder = False @@ -642,50 +1367,76 @@ class RepositoryController( BaseController, ItemRatings ): "key": full_path } folder_contents.append( node ) return folder_contents - def __get_files( self, trans, repository, folder_path ): - ok = True - def print_ticks( d ): - pass - cmd = "ls -p '%s'" % folder_path - # Handle the authentication message if keys are not set - the message is - output = pexpect.run( cmd, - events={ pexpect.TIMEOUT : print_ticks }, - timeout=10 ) - if 'No such file or directory' in output: - status = 'error' - message = "No folder named (%s) exists." % folder_path - ok = False - if ok: - return output.split() - return trans.response.send_redirect( web.url_for( controller='repository', - action='browse_repositories', - operation="view_or_manage_repository", - id=trans.security.encode_id( repository.id ), - status=status, - message=message ) ) + def __get_files( self, trans, folder_path ): + contents = [] + for item in os.listdir( folder_path ): + # Skip .hg directories + if str( item ).startswith( '.hg' ): + continue + if os.path.isdir( os.path.join( folder_path, item ) ): + # Append a '/' character so that our jquery dynatree will + # function properly. + item = '%s/' % item + contents.append( item ) + if contents: + contents.sort() + return contents @web.json def get_file_contents( self, trans, file_path ): - def print_ticks( d ): - # pexpect timeout method - pass # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - if os.stat( file_path ).st_size > 32768: - return 'File size larger than maximum viewing size of 32 kb' - cmd = "cat %s" % file_path - # Handle the authentication message if ssh keys are not set - the message is - # something like: "Are you sure you want to continue connecting (yes/no)." - output = pexpect.run( cmd, - events={ pexpect.TIMEOUT : print_ticks }, - timeout=10 ) - return unicode( output.replace( '\r\n', '
    ' ).replace( ' ', ' ' ) ) + if is_gzip( file_path ): + to_html = self.to_html_str( '\ngzip compressed file\n' ) + elif is_bz2( file_path ): + to_html = self.to_html_str( '\nbz2 compressed file\n' ) + elif check_zip( file_path ): + to_html = self.to_html_str( '\nzip compressed file\n' ) + elif check_binary( file_path ): + to_html = self.to_html_str( '\nBinary file\n' ) + else: + to_html = '' + for i, line in enumerate( open( file_path ) ): + to_html = '%s%s' % ( to_html, self.to_html_str( line ) ) + if len( to_html ) > MAX_CONTENT_SIZE: + large_str = '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( MAX_CONTENT_SIZE ) + to_html = '%s%s' % ( to_html, self.to_html_str( large_str ) ) + break + return to_html @web.expose def help( self, trans, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) return trans.fill_template( '/webapps/community/repository/help.mako', message=message, status=status, **kwd ) + def to_html_escaped( self, text ): + """Translates the characters in text to html values""" + translated = [] + for c in text: + if c in [ '\r\n', '\n', ' ', '\t' ] or c in VALID_CHARS: + translated.append( c ) + elif c in MAPPED_CHARS: + translated.append( MAPPED_CHARS[ c ] ) + else: + translated.append( 'X' ) + return ''.join( translated ) + def to_html_str( self, text ): + """Translates the characters in text to sn html string""" + translated = [] + for c in text: + if c in VALID_CHARS: + translated.append( c ) + elif c in MAPPED_CHARS: + translated.append( MAPPED_CHARS[ c ] ) + elif c == ' ': + translated.append( ' ' ) + elif c == '\t': + translated.append( '    ' ) + elif c == '\n': + translated.append( '
    ' ) + elif c not in [ '\r' ]: + translated.append( 'X' ) + return ''.join( translated ) def __build_allow_push_select_field( self, trans, current_push_list, selected_value='none' ): options = [] for user in trans.sa_session.query( trans.model.User ): diff --git a/lib/galaxy/webapps/community/controllers/tool.py b/lib/galaxy/webapps/community/controllers/tool.py deleted file mode 100644 index 4efc9c2e03a..00000000000 --- a/lib/galaxy/webapps/community/controllers/tool.py +++ /dev/null @@ -1,214 +0,0 @@ -import os, logging, urllib, tarfile - -from galaxy.web.base.controller import * -from galaxy.webapps.community import model -from galaxy.web.framework.helpers import time_ago, iff, grids -from galaxy.model.orm import * -# TODO: the following is bad because it imports the common controller. -from common import * - -log = logging.getLogger( __name__ ) - -class StateColumn( grids.StateColumn ): - def get_value( self, trans, grid, tool ): - state = tool.state - if state == trans.model.Tool.states.APPROVED: - state_color = 'ok' - elif state == trans.model.Tool.states.REJECTED: - state_color = 'error' - elif state == trans.model.Tool.states.ARCHIVED: - state_color = 'upload' - else: - state_color = state - return '
    %s
    ' % ( state_color, state ) - -class ToolStateColumn( grids.StateColumn ): - def filter( self, trans, user, query, column_filter ): - """Modify query to filter self.model_class by state.""" - if column_filter == "All": - pass - elif column_filter in [ v for k, v in self.model_class.states.items() ]: - # Get all of the latest Events associated with the current version of each tool - latest_event_id_for_current_versions_of_tools = [ tool.latest_event.id for tool in get_latest_versions_of_tools_by_state( trans, column_filter ) ] - # Filter query by the latest state for the current version of each tool - return query.filter( and_( model.Event.table.c.state == column_filter, - model.Event.table.c.id.in_( latest_event_id_for_current_versions_of_tools ) ) ) - return query - -class ApprovedToolListGrid( ToolListGrid ): - columns = [ col for col in ToolListGrid.columns ] - columns.append( - StateColumn( "Status", - link=( lambda item: dict( operation="tools_by_state", id=item.id, webapp="community" ) ), - visible=False, - attach_popup=False ) - ) - columns.append( - ToolStateColumn( "State", - key="state", - visible=False, - filterable="advanced" ) - ) - -class MyToolsListGrid( ApprovedToolListGrid ): - columns = [ col for col in ToolListGrid.columns ] - columns.append( - StateColumn( "Status", - link=( lambda item: dict( operation="tools_by_state", id=item.id, webapp="community" ) ), - visible=True, - attach_popup=False ) - ) - columns.append( - ToolStateColumn( "State", - key="state", - visible=False, - filterable="advanced" ) - ) - -class ToolCategoryListGrid( CategoryListGrid ): - """ - Replaces the tools column in the Category grid with a similar column, - but displaying the number of APPROVED tools in the category. - """ - class ToolsColumn( grids.TextColumn ): - def get_value( self, trans, grid, category ): - if category.tools: - viewable_tools = 0 - for tca in category.tools: - tool = tca.tool - if tool.is_approved: - viewable_tools += 1 - return viewable_tools - return 0 - - columns = [] - for col in CategoryListGrid.columns: - if not isinstance( col, CategoryListGrid.ToolsColumn ): - columns.append( col ) - columns.append( - ToolsColumn( "Tools", - model_class=model.Tool, - attach_popup=False ) - ) - -class ToolController( BaseController ): - - tool_list_grid = ApprovedToolListGrid() - my_tools_list_grid = MyToolsListGrid() - category_list_grid = ToolCategoryListGrid() - - @web.expose - def index( self, trans, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - return trans.fill_template( '/webapps/community/index.mako', message=message, status=status ) - @web.expose - def browse_categories( self, trans, **kwd ): - if 'operation' in kwd: - operation = kwd['operation'].lower() - if operation in [ "tools_by_category", "tools_by_state", "tools_by_user" ]: - # Eliminate the current filters if any exist. - for k, v in kwd.items(): - if k.startswith( 'f-' ): - del kwd[ k ] - return trans.response.send_redirect( web.url_for( controller='tool', - action='browse_tools', - cntrller='tool', - **kwd ) ) - # Render the list view - return self.category_list_grid( trans, **kwd ) - @web.expose - def browse_tools( self, trans, **kwd ): - # We add params to the keyword dict in this method in order to rename the param - # with an "f-" prefix, simulating filtering by clicking a search link. We have - # to take this approach because the "-" character is illegal in HTTP requests. - if 'operation' not in kwd: - # We may have been redirected here after performing an action. If we were - # redirected from the tool controller, we have to add the default tools_by_category - # operation to kwd so only tools we should see are displayed. This implies that - # all redirectes from the tool controller added the cntrller value to kwd when - # redirecting. - cntrller = kwd.get( 'cntrller', None ) - if cntrller == 'tool': - kwd[ 'operation' ] = 'approved_tools' - if 'operation' in kwd: - operation = kwd['operation'].lower() - if operation == "view_tool": - return trans.response.send_redirect( web.url_for( controller='common', - action='view_tool', - cntrller='tool', - **kwd ) ) - elif operation == "edit_tool": - return trans.response.send_redirect( web.url_for( controller='common', - action='edit_tool', - cntrller='tool', - **kwd ) ) - elif operation == "download tool": - return trans.response.send_redirect( web.url_for( controller='common', - action='download_tool', - cntrller='tool', - **kwd ) ) - elif operation == "tools_by_user": - # Eliminate the current filters if any exist. - for k, v in kwd.items(): - if k.startswith( 'f-' ): - del kwd[ k ] - if 'user_id' in kwd: - user = get_user( trans, kwd[ 'user_id' ] ) - kwd[ 'f-email' ] = user.email - del kwd[ 'user_id' ] - else: - # The received id is the tool id, so we need to get the id of the user - # that uploaded the tool. - tool_id = kwd.get( 'id', None ) - tool = get_tool( trans, tool_id ) - kwd[ 'f-email' ] = tool.user.email - elif operation == "my_tools": - # Eliminate the current filters if any exist. - for k, v in kwd.items(): - if k.startswith( 'f-' ): - del kwd[ k ] - kwd[ 'f-email' ] = trans.user.email - return self.my_tools_list_grid( trans, **kwd ) - elif operation == "approved_tools": - # Eliminate the current filters if any exist. - for k, v in kwd.items(): - if k.startswith( 'f-' ): - del kwd[ k ] - # Make sure only the latest version of a tool whose state is APPROVED are displayed. - kwd[ 'f-state' ] = trans.model.Tool.states.APPROVED - return self.tool_list_grid( trans, **kwd ) - elif operation == "tools_by_category": - # Eliminate the current filters if any exist. - for k, v in kwd.items(): - if k.startswith( 'f-' ): - del kwd[ k ] - category_id = kwd.get( 'id', None ) - category = get_category( trans, category_id ) - kwd[ 'f-Category.name' ] = category.name - # Make sure only the latest version of a tool whose state is APPROVED are displayed. - kwd[ 'f-state' ] = trans.model.Tool.states.APPROVED - # Render the list view - return self.tool_list_grid( trans, **kwd ) - @web.expose - def view_tool_file( self, trans, **kwd ): - params = util.Params( kwd ) - id = params.get( 'id', None ) - if not id: - return trans.response.send_redirect( web.url_for( controller='tool', - action='browse_tools', - cntrller='tool', - message='Select a tool to download', - status='error' ) ) - tool = get_tool( trans, id ) - tool_file_name = urllib.unquote_plus( kwd['file_name'] ) - tool_file = tarfile.open( tool.file_name ).extractfile( tool_file_name ) - trans.response.set_content_type( 'text/plain' ) - return tool_file - @web.expose - def help( self, trans, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - return trans.fill_template( '/webapps/community/tool/help.mako', message=message, status=status, **kwd ) diff --git a/lib/galaxy/webapps/community/controllers/tool_upload.py b/lib/galaxy/webapps/community/controllers/tool_upload.py deleted file mode 100644 index 0c4c45e7cab..00000000000 --- a/lib/galaxy/webapps/community/controllers/tool_upload.py +++ /dev/null @@ -1,183 +0,0 @@ -import sys, os, shutil, logging, urllib2 -from galaxy.web.base.controller import * -from galaxy.web.framework.helpers import time_ago, iff, grids -from galaxy.model.orm import * -from galaxy.web.form_builder import SelectField, build_select_field -from galaxy.webapps.community import datatypes -from common import get_categories, get_category, get_versions - -log = logging.getLogger( __name__ ) - -# States for passing messages -SUCCESS, INFO, WARNING, ERROR = "done", "info", "warning", "error" - -class UploadError( Exception ): - pass - -class ToolUploadController( BaseController ): - - @web.expose - @web.require_login( 'upload', use_panels=True, webapp='community' ) - def upload( self, trans, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - category_ids = util.listify( params.get( 'category_id', '' ) ) - replace_id = params.get( 'replace_id', None ) - if replace_id: - replace_version = trans.sa_session.query( trans.app.model.Tool ).get( trans.security.decode_id( replace_id ) ) - upload_type = replace_version.type - else: - replace_version = None - upload_type = params.get( 'upload_type', 'tool' ) - uploaded_file = None - categories = get_categories( trans ) - if not categories: - message = 'No categories have been configured in this instance of the Galaxy Tool Shed. ' + \ - 'An administrator needs to create some via the Administrator control panel before anything can be uploaded', - status = 'error' - return trans.response.send_redirect( web.url_for( controller='tool', - action='browse_tools', - cntrller='tool', - message=message, - status=status ) ) - if params.get( 'upload_button', False ): - url_paste = params.get( 'url', '' ).strip() - file_data = params.get( 'file_data', '' ) - if file_data == '' and url_paste == '': - message = 'No files were entered on the upload form.' - status = 'error' - elif file_data == '': - try: - uploaded_file = urllib2.urlopen( url_paste ) - except ( ValueError, urllib2.HTTPError ), e: - message = 'An error occurred trying to retrieve the URL entered on the upload form: %s' % str( e ) - status = 'error' - except urllib2.URLError, e: - message = 'An error occurred trying to retrieve the URL entered on the upload form: %s' % e.reason - status = 'error' - elif file_data not in ( '', None ): - uploaded_file = file_data.file - if uploaded_file: - datatype = trans.app.datatypes_registry.get_datatype_by_extension( upload_type ) - if datatype is None: - message = 'An unknown file type was selected. This should not be possible, please report the error.' - status = 'error' - else: - try: - # Initialize the tool object - meta = datatype.verify( uploaded_file ) - meta.user = trans.user - meta.guid = trans.app.security.get_new_guid() - meta.suite = upload_type == 'toolsuite' - obj = datatype.create_model_object( meta ) - trans.sa_session.add( obj ) - if isinstance( obj, trans.app.model.Tool ): - existing = trans.sa_session.query( trans.app.model.Tool ) \ - .filter_by( tool_id = meta.id ) \ - .first() - if existing and not replace_id: - raise UploadError( 'A %s with the same Id already exists. If you are trying to update this %s to a new version, use the upload form on the "Edit Tool" page. Otherwise, change the Id in the %s config.' % \ - ( obj.label, obj.label, obj.label ) ) - elif replace_id and not existing: - raise UploadError( 'The new %s id (%s) does not match the old %s id (%s). Check the %s config files.' % \ - ( obj.label, str( meta.id ), obj.label, str( replace_version.tool_id ), obj.label ) ) - elif existing and replace_id: - if replace_version.newer_version: - # If the user has picked an old version, switch to the newest version - replace_version = get_versions( replace_version )[0] - if replace_version.tool_id != meta.id: - raise UploadError( 'The new %s id (%s) does not match the old %s id (%s). Check the %s config files.' % \ - ( obj.label, str( meta.id ), obj.label, str( replace_version.tool_id ), obj.label ) ) - for old_version in get_versions( replace_version ): - if old_version.version == meta.version: - raise UploadError( 'The new version (%s) matches an old version. Check your version in the %s config file.' % \ - ( str( meta.version ), obj.label ) ) - if old_version.is_new: - raise UploadError( 'There is an existing version of this %s which has not yet been submitted for approval, so either submit it or delete it before uploading a new version.' % \ - ( obj.label, - url_for( controller='common', - action='view_tool', - cntrller='tool', - id=trans.security.encode_id( old_version.id ) ) ) ) - if old_version.is_waiting: - raise UploadError( 'There is an existing version of this %s which is waiting for administrative approval, so contact an administrator for help.' % \ - obj.label ) - # Defer setting the id since the newer version id doesn't exist until the new Tool object is flushed - if category_ids: - for category_id in category_ids: - category = trans.app.model.Category.get( trans.security.decode_id( category_id ) ) - # Initialize the tool category - tca = trans.app.model.ToolCategoryAssociation( obj, category ) - trans.sa_session.add( tca ) - # Initialize the tool event - event = trans.app.model.Event( state=trans.app.model.Tool.states.NEW ) - # Flush to get an event id - trans.sa_session.add( event ) - trans.sa_session.flush() - tea = trans.app.model.ToolEventAssociation( obj, event ) - trans.sa_session.add( tea ) - if replace_version and replace_id: - replace_version.newer_version_id = obj.id - trans.sa_session.add( replace_version ) - # TODO: should the state be changed to archived? We'll leave it alone for now - # because if the newer version is deleted, we'll need to add logic to reset the - # the older version back to it's previous state ( possible approved ). - comment = "Replaced by new version %s" % obj.version - event = trans.app.model.Event( state=replace_version.state, comment=comment ) - # Flush to get an event id - trans.sa_session.add( event ) - trans.sa_session.flush() - tea = trans.app.model.ToolEventAssociation( replace_version, event ) - trans.sa_session.flush() - try: - os.link( uploaded_file.name, obj.file_name ) - except OSError: - shutil.copy( uploaded_file.name, obj.file_name ) - # We're setting cntrller to 'tool' since that is the only controller from which we can upload - # TODO: this will need tweaking when we can upload histories or workflows - return trans.response.send_redirect( web.url_for( controller='common', - action='edit_tool', - cntrller='tool', - id=trans.app.security.encode_id( obj.id ), - message='Uploaded %s' % meta.message, - status='done' ) ) - except ( datatypes.DatatypeVerificationError, UploadError ), e: - message = str( e ) - status = 'error' - uploaded_file.close() - elif replace_id is not None: - old_version = None - for old_version in get_versions( replace_version ): - if old_version.is_new: - message = 'There is an existing version of this tool which has not been submitted for approval, so either submit or delete it before uploading a new version.' - break - if old_version.is_waiting: - message = 'There is an existing version of this tool which is waiting for administrative approval, so contact an administrator for help.' - break - else: - old_version = None - if old_version is not None: - return trans.response.send_redirect( web.url_for( controller='common', - action='view_tool', - cntrller='tool', - id=trans.app.security.encode_id( old_version.id ), - message=message, - status='error' ) ) - selected_categories = [ trans.security.decode_id( id ) for id in category_ids ] - datatype_extensions = trans.app.datatypes_registry.get_datatype_extensions() - upload_type_select_list = build_select_field( trans, - objs=datatype_extensions, - label_attr='self', - select_field_name='upload_type', - initial_value=upload_type, - selected_value=upload_type, - refresh_on_change=True ) - return trans.fill_template( '/webapps/community/upload/upload.mako', - message=message, - status=status, - selected_upload_type=upload_type, - upload_type_select_list=upload_type_select_list, - replace_id=replace_id, - selected_categories=selected_categories, - categories=get_categories( trans ) ) diff --git a/lib/galaxy/webapps/community/controllers/upload.py b/lib/galaxy/webapps/community/controllers/upload.py index be9c32aaa99..bc206ac6693 100644 --- a/lib/galaxy/webapps/community/controllers/upload.py +++ b/lib/galaxy/webapps/community/controllers/upload.py @@ -1,18 +1,20 @@ import sys, os, shutil, logging, tarfile, tempfile from galaxy.web.base.controller import * from galaxy.model.orm import * -from common import get_categories, get_repository -from mercurial import hg, ui +from galaxy.datatypes.checkers import * +from common import * +from mercurial import hg, ui, commands log = logging.getLogger( __name__ ) # States for passing messages SUCCESS, INFO, WARNING, ERROR = "done", "info", "warning", "error" +CHUNK_SIZE = 2**20 # 1Mb class UploadError( Exception ): pass -class UploadController( BaseController ): +class UploadController( BaseUIController ): @web.expose @web.require_login( 'upload', use_panels=True, webapp='community' ) def upload( self, trans, **kwd ): @@ -25,9 +27,260 @@ class UploadController( BaseController ): repository_id = params.get( 'repository_id', '' ) repository = get_repository( trans, repository_id ) repo_dir = repository.repo_path - repo = hg.repository( ui.ui(), repo_dir ) + repo = hg.repository( get_configured_ui(), repo_dir ) + uncompress_file = util.string_as_bool( params.get( 'uncompress_file', 'true' ) ) + remove_repo_files_not_in_tar = util.string_as_bool( params.get( 'remove_repo_files_not_in_tar', 'true' ) ) uploaded_file = None - upload_point = params.get( 'upload_point', None ) + upload_point = self.__get_upload_point( repository, **kwd ) + # Get the current repository tip. + tip = repository.tip + if params.get( 'upload_button', False ): + current_working_dir = os.getcwd() + file_data = params.get( 'file_data', '' ) + if file_data == '': + message = 'No files were entered on the upload form.' + status = 'error' + uploaded_file = None + elif file_data not in ( '', None ): + uploaded_file = file_data.file + uploaded_file_name = uploaded_file.name + uploaded_file_filename = file_data.filename + isempty = os.path.getsize( os.path.abspath( uploaded_file_name ) ) == 0 + if uploaded_file: + isgzip = False + isbz2 = False + if uncompress_file: + isgzip = is_gzip( uploaded_file_name ) + if not isgzip: + isbz2 = is_bz2( uploaded_file_name ) + ok = True + if isempty: + tar = None + istar = False + else: + # Determine what we have - a single file or an archive + try: + if ( isgzip or isbz2 ) and uncompress_file: + # Open for reading with transparent compression. + tar = tarfile.open( uploaded_file_name, 'r:*' ) + else: + tar = tarfile.open( uploaded_file_name ) + istar = True + except tarfile.ReadError, e: + tar = None + istar = False + if istar: + ok, message, files_to_remove = self.upload_tar( trans, + repository, + tar, + uploaded_file, + upload_point, + remove_repo_files_not_in_tar, + commit_message ) + else: + if ( isgzip or isbz2 ) and uncompress_file: + uploaded_file_filename = self.uncompress( repository, uploaded_file_name, uploaded_file_filename, isgzip, isbz2 ) + if upload_point is not None: + full_path = os.path.abspath( os.path.join( repo_dir, upload_point, uploaded_file_filename ) ) + else: + full_path = os.path.abspath( os.path.join( repo_dir, uploaded_file_filename ) ) + # Move the uploaded file to the load_point within the repository hierarchy. + shutil.move( uploaded_file_name, full_path ) + commands.add( repo.ui, repo, full_path ) + try: + commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message ) + except Exception, e: + # I never have a problem with commands.commit on a Mac, but in the test/production + # tool shed environment, it occasionally throws a "TypeError: array item must be char" + # exception. If this happens, we'll try the following. + repo.dirstate.write() + repo.commit( user=trans.user.username, text=commit_message ) + if full_path.endswith( 'tool_data_table_conf.xml.sample' ): + # Handle the special case where a tool_data_table_conf.xml.sample + # file is being uploaded by parsing the file and adding new entries + # to the in-memory trans.app.tool_data_tables dictionary as well as + # appending them to the shed's tool_data_table_conf.xml file on disk. + error, error_message = handle_sample_tool_data_table_conf_file( trans, full_path ) + if error: + message = '%s
    %s' % ( message, error_message ) + if full_path.endswith( '.loc.sample' ): + # Handle the special case where a xxx.loc.sample file is + # being uploaded by copying it to ~/tool-data/xxx.loc. + copy_sample_loc_file( trans, full_path ) + handle_email_alerts( trans, repository ) + if ok: + # Update the repository files for browsing. + update_for_browsing( trans, repository, current_working_dir, commit_message=commit_message ) + # Get the new repository tip. + if tip != repository.tip: + if ( isgzip or isbz2 ) and uncompress_file: + uncompress_str = ' uncompressed and ' + else: + uncompress_str = ' ' + message = "The file '%s' has been successfully%suploaded to the repository." % ( uploaded_file_filename, uncompress_str ) + if istar and remove_repo_files_not_in_tar and files_to_remove: + if upload_point is not None: + message += " %d files were removed from the repository relative to the selected upload point '%s'." % ( len( files_to_remove ), upload_point ) + else: + message += " %d files were removed from the repository root." % len( files_to_remove ) + else: + message = 'No changes to repository.' + # Set metadata on the repository tip + error_message, status = set_repository_metadata( trans, repository_id, repository.tip, **kwd ) + if error_message: + message = '%s
    %s' % ( message, error_message ) + return trans.response.send_redirect( web.url_for( controller='repository', + action='manage_repository', + id=repository_id, + message=message, + status=status ) ) + trans.response.send_redirect( web.url_for( controller='repository', + action='browse_repository', + id=repository_id, + commit_message='Deleted selected files', + message=message, + status=status ) ) + else: + status = 'error' + selected_categories = [ trans.security.decode_id( id ) for id in category_ids ] + return trans.fill_template( '/webapps/community/repository/upload.mako', + repository=repository, + commit_message=commit_message, + uncompress_file=uncompress_file, + remove_repo_files_not_in_tar=remove_repo_files_not_in_tar, + message=message, + status=status ) + def upload_tar( self, trans, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message ): + # Upload a tar archive of files. + repo_dir = repository.repo_path + repo = hg.repository( get_configured_ui(), repo_dir ) + files_to_remove = [] + ok, message = self.__check_archive( tar ) + if not ok: + tar.close() + uploaded_file.close() + return ok, message, files_to_remove + else: + if upload_point is not None: + full_path = os.path.abspath( os.path.join( repo_dir, upload_point ) ) + else: + full_path = os.path.abspath( repo_dir ) + filenames_in_archive = [ tarinfo_obj.name for tarinfo_obj in tar.getmembers() ] + filenames_in_archive = [ os.path.join( full_path, name ) for name in filenames_in_archive ] + # Extract the uploaded tar to the load_point within the repository hierarchy. + tar.extractall( path=full_path ) + tar.close() + uploaded_file.close() + if remove_repo_files_not_in_tar and not repository.is_new: + # We have a repository that is not new (it contains files), so discover + # those files that are in the repository, but not in the uploaded archive. + for root, dirs, files in os.walk( full_path ): + if not root.find( '.hg' ) >= 0 and not root.find( 'hgrc' ) >= 0: + if '.hg' in dirs: + # Don't visit .hg directories - should be impossible since we don't + # allow uploaded archives that contain .hg dirs, but just in case... + dirs.remove( '.hg' ) + if 'hgrc' in files: + # Don't include hgrc files in commit. + files.remove( 'hgrc' ) + for name in files: + full_name = os.path.join( root, name ) + if full_name not in filenames_in_archive: + files_to_remove.append( full_name ) + for repo_file in files_to_remove: + # Remove files in the repository (relative to the upload point) + # that are not in the uploaded archive. + try: + commands.remove( repo.ui, repo, repo_file, force=True ) + except Exception, e: + # I never have a problem with commands.remove on a Mac, but in the test/production + # tool shed environment, it throws an exception whenever I delete all files from a + # repository. If this happens, we'll try the following. + relative_selected_file = selected_file.split( 'repo_%d' % repository.id )[1].lstrip( '/' ) + repo.dirstate.remove( relative_selected_file ) + repo.dirstate.write() + absolute_selected_file = os.path.abspath( selected_file ) + if os.path.isdir( absolute_selected_file ): + try: + os.rmdir( absolute_selected_file ) + except OSError, e: + # The directory is not empty + pass + elif os.path.isfile( absolute_selected_file ): + os.remove( absolute_selected_file ) + dir = os.path.split( absolute_selected_file )[0] + try: + os.rmdir( dir ) + except OSError, e: + # The directory is not empty + pass + for filename_in_archive in filenames_in_archive: + commands.add( repo.ui, repo, filename_in_archive ) + if filename_in_archive.endswith( 'tool_data_table_conf.xml.sample' ): + # Handle the special case where a tool_data_table_conf.xml.sample + # file is being uploaded by parsing the file and adding new entries + # to the in-memory trans.app.tool_data_tables dictionary as well as + # appending them to the shed's tool_data_table_conf.xml file on disk. + error, message = handle_sample_tool_data_table_conf_file( trans, filename_in_archive ) + if error: + return False, message, files_to_remove + if filename_in_archive.endswith( '.loc.sample' ): + # Handle the special case where a xxx.loc.sample file is + # being uploaded by copying it to ~/tool-data/xxx.loc. + copy_sample_loc_file( trans, filename_in_archive ) + try: + commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message ) + except Exception, e: + # I never have a problem with commands.commit on a Mac, but in the test/production + # tool shed environment, it occasionally throws a "TypeError: array item must be char" + # exception. If this happens, we'll try the following. + repo.dirstate.write() + repo.commit( user=trans.user.username, text=commit_message ) + handle_email_alerts( trans, repository ) + return True, '', files_to_remove + def uncompress( self, repository, uploaded_file_name, uploaded_file_filename, isgzip, isbz2 ): + if isgzip: + self.__handle_gzip( repository, uploaded_file_name ) + return uploaded_file_filename.rstrip( '.gz' ) + if isbz2: + self.__handle_bz2( repository, uploaded_file_name ) + return uploaded_file_filename.rstrip( '.bz2' ) + def __handle_gzip( self, repository, uploaded_file_name ): + fd, uncompressed = tempfile.mkstemp( prefix='repo_%d_upload_gunzip_' % repository.id, dir=os.path.dirname( uploaded_file_name ), text=False ) + gzipped_file = gzip.GzipFile( uploaded_file_name, 'rb' ) + while 1: + try: + chunk = gzipped_file.read( CHUNK_SIZE ) + except IOError, e: + os.close( fd ) + os.remove( uncompressed ) + log.exception( 'Problem uncompressing gz data "%s": %s' % ( uploaded_file_name, str( e ) ) ) + return + if not chunk: + break + os.write( fd, chunk ) + os.close( fd ) + gzipped_file.close() + shutil.move( uncompressed, uploaded_file_name ) + def __handle_bz2( self, repository, uploaded_file_name ): + fd, uncompressed = tempfile.mkstemp( prefix='repo_%d_upload_bunzip2_' % repository.id, dir=os.path.dirname( uploaded_file_name ), text=False ) + bzipped_file = bz2.BZ2File( uploaded_file_name, 'rb' ) + while 1: + try: + chunk = bzipped_file.read( CHUNK_SIZE ) + except IOError: + os.close( fd ) + os.remove( uncompressed ) + log.exception( 'Problem uncompressing bz2 data "%s": %s' % ( uploaded_file_name, str( e ) ) ) + return + if not chunk: + break + os.write( fd, chunk ) + os.close( fd ) + bzipped_file.close() + shutil.move( uncompressed, uploaded_file_name ) + def __get_upload_point( self, repository, **kwd ): + upload_point = kwd.get( 'upload_point', None ) if upload_point is not None: # The value of upload_point will be something like: database/community_files/000/repo_12/1.bed if os.path.exists( upload_point ): @@ -44,270 +297,8 @@ class UploadController( BaseController ): upload_point = None else: # Must have been an error selecting something that didn't exist, so default to repository root - # TODO: throw an exception???? upload_point = None - else: - # Default to repository root - upload_point = None - if params.get( 'upload_button', False ): - ctx = repo.changectx( "tip" ) - current_working_dir = os.getcwd() - file_data = params.get( 'file_data', '' ) - if file_data == '': - message = 'No files were entered on the upload form.' - status = 'error' - uploaded_file = None - elif file_data not in ( '', None ): - uploaded_file = file_data.file - if uploaded_file: - # TODO: our current support for browsing repo contents requires a copy - # of the repository files in the repo root directory. To produce these - # copies, we update without passing the -r null flag (see below). When - # we're uploading more files, we have to clean out the repo root directory - # so we can move them into it. We need to eliminate all this when we figure - # out how to browse the repository files. - os.chdir( repo_dir ) - os.system( 'hg update -r null > /dev/null 2>&1' ) - os.chdir( current_working_dir ) - ok = True - files_to_commit = [] - # Determine what we have - a single file or an archive - try: - tar = tarfile.open( uploaded_file.name ) - istar = True - except tarfile.ReadError, e: - istar = False - if istar: - ok, message = self.__check_archive( tar ) - if ok: - if repository.is_new: - tar.extractall( path=repo_dir ) - tar.close() - uploaded_file.close() - # TODO: The following will only work on new, empty repos, - # need to also handle repos with existing contents - for root, dirs, files in os.walk( repo_dir, topdown=False ): - # Don't visit .hg directories and don't include hgrc files in commit. - if not root.find( '.hg' ) >= 0 and not root.find( 'hgrc' ) >= 0: - if '.hg' in dirs: - # Don't visit .hg directories - dirs.remove( '.hg' ) - if 'hgrc' in files: - # Don't include hgrc files in commit - should be impossible - # since we don't visit .hg dirs, but just in case... - files.remove( 'hgrc' ) - for name in files: - relative_root = root.split( 'repo_%d' % repository.id )[ 1 ].lstrip ( '/' ) - if upload_point is not None: - file_path = os.path.join( relative_root, upload_point, name ) - else: - file_path = os.path.join( relative_root, name ) - # Check if the file is tracked and make it tracked if not. - repo_contains = file_path in [ i for i in ctx.manifest() ] - if not repo_contains: - # Add the file to the dirstate - repo.dirstate.add( file_path ) - files_to_commit.append( file_path ) - else: - # The repo already contains the file, so we need to make sure the file being - # uploaded is different from the file in the repo. This is a temporary brute- - # force method. - # Make a clone of the repository in a temporary location - tmp_dir = tempfile.mkdtemp() - tmp_archive_dir = os.path.join( tmp_dir, 'tmp_archive_dir' ) - if not os.path.exists( tmp_archive_dir ): - os.makedirs( tmp_archive_dir ) - cmd = "hg clone %s > /dev/null 2>&1" % os.path.abspath( repo_dir ) - os.chdir( tmp_archive_dir ) - os.system( cmd ) - os.chdir( current_working_dir ) - cloned_repo_dir = os.path.join( tmp_archive_dir, 'repo_%d' % repository.id ) - if upload_point is not None: - full_path = os.path.abspath( os.path.join( cloned_repo_dir, upload_point, file_data.filename ) ) - else: - full_path = os.path.abspath( os.path.join( cloned_repo_dir, file_data.filename ) ) - # Extract the uploaded tarball to the load_point within the cloned repository hierarchy - tar.extractall( path=full_path ) - tar.close() - uploaded_file.close() - # We want these change sets to be associated with the owner of the repository, so we'll - # set the HGUSER environment variable accordingly. - os.environ[ 'HGUSER' ] = trans.user.username - # Add the file to the cloned repository. If it's already tracked, this should do nothing. - os.chdir( cloned_repo_dir ) - os.system( 'hg add > /dev/null 2>&1' ) - os.chdir( current_working_dir ) - os.chdir( cloned_repo_dir ) - # Commit the change set to the cloned repository - os.system( "hg commit -m '%s' > /dev/null 2>&1" % commit_message ) - os.chdir( current_working_dir ) - # Push the change set to the master repository - cmd = "hg push %s > /dev/null 2>&1" % os.path.abspath( repo_dir ) - os.chdir( cloned_repo_dir ) - os.system( cmd ) - # Change the current working directory to the original - os.chdir( current_working_dir ) - # Since we extracted the archive into repo_dir, a copy of the archive's - # files remains there. The following will remove them. It would be - # more ideal if we could use the mercurial api to do this, but I haven't - # yet discovered a way to pass the -r null flag to repo.update(). - # TODO: our current support for browsing repo contents requires a copy - # of the repository files in the repo root directory. To produce these - # copies, we'll update without passing the -r null flag. When we figure - # out how to browse the repository files, uncomment the -r flag below. - os.chdir( repo_dir ) - os.system( 'hg update > /dev/null 2>&1' ) - os.chdir( current_working_dir ) - # Remove tmp directory - shutil.rmtree( tmp_dir ) - message = "The file '%s' has been successfully uploaded to the repository." % file_data.filename - trans.response.send_redirect( web.url_for( controller='repository', - action='browse_repository', - message=message, - id=trans.security.encode_id( repository.id ) ) ) - - - else: - tar.close() - else: - """ - # TODO: This segment uses the mercurial api (and works), but we need the - # api section below to be functional in order for this segment to be used. - # In the meantime, we use the repository.is_new check below... - repo_contains = file_path in [ i for i in ctx.manifest() ] - if not repo_contains: - repo.dirstate.add( file_path ) - files_to_commit.append( file_path ) - """ - if repository.is_new: - # We're uploading a single file - if upload_point is not None: - full_path = os.path.abspath( os.path.join( upload_point, file_data.filename ) ) - file_path = os.path.join( upload_point, file_data.filename ) - else: - full_path = os.path.abspath( os.path.join( repo_dir, file_data.filename ) ) - file_path = os.path.join( file_data.filename ) - shutil.move( uploaded_file.name, full_path ) - repo.dirstate.add( file_path ) - files_to_commit.append( file_path ) - else: - """ - # TODO: This segment attempts to use the mercurial api, but is not functional. - # Until we get it working, we're using the brute force method below it. - fctx = None - for changeset in repo.changelog: - ctx = repo.changectx( changeset ) - if file_path not in ctx.files(): - continue - fctx = ctx[ file_path ] - break - # We now have the parent version of the upload file. - if fctx: - data = fctx.data() - # TODO: obviously very bad way of comparing files... - file_path_data = open( full_path ).read() - different = data != file_path_data - if different: - # TODO: how do you insert a new version of an existing file using the mercurial api??? - # the follwoin gis not correct! - #repo.dirstate.normallookup( file_path ) - #files_to_commit.append( file_path ) - pass - """ - # The repo already contains the file, so we need to make sure the file being - # uploaded is different from the file in the repo. This is a temporary brute- - # force method. - # Make a clone of the repository in a temporary location - tmp_dir = tempfile.mkdtemp() - tmp_archive_dir = os.path.join( tmp_dir, 'tmp_archive_dir' ) - if not os.path.exists( tmp_archive_dir ): - os.makedirs( tmp_archive_dir ) - cmd = "hg clone %s > /dev/null 2>&1" % os.path.abspath( repo_dir ) - os.chdir( tmp_archive_dir ) - os.system( cmd ) - os.chdir( current_working_dir ) - cloned_repo_dir = os.path.join( tmp_archive_dir, 'repo_%d' % repository.id ) - if upload_point is not None: - full_path = os.path.abspath( os.path.join( cloned_repo_dir, upload_point, file_data.filename ) ) - else: - full_path = os.path.abspath( os.path.join( cloned_repo_dir, file_data.filename ) ) - # Move the uploaded file to the load_point within the cloned repository hierarchy - shutil.move( uploaded_file.name, full_path ) - # We want these change sets to be associated with the owner of the repository, so we'll - # set the HGUSER environment variable accordingly. - os.environ[ 'HGUSER' ] = trans.user.username - # Add the file to the cloned repository. If it's already tracked, this should do nothing. - os.chdir( cloned_repo_dir ) - os.system( 'hg add > /dev/null 2>&1' ) - os.chdir( current_working_dir ) - os.chdir( cloned_repo_dir ) - # Commit the change set to the cloned repository - os.system( "hg commit -m '%s' > /dev/null 2>&1" % commit_message ) - os.chdir( current_working_dir ) - # Push the change set to the master repository - cmd = "hg push %s > /dev/null 2>&1" % os.path.abspath( repo_dir ) - os.chdir( cloned_repo_dir ) - os.system( cmd ) - os.chdir( current_working_dir ) - # Since we extracted the archive into repo_dir, a copy of the archive's - # files remains there. The following will remove them. It would be - # more ideal if we could use the mercurial api to do this, but I haven't - # yet discovered a way to pass the -r null flag to repo.update(). - # TODO: our current support for browsing repo contents requires a copy - # of the repository files in the repo root directory. To produce these - # copies, we'll update without passing the -r null flag. When we figure - # out how to browse the repository files, uncomment the -r flag below. - os.chdir( repo_dir ) - os.system( 'hg update > /dev/null 2>&1' ) - os.chdir( current_working_dir ) - # Remove tmp directory - shutil.rmtree( tmp_dir ) - message = "The file '%s' has been successfully uploaded to the repository." % file_data.filename - trans.response.send_redirect( web.url_for( controller='repository', - action='browse_repository', - message=message, - id=trans.security.encode_id( repository.id ) ) ) - if ok: - if files_to_commit: - repo.dirstate.write() - repo.commit( text=commit_message ) - # Since we extracted the archive into repo_dir, a copy of the archive's - # files remains there. The following will remove them. It would be - # more ideal if we could use the mercurial api to do this, but I haven't - # yet discovered a way to pass the -r null flag to repo.update(). - # TODO: our current support for browsing repo contents requires a copy - # of the repository files in the repo root directory. To produce these - # copies, we'll update without passing the -r null flag. When we figure - # out how to browse the repository files, uncomment the -r flag below. - os.chdir( repo_dir ) - os.system( 'hg update > /dev/null 2>&1' ) - #os.system( 'hg update -r null' ) - os.chdir( current_working_dir ) - message = "The file '%s' has been successfully uploaded to the repository." % file_data.filename - trans.response.send_redirect( web.url_for( controller='repository', - action='browse_repository', - message=message, - id=trans.security.encode_id( repository.id ) ) ) - else: - status = 'error' - # Since we extracted the archive into repo_dir, a copy of the archive's - # files remains there. The following will remove them. It would be - # more ideal if we could use the mercurial api to do this, but I haven't - # yet discovered a way to pass the -r null flag to repo.update(). - # TODO: our current support for browsing repo contents requires a copy - # of the repository files in the repo root directory. To produce these - # copies, we'll update without passing the -r null flag. When we figure - # out how to browse the repository files, uncomment the -r flag below. - os.chdir( repo_dir ) - os.system( 'hg update > /dev/null 2>&1' ) - #os.system( 'hg update -r null' ) - os.chdir( current_working_dir ) - selected_categories = [ trans.security.decode_id( id ) for id in category_ids ] - return trans.fill_template( '/webapps/community/repository/upload.mako', - repository=repository, - commit_message=commit_message, - message=message, - status=status ) + return upload_point def __check_archive( self, archive ): for member in archive.getmembers(): # Allow regular files and directories only @@ -322,4 +313,4 @@ class UploadController( BaseController ): message = "Uploaded archives cannot contain hgrc files." return False, message return True, '' - \ No newline at end of file + diff --git a/lib/galaxy/webapps/community/datatypes/__init__.py b/lib/galaxy/webapps/community/datatypes/__init__.py deleted file mode 100644 index 3b6e0a0413b..00000000000 --- a/lib/galaxy/webapps/community/datatypes/__init__.py +++ /dev/null @@ -1,196 +0,0 @@ -import sys, logging, tarfile -from galaxy.util import parse_xml -from galaxy.util.bunch import Bunch - -log = logging.getLogger( __name__ ) - -if sys.version_info[:2] == ( 2, 4 ): - from galaxy import eggs - eggs.require( 'ElementTree' ) - from elementtree import ElementTree -else: - from xml.etree import ElementTree - -class DatatypeVerificationError( Exception ): - pass - -class Registry( object ): - def __init__( self, root_dir=None, config=None ): - self.datatypes_by_extension = {} - if root_dir and config: - # Parse datatypes_conf.xml - tree = parse_xml( config ) - root = tree.getroot() - # Load datatypes and converters from config - log.debug( 'Loading datatypes from %s' % config ) - registration = root.find( 'registration' ) - for elem in registration.findall( 'datatype' ): - try: - extension = elem.get( 'extension', None ) - dtype = elem.get( 'type', None ) - model_object = elem.get( 'model', None ) - if extension and dtype: - fields = dtype.split( ':' ) - datatype_module = fields[0] - datatype_class = fields[1] - fields = datatype_module.split( '.' ) - module = __import__( fields.pop(0) ) - for mod in fields: - module = getattr( module, mod ) - self.datatypes_by_extension[extension] = getattr( module, datatype_class )() - log.debug( 'Loaded datatype: %s' % dtype ) - if model_object: - model_module, model_class = model_object.split( ':' ) - fields = model_module.split( '.' ) - module = __import__( fields.pop(0) ) - for mod in fields: - module = getattr( module, mod ) - self.datatypes_by_extension[extension].model_object = getattr( module, model_class ) - log.debug( 'Added model class: %s to datatype: %s' % ( model_class, dtype ) ) - except Exception, e: - log.warning( 'Error loading datatype "%s", problem: %s' % ( extension, str( e ) ) ) - def get_datatype_by_extension( self, ext ): - return self.datatypes_by_extension.get( ext, None ) - def get_datatype_extensions( self ): - rval = [] - for ext, datatype in self.datatypes_by_extension.items(): - rval.append( ext ) - return rval - -class Tool( object ): - def __init__( self, model_object=None ): - self.model_object = model_object - self.label = 'Tool' - def verify( self, f, xml_files=[], tool_tags={} ): - # xml_files and tool_tags will only be received if we're called from the ToolSuite.verify() method. - try: - tar = tarfile.open( f.name ) - except tarfile.ReadError, e: - raise DatatypeVerificationError( 'Error reading the archive, problem: %s' % str( e ) ) - if not xml_files: - # Make sure we're not uploading a tool suite - if filter( lambda x: x.lower().find( 'suite_config.xml' ) >= 0, tar.getnames() ): - raise DatatypeVerificationError( 'The archive includes a suite_config.xml file, so set the upload type to "Tool Suite".' ) - xml_files = filter( lambda x: x.lower().endswith( '.xml' ), tar.getnames() ) - if not xml_files: - raise DatatypeVerificationError( 'The archive does not contain any xml config files.' ) - for xml_file in xml_files: - try: - tree = ElementTree.parse( tar.extractfile( xml_file ) ) - root = tree.getroot() - except Exception, e: - raise DatatypeVerificationError( 'Error parsing file "%s", problem: %s' % ( str( xml_file ), str( e ) ) ) - if root.tag == 'tool': - if 'id' not in root.keys(): - raise DatatypeVerificationError( "Tool xml file (%s) does not include the required 'id' attribute in the <tool> tag" % str( xml_file ) ) - if 'name' not in root.keys(): - raise DatatypeVerificationError( "Tool xml file (%s) does not include the required 'name' attribute in the <tool> tag" % str( xml_file ) ) - if 'version' not in root.keys(): - raise DatatypeVerificationError( "Tool xml file (%s) does not include the required 'version' attribute in the <tool> tag" % str( xml_file ) ) - if tool_tags: - # We are verifying the tools inside a tool suite, so the current tag should have been found in the suite_config.xml - # file parsed in the ToolSuite verify() method. The tool_tags dictionary should include a key matching the current - # tool Id, and a tuple value matching the tool name and version. - if root.attrib[ 'id' ] not in tool_tags: - raise DatatypeVerificationError( 'Tool Id (%s) is not included in the suite_config.xml file.' % \ - ( str( root.attrib[ 'id' ] ) ) ) - tup = tool_tags[ root.attrib[ 'id' ] ] - if root.attrib[ 'name' ] != tup[ 0 ]: - raise DatatypeVerificationError( 'Tool name (%s) differs between suite_config.xml and the tool config file for tool Id (%s).' % \ - ( str( root.attrib[ 'name' ] ), str( root.attrib[ 'id' ] ) ) ) - if root.attrib[ 'version' ] != tup[ 1 ]: - raise DatatypeVerificationError( 'Tool version (%s) differs between suite_config.xml and the tool config file for tool Id (%s).' % \ - ( str( root.attrib[ 'version' ] ), str( root.attrib[ 'id' ] ) ) ) - else: - # We are not verifying a tool suite, so we'll create a bunch for returning to the caller. - tool_bunch = Bunch() - try: - tool_bunch.id = root.attrib['id'] - tool_bunch.name = root.attrib['name'] - tool_bunch.version = root.attrib['version'] - except KeyError, e: - raise DatatypeVerificationError( 'Tool XML file does not conform to the specification. Missing required <tool> tag attribute: %s' % str( e ) ) - tool_bunch.description = '' - desc_tag = root.find( 'description' ) - if desc_tag is not None: - description = desc_tag.text - if description: - tool_bunch.description = description.strip() - tool_bunch.message = 'Tool: %s %s, Version: %s, Id: %s' % \ - ( str( tool_bunch.name ), str( tool_bunch.description ), str( tool_bunch.version ), str( tool_bunch.id ) ) - return tool_bunch - else: - # TODO: should we verify files that are not tool configs? - log.debug( "The file named (%s) is not a tool config, so skipping verification." % str( xml_file ) ) - def create_model_object( self, datatype_bunch ): - if self.model_object is None: - raise Exception( 'No model object configured for %s, check the datatype configuration file' % self.__class__.__name__ ) - if datatype_bunch is None: - # TODO: do it automatically - raise Exception( 'Unable to create %s model object without passing in data' % self.__class__.__name__ ) - o = self.model_object() - o.create_from_datatype( datatype_bunch ) - return o - -class ToolSuite( Tool ): - def __init__( self, model_object=None ): - self.model_object = model_object - self.label = 'Tool Suite' - def verify( self, f ): - """ - A sample tool suite config: - - ONTO-Toolkit is a collection of Galaxy tools which support the manipulation of bio-ontologies. - - Collects the ancestor terms from a given term in the given OBO ontology - - - Collects the child terms from a given term in the given OBO ontology - - - """ - try: - tar = tarfile.open( f.name ) - except tarfile.ReadError: - raise DatatypeVerificationError( 'The archive is not a readable tar file.' ) - suite_config = filter( lambda x: x.lower().find( 'suite_config.xml' ) >=0, tar.getnames() ) - if not suite_config: - raise DatatypeVerificationError( 'The archive does not contain the required suite_config.xml config file. If you are uploading a single tool archive, set the upload type to "Tool".' ) - suite_config = suite_config[ 0 ] - # Parse and verify suite_config - archive_ok = False - try: - tree = ElementTree.parse( tar.extractfile( suite_config ) ) - root = tree.getroot() - archive_ok = True - except: - log.exception( 'fail:' ) - if archive_ok and root.tag == 'suite': - suite_bunch = Bunch() - try: - suite_bunch.id = root.attrib['id'] - suite_bunch.name = root.attrib['name'] - suite_bunch.version = root.attrib['version'] - except KeyError, e: - raise DatatypeVerificationError( 'The file named tool-suite.xml does not conform to the specification. Missing required <suite> tag attribute: %s' % str( e ) ) - suite_bunch.description = '' - desc_tag = root.find( 'description' ) - if desc_tag is not None: - description = desc_tag.text - if description: - suite_bunch.description = description.strip() - suite_bunch.message = 'Tool suite: %s %s, Version: %s, Id: %s' % \ - ( str( suite_bunch.name ), str( suite_bunch.description ), str( suite_bunch.version ), str( suite_bunch.id ) ) - # Create a dictionary of the tools in the suite where the keys are tool_ids and the - # values are tuples of tool name and version - tool_tags = {} - for elem in root.findall( 'tool' ): - tool_tags[ elem.attrib['id'] ] = ( elem.attrib['name'], elem.attrib['version'] ) - else: - raise DatatypeVerificationError( "The file named %s is not a valid tool suite config." % str( suite_config ) ) - # Verify all included tool config files - xml_files = filter( lambda x: x.lower().endswith( '.xml' ) and x.lower() != 'suite_config.xml', tar.getnames() ) - if not xml_files: - raise DatatypeVerificationError( 'The archive does not contain any tool config (xml) files.' ) - Tool.verify( self, f, xml_files=xml_files, tool_tags=tool_tags ) - return suite_bunch diff --git a/lib/galaxy/webapps/community/framework/middleware/hg.py b/lib/galaxy/webapps/community/framework/middleware/hg.py index 1c017a2a9e6..176e534dd66 100644 --- a/lib/galaxy/webapps/community/framework/middleware/hg.py +++ b/lib/galaxy/webapps/community/framework/middleware/hg.py @@ -23,14 +23,41 @@ class Hg( object ): self.username = None self.action = None def __call__( self, environ, start_response ): - # Handle authentication for hg push commands cmd = self.__get_hg_command( **environ ) + if cmd == 'changegroup': + # This is an hg clone from the command line. When doing this, the following 5 commands, in order, + # will be retrieved from environ: + # between -> heads -> changegroup -> capabilities -> listkeys + # + # Increment the value of the times_downloaded column in the repository table for the cloned repository. + if 'PATH_INFO' in environ: + path_info = environ[ 'PATH_INFO' ].lstrip( '/' ) + # An example of path_info is: '/repos/test/column1' + path_info_components = path_info.split( '/' ) + username = path_info_components[1] + name = path_info_components[2] + # Instantiate a database connection + db_url = self.config[ 'database_connection' ] + engine = create_engine( db_url ) + connection = engine.connect() + result_set = connection.execute( "select id from galaxy_user where username = '%s'" % username.lower() ) + for row in result_set: + # Should only be 1 row... + user_id = row[ 'id' ] + result_set = connection.execute( "select times_downloaded from repository where user_id = %d and name = '%s'" % ( user_id, name.lower() ) ) + for row in result_set: + # Should only be 1 row... + times_downloaded = row[ 'times_downloaded' ] + times_downloaded += 1 + connection.execute( "update repository set times_downloaded = %d where user_id = %d and name = '%s'" % ( times_downloaded, user_id, name.lower() ) ) + connection.close() if cmd == 'unbundle': + # This is an hg push from the command line. When doing this, the following 7 commands, in order, + # will be retrieved from environ: + # between -> capabilities -> heads -> branchmap -> unbundle -> unbundle -> listkeys + # # The mercurial API unbundle() ( i.e., hg push ) method ultimately requires authorization. - # We'll force password entry every time a change set is pushed. The user that pushes the changes - # sets may not be the same user that committed the change sets. In other words, the user that is - # pushing is the one being authenticated, but the owner of a specific change set in the change log - # may be different. + # We'll force password entry every time a change set is pushed. # # When a user executes hg commit, it is not guaranteed to succeed. Mercurial records your name # and address with each change that you commit, so that you and others will later be able to @@ -40,7 +67,7 @@ class Hg( object ): # 1) If you specify a -u option to the hg commit command on the command line, followed by a username, # this is always given the highest precedence. # 2) If you have set the HGUSER environment variable, this is checked next. - # 3) If you create a file in your home directory called .hgrc (~/.hgrc), with a username entry, that + # 3) If you create a file in your home directory called .hgrc with a username entry, that # will be used next. # 4) If you have set the EMAIL environment variable, this will be used next. # 5) Mercurial will query your system to find out your local user name and host name, and construct diff --git a/lib/galaxy/webapps/community/model/__init__.py b/lib/galaxy/webapps/community/model/__init__.py index 294c72fd890..31d6d24cc00 100644 --- a/lib/galaxy/webapps/community/model/__init__.py +++ b/lib/galaxy/webapps/community/model/__init__.py @@ -4,7 +4,8 @@ Galaxy Tool Shed data model classes Naming: try to use class names that have a distinct plural form so that the relationship cardinalities are obvious (e.g. prefer Dataset to Data) """ -import os.path, os, errno, sys, codecs, operator, tempfile, logging, tarfile, mimetypes, ConfigParser +import os.path, os, errno, sys, codecs, operator, logging, tarfile, mimetypes, ConfigParser +from galaxy import util from galaxy.util.bunch import Bunch from galaxy.util.hash_util import * from galaxy.web.form_builder import * @@ -19,8 +20,6 @@ class User( object ): self.deleted = False self.purged = False self.username = None - # Relationships - self.tools = [] def set_password_cleartext( self, cleartext ): """Set 'self.password' to the digest of 'cleartext'.""" self.password = new_secure_hash( text_type=cleartext ) @@ -90,11 +89,14 @@ class Repository( object ): MARKED_FOR_REMOVAL = 'r', MARKED_FOR_ADDITION = 'a', NOT_TRACKED = '?' ) - def __init__( self, name=None, description=None, user_id=None, private=False ): + def __init__( self, name=None, description=None, long_description=None, user_id=None, private=False, email_alerts=None, times_downloaded=0 ): self.name = name or "Unnamed repository" self.description = description + self.long_description = long_description self.user_id = user_id self.private = private + self.email_alerts = email_alerts + self.times_downloaded = times_downloaded @property def repo_path( self ): # Repository locations on disk are defined in the hgweb.config file @@ -112,168 +114,51 @@ class Repository( object ): return config.get( "paths", option ) raise Exception( "Entry for repository %s missing in %s/hgweb.config file." % ( lhs, os.getcwd() ) ) @property + def revision( self ): + repo = hg.repository( ui.ui(), self.repo_path ) + tip_ctx = repo.changectx( repo.changelog.tip() ) + return "%s:%s" % ( str( tip_ctx.rev() ), str( repo.changectx( repo.changelog.tip() ) ) ) + @property + def tip( self ): + repo = hg.repository( ui.ui(), self.repo_path ) + return str( repo.changectx( repo.changelog.tip() ) ) + @property def is_new( self ): repo = hg.repository( ui.ui(), self.repo_path ) tip_ctx = repo.changectx( repo.changelog.tip() ) return tip_ctx.rev() < 0 -class Tool( object ): - file_path = '/tmp' # Overridden in mapping.__init__() - states = Bunch( NEW = 'new', - ERROR = 'error', - DELETED = 'deleted', - WAITING = 'waiting', - APPROVED = 'approved', - REJECTED = 'rejected', - ARCHIVED = 'archived' ) - def __init__( self, guid=None, tool_id=None, name=None, description=None, user_description=None, - category=None, version=None, user_id=None, external_filename=None, suite=False ): - self.guid = guid - self.tool_id = tool_id - self.name = name or "Unnamed tool" - self.description = description - self.user_description = user_description - self.version = version or "1.0.0" - self.user_id = user_id - self.external_filename = external_filename - self.deleted = False - self.__extension = None - self.suite = suite - def get_file_name( self ): - if not self.external_filename: - assert self.id is not None, "ID must be set before filename used (commit the object)" - dir = os.path.join( self.file_path, 'tools', *directory_hash_id( self.id ) ) - # Create directory if it does not exist - if not os.path.exists( dir ): - os.makedirs( dir ) - # Return filename inside hashed directory - filename = os.path.join( dir, "tool_%d.dat" % self.id ) + @property + def allow_push( self ): + repo = hg.repository( ui.ui(), self.repo_path ) + return repo.ui.config( 'web', 'allow_push' ) + def set_allow_push( self, usernames, remove_auth='' ): + allow_push = util.listify( self.allow_push ) + if remove_auth: + allow_push.remove( remove_auth ) else: - filename = self.external_filename - # Make filename absolute - return os.path.abspath( filename ) - def set_file_name( self, filename ): - if not filename: - self.external_filename = None - else: - self.external_filename = filename - file_name = property( get_file_name, set_file_name ) - def create_from_datatype( self, datatype_bunch ): - # TODO: ensure guid is unique and generate a new one if not. - self.guid = datatype_bunch.guid - self.tool_id = datatype_bunch.id - self.name = datatype_bunch.name - self.description = datatype_bunch.description - self.version = datatype_bunch.version - self.user_id = datatype_bunch.user.id - self.suite = datatype_bunch.suite - @property - def state( self ): - latest_event = self.latest_event - if latest_event: - return latest_event.state - return None - @property - def latest_event( self ): - if self.events: - events = [ tea.event for tea in self.events ] - # Get the last event that occurred ( events mapper is sorted descending ) - return events[0] - return None - # Tool states - @property - def is_new( self ): - return self.state == self.states.NEW - @property - def is_error( self ): - return self.state == self.states.ERROR - @property - def is_deleted( self ): - return self.state == self.states.DELETED - @property - def is_waiting( self ): - return self.state == self.states.WAITING - @property - def is_approved( self ): - return self.state == self.states.APPROVED - @property - def is_rejected( self ): - return self.state == self.states.REJECTED - @property - def is_archived( self ): - return self.state == self.states.ARCHIVED - def get_state_message( self ): - if self.is_suite: - label = 'tool suite' - else: - label = 'tool' - if self.is_new: - return 'This is an unsubmitted version of this %s' % label - if self.is_error: - return 'This %s is in an error state' % label - if self.is_deleted: - return 'This is a deleted version of this %s' % label - if self.is_waiting: - return 'This version of this %s is awaiting administrative approval' % label - if self.is_approved: - return 'This is the latest approved version of this %s' % label - if self.is_rejected: - return 'This version of this %s has been rejected by an administrator' % label - if self.is_archived: - return 'This is an archived version of this %s' % label - @property - def extension( self ): - # if instantiated via a query, this unmapped property won't exist - if '_Tool__extension' not in dir( self ): - self.__extension = None - if self.__extension is None: - head = open( self.file_name, 'rb' ).read( 4 ) - try: - assert head[:3] == 'BZh' - assert int( head[-1] ) in range( 0, 10 ) - self.__extension = 'tar.bz2' - except AssertionError: - pass - if self.__extension is None: - try: - assert head[:2] == '\037\213' - self.__extension = 'tar.gz' - except: - pass - if self.__extension is None: - self.__extension = 'tar' - return self.__extension - @property - def is_suite( self ): - return self.suite - @property - def label( self ): - if self.is_suite: - return 'tool suite' - else: - return 'tool' - @property - def type( self ): - # Hack - if self.is_suite: - return 'toolsuite' - return 'tool' - @property - def download_file_name( self ): - return '%s_%s.%s' % ( self.tool_id, self.version, self.extension ) - @property - def mimetype( self ): - return mimetypes.guess_type( self.download_file_name )[0] - -class Event( object ): - def __init__( self, state=None, comment='' ): - self.state = state - self.comment = comment - -class ToolEventAssociation( object ): - def __init__( self, tool=None, event=None ): - self.tool = tool - self.event = event + for username in util.listify( usernames ): + if username not in allow_push: + allow_push.append( username ) + allow_push = '%s\n' % ','.join( allow_push ) + repo = hg.repository( ui.ui(), path=self.repo_path ) + # Why doesn't the following work? + #repo.ui.setconfig( 'web', 'allow_push', allow_push ) + lines = repo.opener( 'hgrc', 'rb' ).readlines() + fp = repo.opener( 'hgrc', 'wb' ) + for line in lines: + if line.startswith( 'allow_push' ): + fp.write( 'allow_push = %s' % allow_push ) + else: + fp.write( line ) + fp.close() +class RepositoryMetadata( object ): + def __init__( self, repository_id=None, changeset_revision=None, metadata=None, malicious=False ): + self.repository_id = repository_id + self.changeset_revision = changeset_revision + self.metadata = metadata or dict() + self.malicious = malicious + class ItemRatingAssociation( object ): def __init__( self, id=None, user=None, item=None, rating=0, comment='' ): self.id = id @@ -285,10 +170,6 @@ class ItemRatingAssociation( object ): """ Set association's item. """ pass -class ToolRatingAssociation( ItemRatingAssociation ): - def set_item( self, tool ): - self.tool = tool - class RepositoryRatingAssociation( ItemRatingAssociation ): def set_item( self, repository ): self.repository = repository @@ -299,11 +180,6 @@ class Category( object ): self.description = description self.deleted = deleted -class ToolCategoryAssociation( object ): - def __init__( self, tool=None, category=None ): - self.tool = tool - self.category = category - class RepositoryCategoryAssociation( object ): def __init__( self, repository=None, category=None ): self.repository = repository @@ -327,12 +203,6 @@ class ItemTagAssociation ( object ): self.user_tname = user_tname self.value = None self.user_value = None - -class ToolTagAssociation ( ItemTagAssociation ): - pass - -class ToolAnnotationAssociation( object ): - pass ## ---- Utility methods ------------------------------------------------------- def sort_by_attr( seq, attr ): diff --git a/lib/galaxy/webapps/community/model/mapping.py b/lib/galaxy/webapps/community/model/mapping.py index 355a620955b..8f2e6c36e9e 100644 --- a/lib/galaxy/webapps/community/model/mapping.py +++ b/lib/galaxy/webapps/community/model/mapping.py @@ -104,9 +104,21 @@ Repository.table = Table( "repository", metadata, Column( "update_time", DateTime, default=now, onupdate=now ), Column( "name", TrimmedString( 255 ), index=True ), Column( "description" , TEXT ), + Column( "long_description" , TEXT ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), Column( "private", Boolean, default=False ), - Column( "deleted", Boolean, index=True, default=False ) ) + Column( "deleted", Boolean, index=True, default=False ), + Column( "email_alerts", JSONType, nullable=True ), + Column( "times_downloaded", Integer ) ) + +RepositoryMetadata.table = Table( "repository_metadata", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "repository_id", Integer, ForeignKey( "repository.id" ), index=True ), + Column( "changeset_revision", TrimmedString( 255 ), index=True ), + Column( "metadata", JSONType, nullable=True ), + Column( "malicious", Boolean, default=False ) ) RepositoryRatingAssociation.table = Table( "repository_rating_association", metadata, Column( "id", Integer, primary_key=True ), @@ -122,22 +134,6 @@ RepositoryCategoryAssociation.table = Table( "repository_category_association", Column( "repository_id", Integer, ForeignKey( "repository.id" ), index=True ), Column( "category_id", Integer, ForeignKey( "category.id" ), index=True ) ) -Tool.table = Table( "tool", metadata, - Column( "id", Integer, primary_key=True ), - Column( "guid", TrimmedString( 255 ), index=True, unique=True ), - Column( "tool_id", TrimmedString( 255 ), index=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "newer_version_id", Integer, ForeignKey( "tool.id" ), nullable=True ), - Column( "name", TrimmedString( 255 ), index=True ), - Column( "description" , TEXT ), - Column( "user_description" , TEXT ), - Column( "version", TrimmedString( 255 ) ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), - Column( "external_filename" , TEXT ), - Column( "deleted", Boolean, index=True, default=False ), - Column( "suite", Boolean, default=False, index=True ) ) - Category.table = Table( "category", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -146,32 +142,6 @@ Category.table = Table( "category", metadata, Column( "description" , TEXT ), Column( "deleted", Boolean, index=True, default=False ) ) -ToolCategoryAssociation.table = Table( "tool_category_association", metadata, - Column( "id", Integer, primary_key=True ), - Column( "tool_id", Integer, ForeignKey( "tool.id" ), index=True ), - Column( "category_id", Integer, ForeignKey( "category.id" ), index=True ) ) - -Event.table = Table( 'event', metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "state", TrimmedString( 255 ), index=True ), - Column( "comment", TEXT ) ) - -ToolEventAssociation.table = Table( "tool_event_association", metadata, - Column( "id", Integer, primary_key=True ), - Column( "tool_id", Integer, ForeignKey( "tool.id" ), index=True ), - Column( "event_id", Integer, ForeignKey( "event.id" ), index=True ) ) - -ToolRatingAssociation.table = Table( "tool_rating_association", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "tool_id", Integer, ForeignKey( "tool.id" ), index=True ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), - Column( "rating", Integer, index=True ), - Column( "comment", TEXT ) ) - Tag.table = Table( "tag", metadata, Column( "id", Integer, primary_key=True ), Column( "type", Integer ), @@ -179,27 +149,10 @@ Tag.table = Table( "tag", metadata, Column( "name", TrimmedString(255) ), UniqueConstraint( "name" ) ) -ToolTagAssociation.table = Table( "tool_tag_association", metadata, - Column( "id", Integer, primary_key=True ), - Column( "tool_id", Integer, ForeignKey( "tool.id" ), index=True ), - Column( "tag_id", Integer, ForeignKey( "tag.id" ), index=True ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), - Column( "user_tname", TrimmedString(255), index=True), - Column( "value", TrimmedString(255), index=True), - Column( "user_value", TrimmedString(255), index=True) ) - -ToolAnnotationAssociation.table = Table( "tool_annotation_association", metadata, - Column( "id", Integer, primary_key=True ), - Column( "tool_id", Integer, ForeignKey( "tool.id" ), index=True ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), - Column( "annotation", TEXT, index=True) ) - # With the tables defined we can define the mappers and setup the # relationships between the model objects. assign_mapper( context, User, User.table, - properties=dict( tools=relation( Tool, primaryjoin=( Tool.table.c.user_id == User.table.c.id ), order_by=( Tool.table.c.name ) ), - active_tools=relation( Tool, primaryjoin=( ( Tool.table.c.user_id == User.table.c.id ) & ( not_( Tool.table.c.deleted ) ) ), order_by=( Tool.table.c.name ) ), - active_repositories=relation( Repository, primaryjoin=( ( Repository.table.c.user_id == User.table.c.id ) & ( not_( Repository.table.c.deleted ) ) ), order_by=( Repository.table.c.name ) ), + properties=dict( active_repositories=relation( Repository, primaryjoin=( ( Repository.table.c.user_id == User.table.c.id ) & ( not_( Repository.table.c.deleted ) ) ), order_by=( Repository.table.c.name ) ), galaxy_sessions=relation( GalaxySession, order_by=desc( GalaxySession.table.c.update_time ) ) ) ) assign_mapper( context, Group, Group.table, @@ -233,55 +186,18 @@ assign_mapper( context, GalaxySession, GalaxySession.table, assign_mapper( context, Tag, Tag.table, properties=dict( children=relation(Tag, backref=backref( 'parent', remote_side=[Tag.table.c.id] ) ) ) ) -assign_mapper( context, ToolTagAssociation, ToolTagAssociation.table, - properties=dict( tag=relation(Tag, backref="tagged_tools"), user=relation( User ) ) ) - -assign_mapper( context, ToolAnnotationAssociation, ToolAnnotationAssociation.table, - properties=dict( tool=relation( Tool ), user=relation( User ) ) ) - -assign_mapper( context, Tool, Tool.table, - properties = dict( - categories=relation( ToolCategoryAssociation ), - events=relation( ToolEventAssociation, secondary=Event.table, - primaryjoin=( Tool.table.c.id==ToolEventAssociation.table.c.tool_id ), - secondaryjoin=( ToolEventAssociation.table.c.event_id==Event.table.c.id ), - order_by=desc( Event.table.c.update_time ), - viewonly=True, - uselist=True ), - ratings=relation( ToolRatingAssociation, order_by=desc( ToolRatingAssociation.table.c.update_time ), backref="tools" ), - user=relation( User.mapper ), - older_version=relation( - Tool, - primaryjoin=( Tool.table.c.newer_version_id == Tool.table.c.id ), - backref=backref( "newer_version", primaryjoin=( Tool.table.c.newer_version_id == Tool.table.c.id ), remote_side=[Tool.table.c.id] ) ) - ) ) - - -assign_mapper( context, ToolCategoryAssociation, ToolCategoryAssociation.table, - properties=dict( - category=relation( Category ), - tool=relation( Tool ) ) ) - -assign_mapper( context, ToolRatingAssociation, ToolRatingAssociation.table, - properties=dict( tool=relation( Tool ), user=relation( User ) ) ) - -assign_mapper( context, Event, Event.table, - properties=None ) - -assign_mapper( context, ToolEventAssociation, ToolEventAssociation.table, - properties=dict( - tool=relation( Tool ), - event=relation( Event ) ) ) - assign_mapper( context, Category, Category.table, - properties=dict( tools=relation( ToolCategoryAssociation ), - repositories=relation( RepositoryCategoryAssociation ) ) ) + properties=dict( repositories=relation( RepositoryCategoryAssociation ) ) ) assign_mapper( context, Repository, Repository.table, properties = dict( categories=relation( RepositoryCategoryAssociation ), ratings=relation( RepositoryRatingAssociation, order_by=desc( RepositoryRatingAssociation.table.c.update_time ), backref="repositories" ), - user=relation( User.mapper ) ) ) + user=relation( User.mapper ), + downloadable_revisions=relation( RepositoryMetadata, order_by=desc( RepositoryMetadata.table.c.id ) ) ) ) + +assign_mapper( context, RepositoryMetadata, RepositoryMetadata.table, + properties=dict( repository=relation( Repository ) ) ) assign_mapper( context, RepositoryRatingAssociation, RepositoryRatingAssociation.table, properties=dict( repository=relation( Repository ), user=relation( User ) ) ) @@ -309,11 +225,8 @@ def load_egg_for_url( url ): # Let this go, it could possibly work with db's we don't support log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) -def init( enable_next_gen_tool_shed, file_path, url, engine_options={}, create_tables=False ): +def init( file_path, url, engine_options={}, create_tables=False ): """Connect mappings to the database""" - if not enable_next_gen_tool_shed: - # Connect tool archive location to the file path - Tool.file_path = file_path # Load the appropriate db module load_egg_for_url( url ) # Create the database engine diff --git a/lib/galaxy/webapps/community/model/migrate/versions/0001_initial_tables.py b/lib/galaxy/webapps/community/model/migrate/versions/0001_initial_tables.py index 8dd89a94ed4..00cc5dec61e 100644 --- a/lib/galaxy/webapps/community/model/migrate/versions/0001_initial_tables.py +++ b/lib/galaxy/webapps/community/model/migrate/versions/0001_initial_tables.py @@ -11,8 +11,14 @@ now = datetime.datetime.utcnow # Need our custom types, but don't import anything else from model from galaxy.model.custom_types import * -import logging +import sys, logging log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) metadata = MetaData( migrate_engine ) diff --git a/lib/galaxy/webapps/community/model/migrate/versions/0002_add_tool_suite_column.py b/lib/galaxy/webapps/community/model/migrate/versions/0002_add_tool_suite_column.py index 591c2e8bf32..6a7840fcdd7 100644 --- a/lib/galaxy/webapps/community/model/migrate/versions/0002_add_tool_suite_column.py +++ b/lib/galaxy/webapps/community/model/migrate/versions/0002_add_tool_suite_column.py @@ -7,8 +7,14 @@ from sqlalchemy.orm import * from migrate import * from migrate.changeset import * -import logging +import sys, logging log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) metadata = MetaData( migrate_engine ) db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) @@ -16,7 +22,6 @@ db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, def upgrade(): print __doc__ metadata.reflect() - # Create and initialize imported column in job table. Tool_table = Table( "tool", metadata, autoload=True ) c = Column( "suite", Boolean, default=False, index=True ) @@ -24,22 +29,19 @@ def upgrade(): # Create c.create( Tool_table ) assert c is Tool_table.c.suite - # Initialize. if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': default_false = "0" elif migrate_engine.name == 'postgres': default_false = "false" db_session.execute( "UPDATE tool SET suite=%s" % default_false ) - except Exception, e: print "Adding suite column to the tool table failed: %s" % str( e ) log.debug( "Adding suite column to the tool table failed: %s" % str( e ) ) def downgrade(): metadata.reflect() - - # Drop imported column from job table. + # Drop suite column from tool table. Tool_table = Table( "tool", metadata, autoload=True ) try: Tool_table.c.suite.drop() diff --git a/lib/galaxy/webapps/community/model/migrate/versions/0004_repository_tables.py b/lib/galaxy/webapps/community/model/migrate/versions/0004_repository_tables.py index 681d5d77868..b15bfcd5b0b 100644 --- a/lib/galaxy/webapps/community/model/migrate/versions/0004_repository_tables.py +++ b/lib/galaxy/webapps/community/model/migrate/versions/0004_repository_tables.py @@ -30,7 +30,7 @@ Repository_table = Table( "repository", metadata, Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), Column( "name", TrimmedString( 255 ), index=True ), - Column( "description" , TEXT ), + Column( "description", TEXT ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), Column( "private", Boolean, default=False ), Column( "deleted", Boolean, index=True, default=False ) ) diff --git a/lib/galaxy/webapps/community/model/migrate/versions/0005_drop_tool_related_tables.py b/lib/galaxy/webapps/community/model/migrate/versions/0005_drop_tool_related_tables.py new file mode 100644 index 00000000000..e15738c9f83 --- /dev/null +++ b/lib/galaxy/webapps/community/model/migrate/versions/0005_drop_tool_related_tables.py @@ -0,0 +1,193 @@ +""" +Drops the tool, tool_category_association, event, tool_event_association, tool_rating_association, +tool_tag_association and tool_annotation_association tables since they are no longer used in the +next-gen tool shed. +""" +from sqlalchemy import * +from sqlalchemy.orm import * +from sqlalchemy.exc import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel( logging.DEBUG ) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def upgrade(): + print __doc__ + # Load existing tables + metadata.reflect() + # Load and then drop the tool_category_association table + try: + ToolCategoryAssociation_table = Table( "tool_category_association", metadata, autoload=True ) + except NoSuchTableError: + log.debug( "Failed loading table tool_category_association" ) + try: + ToolCategoryAssociation_table.drop() + except Exception, e: + log.debug( "Dropping tool_category_association table failed: %s" % str( e ) ) + # Load and then drop the tool_event_association table + try: + ToolEventAssociation_table = Table( "tool_event_association", metadata, autoload=True ) + except NoSuchTableError: + log.debug( "Failed loading table tool_event_association" ) + try: + ToolEventAssociation_table.drop() + except Exception, e: + log.debug( "Dropping tool_event_association table failed: %s" % str( e ) ) + # Load and then drop the tool_rating_association table + try: + ToolRatingAssociation_table = Table( "tool_rating_association", metadata, autoload=True ) + except NoSuchTableError: + log.debug( "Failed loading table tool_rating_association" ) + try: + ToolRatingAssociation_table.drop() + except Exception, e: + log.debug( "Dropping tool_rating_association table failed: %s" % str( e ) ) + # Load and then drop the tool_tag_association table + try: + ToolTagAssociation_table = Table( "tool_tag_association", metadata, autoload=True ) + except NoSuchTableError: + log.debug( "Failed loading table tool_tag_association" ) + try: + ToolTagAssociation_table.drop() + except Exception, e: + log.debug( "Dropping tool_tag_association table failed: %s" % str( e ) ) + # Load and then drop the tool_annotation_association table + try: + ToolAnnotationAssociation_table = Table( "tool_annotation_association", metadata, autoload=True ) + except NoSuchTableError: + log.debug( "Failed loading table tool_annotation_association" ) + try: + ToolAnnotationAssociation_table.drop() + except Exception, e: + log.debug( "Dropping tool_annotation_association table failed: %s" % str( e ) ) + # Load and then drop the event table + try: + Event_table = Table( "event", metadata, autoload=True ) + except NoSuchTableError: + log.debug( "Failed loading table event" ) + try: + Event_table.drop() + except Exception, e: + log.debug( "Dropping event table failed: %s" % str( e ) ) + # Load and then drop the tool table + try: + Tool_table = Table( "tool", metadata, autoload=True ) + except NoSuchTableError: + log.debug( "Failed loading table tool" ) + try: + Tool_table.drop() + except Exception, e: + log.debug( "Dropping tool table failed: %s" % str( e ) ) +def downgrade(): + # Load existing tables + metadata.reflect() + # We've lost all of our data, so downgrading is useless. However, we'll + # at least re-create the dropped tables. + Event_table = Table( 'event', metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "state", TrimmedString( 255 ), index=True ), + Column( "comment", TEXT ) ) + + Tool_table = Table( "tool", metadata, + Column( "id", Integer, primary_key=True ), + Column( "guid", TrimmedString( 255 ), index=True, unique=True ), + Column( "tool_id", TrimmedString( 255 ), index=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "newer_version_id", Integer, ForeignKey( "tool.id" ), nullable=True ), + Column( "name", TrimmedString( 255 ), index=True ), + Column( "description" , TEXT ), + Column( "user_description" , TEXT ), + Column( "version", TrimmedString( 255 ) ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), + Column( "external_filename" , TEXT ), + Column( "deleted", Boolean, index=True, default=False ), + Column( "suite", Boolean, default=False, index=True ) ) + + ToolCategoryAssociation_table = Table( "tool_category_association", metadata, + Column( "id", Integer, primary_key=True ), + Column( "tool_id", Integer, ForeignKey( "tool.id" ), index=True ), + Column( "category_id", Integer, ForeignKey( "category.id" ), index=True ) ) + + ToolEventAssociation_table = Table( "tool_event_association", metadata, + Column( "id", Integer, primary_key=True ), + Column( "tool_id", Integer, ForeignKey( "tool.id" ), index=True ), + Column( "event_id", Integer, ForeignKey( "event.id" ), index=True ) ) + + ToolRatingAssociation_table = Table( "tool_rating_association", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "tool_id", Integer, ForeignKey( "tool.id" ), index=True ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), + Column( "rating", Integer, index=True ), + Column( "comment", TEXT ) ) + + ToolTagAssociation_table = Table( "tool_tag_association", metadata, + Column( "id", Integer, primary_key=True ), + Column( "tool_id", Integer, ForeignKey( "tool.id" ), index=True ), + Column( "tag_id", Integer, ForeignKey( "tag.id" ), index=True ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), + Column( "user_tname", TrimmedString(255), index=True), + Column( "value", TrimmedString(255), index=True), + Column( "user_value", TrimmedString(255), index=True) ) + + ToolAnnotationAssociation_table = Table( "tool_annotation_association", metadata, + Column( "id", Integer, primary_key=True ), + Column( "tool_id", Integer, ForeignKey( "tool.id" ), index=True ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), + Column( "annotation", TEXT, index=True) ) + + # Create the event table + try: + Event_table.create() + except Exception, e: + log.debug( "Creating event table failed: %s" % str( e ) ) + # Create the tool table + try: + Tool_table.create() + except Exception, e: + log.debug( "Creating tool table failed: %s" % str( e ) ) + # Create the tool_category_association table + try: + ToolCategoryAssociation_table.create() + except Exception, e: + log.debug( "Creating tool_category_association table failed: %s" % str( e ) ) + # Create the tool_event_association table + try: + ToolEventAssociation_table.create() + except Exception, e: + log.debug( "Creating tool_event_association table failed: %s" % str( e ) ) + # Create the tool_rating_association table + try: + ToolRatingAssociation_table.create() + except Exception, e: + log.debug( "Creating tool_rating_association table failed: %s" % str( e ) ) + # Create the tool_tag_association table + try: + ToolTagAssociation_table.create() + except Exception, e: + log.debug( "Creating tool_tag_association table failed: %s" % str( e ) ) + # Create the tool_annotation_association table + try: + ToolAnnotationAssociation_table.create() + except Exception, e: + log.debug( "Creating tool_annotation_association table failed: %s" % str( e ) ) diff --git a/lib/galaxy/webapps/community/model/migrate/versions/0006_add_email_alerts_column.py b/lib/galaxy/webapps/community/model/migrate/versions/0006_add_email_alerts_column.py new file mode 100644 index 00000000000..02c3e8b2831 --- /dev/null +++ b/lib/galaxy/webapps/community/model/migrate/versions/0006_add_email_alerts_column.py @@ -0,0 +1,47 @@ +""" +Migration script to add the email_alerts column to the repository table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + # Create and initialize imported column in job table. + Repository_table = Table( "repository", metadata, autoload=True ) + c = Column( "email_alerts", JSONType, nullable=True ) + try: + # Create + c.create( Repository_table ) + assert c is Repository_table.c.email_alerts + except Exception, e: + print "Adding email_alerts column to the repository table failed: %s" % str( e ) + log.debug( "Adding email_alerts column to the repository table failed: %s" % str( e ) ) + +def downgrade(): + metadata.reflect() + # Drop email_alerts column from repository table. + Repository_table = Table( "repository", metadata, autoload=True ) + try: + Repository_table.c.email_alerts.drop() + except Exception, e: + print "Dropping column email_alerts from the repository table failed: %s" % str( e ) + log.debug( "Dropping column email_alerts from the repository table failed: %s" % str( e ) ) diff --git a/lib/galaxy/webapps/community/model/migrate/versions/0007_add_long_description_times_downloaded_columns.py b/lib/galaxy/webapps/community/model/migrate/versions/0007_add_long_description_times_downloaded_columns.py new file mode 100644 index 00000000000..eb4578f02b3 --- /dev/null +++ b/lib/galaxy/webapps/community/model/migrate/versions/0007_add_long_description_times_downloaded_columns.py @@ -0,0 +1,66 @@ +""" +Migration script to add the long_description and times_downloaded columns to the repository table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + # Create and initialize imported column in job table. + Repository_table = Table( "repository", metadata, autoload=True ) + c = Column( "long_description" , TEXT ) + try: + # Create + c.create( Repository_table ) + assert c is Repository_table.c.long_description + except Exception, e: + print "Adding long_description column to the repository table failed: %s" % str( e ) + log.debug( "Adding long_description column to the repository table failed: %s" % str( e ) ) + + c = Column( "times_downloaded" , Integer ) + try: + # Create + c.create( Repository_table ) + assert c is Repository_table.c.times_downloaded + except Exception, e: + print "Adding times_downloaded column to the repository table failed: %s" % str( e ) + log.debug( "Adding times_downloaded column to the repository table failed: %s" % str( e ) ) + + cmd = "UPDATE repository SET long_description = ''" + db_session.execute( cmd ) + cmd = "UPDATE repository SET times_downloaded = 0" + db_session.execute( cmd ) + +def downgrade(): + metadata.reflect() + # Drop email_alerts column from repository table. + Repository_table = Table( "repository", metadata, autoload=True ) + try: + Repository_table.c.long_description.drop() + except Exception, e: + print "Dropping column long_description from the repository table failed: %s" % str( e ) + log.debug( "Dropping column long_description from the repository table failed: %s" % str( e ) ) + try: + Repository_table.c.times_downloaded.drop() + except Exception, e: + print "Dropping column times_downloaded from the repository table failed: %s" % str( e ) + log.debug( "Dropping column times_downloaded from the repository table failed: %s" % str( e ) ) diff --git a/lib/galaxy/webapps/community/model/migrate/versions/0008_add_repository_metadata_table.py b/lib/galaxy/webapps/community/model/migrate/versions/0008_add_repository_metadata_table.py new file mode 100644 index 00000000000..36b23f2e7b0 --- /dev/null +++ b/lib/galaxy/webapps/community/model/migrate/versions/0008_add_repository_metadata_table.py @@ -0,0 +1,52 @@ +""" +Migration script to add the repository_metadata table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +RepositoryMetadata_table = Table( "repository_metadata", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "repository_id", Integer, ForeignKey( "repository.id" ), index=True ), + Column( "changeset_revision", TrimmedString( 255 ), index=True ), + Column( "metadata", JSONType, nullable=True ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + # Create repository_metadata table. + try: + RepositoryMetadata_table.create() + except Exception, e: + print str(e) + log.debug( "Creating repository_metadata table failed: %s" % str( e ) ) + +def downgrade(): + metadata.reflect() + # Drop repository_metadata table. + try: + RepositoryMetadata_table.drop() + except Exception, e: + print str(e) + log.debug( "Dropping repository_metadata table failed: %s" % str( e ) ) diff --git a/lib/galaxy/webapps/community/model/migrate/versions/0009_add_malicious_column.py b/lib/galaxy/webapps/community/model/migrate/versions/0009_add_malicious_column.py new file mode 100644 index 00000000000..a8d16a645d9 --- /dev/null +++ b/lib/galaxy/webapps/community/model/migrate/versions/0009_add_malicious_column.py @@ -0,0 +1,50 @@ +""" +Migration script to add the malicious column to the repository_metadata table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + # Create and initialize imported column in job table. + Repository_metadata_table = Table( "repository_metadata", metadata, autoload=True ) + c = Column( "malicious", Boolean, default=False, index=True ) + try: + # Create + c.create( Repository_metadata_table ) + assert c is Repository_metadata_table.c.malicious + # Initialize. + if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + default_false = "0" + elif migrate_engine.name == 'postgres': + default_false = "false" + db_session.execute( "UPDATE repository_metadata SET malicious=%s" % default_false ) + except Exception, e: + print "Adding malicious column to the repository_metadata table failed: %s" % str( e ) + log.debug( "Adding malicious column to the repository_metadata table failed: %s" % str( e ) ) + +def downgrade(): + metadata.reflect() + # Drop malicious column from repository_metadata table. + Repository_metadata_table = Table( "repository_metadata", metadata, autoload=True ) + try: + Repository_metadata_table.c.malicious.drop() + except Exception, e: + print "Dropping column malicious from the repository_metadata table failed: %s" % str( e ) + log.debug( "Dropping column malicious from the repository_metadata table failed: %s" % str( e ) ) diff --git a/lib/galaxy/webapps/community/security/__init__.py b/lib/galaxy/webapps/community/security/__init__.py index 82a67289f2e..a265e547de7 100644 --- a/lib/galaxy/webapps/community/security/__init__.py +++ b/lib/galaxy/webapps/community/security/__init__.py @@ -6,7 +6,7 @@ from datetime import datetime, timedelta from galaxy.util.bunch import Bunch from galaxy.util import listify from galaxy.model.orm import * -from galaxy.webapps.community.controllers.common import get_versions +from mercurial import hg, ui log = logging.getLogger(__name__) @@ -77,8 +77,6 @@ class CommunityRBACAgent( RBACAgent ): elif 'role' in kwd: if 'group' in kwd: return self.associate_group_role( kwd['group'], kwd['role'] ) - elif 'tool' in kwd: - return self.associate_tool_category( kwd['tool'], kwd['category'] ) elif 'repository' in kwd: return self.associate_repository_category( kwd[ 'repository' ], kwd[ 'category' ] ) raise 'No valid method of associating provided components: %s' % kwd @@ -97,11 +95,6 @@ class CommunityRBACAgent( RBACAgent ): self.sa_session.add( assoc ) self.sa_session.flush() return assoc - def associate_tool_category( self, tool, category ): - assoc = self.model.ToolCategoryAssociation( tool, category ) - self.sa_session.add( assoc ) - self.sa_session.flush() - return assoc def associate_repository_category( self, repository, category ): assoc = self.model.RepositoryCategoryAssociation( repository, category ) self.sa_session.add( assoc ) @@ -162,113 +155,10 @@ class CommunityRBACAgent( RBACAgent ): self.associate_components( user=user, role=role ) for group in groups: self.associate_components( user=user, group=group ) - def set_entity_category_associations( self, tools=[], categories=[], delete_existing_assocs=True ): - for tool in tools: - if delete_existing_assocs: - for a in tool.categories: - self.sa_session.delete( a ) - self.sa_session.flush() - self.sa_session.refresh( tool ) - for category in categories: - self.associate_components( tool=tool, category=category ) - def can_rate( self, user, user_is_admin, cntrller, item ): - # The current user can rate and review the item if they are an admin or if - # they did not upload the item and the item is approved or archived. - if user and user_is_admin and cntrller == 'admin': - return True - if cntrller in [ 'tool' ] and ( item.is_approved or item.is_archived ) and user != item.user: - return True - return False - def can_approve_or_reject( self, user, user_is_admin, cntrller, item ): - # The current user can approve or reject the item if the user - # is an admin, and the item's state is WAITING. - return user and user_is_admin and cntrller=='admin' and item.is_waiting - def can_delete( self, user, user_is_admin, cntrller, item ): - # The current user can delete the item if they are an admin or if they uploaded the - # item and in either case the item's state is not DELETED. - if user and user_is_admin and cntrller == 'admin': - can_delete = not item.is_deleted - elif cntrller in [ 'tool' ]: - can_delete = user==item.user and not item.is_deleted - else: - can_delete = False - return can_delete - def can_download( self, user, user_is_admin, cntrller, item ): - # The current user can download the item if they are an admin or if the - # item's state is not one of: NEW, WAITING. - if user and user_is_admin and cntrller == 'admin': - return True - elif cntrller in [ 'tool' ]: - can_download = not( item.is_new or item.is_waiting ) - else: - can_download = False - return can_download - def can_edit( self, user, user_is_admin, cntrller, item ): - # The current user can edit the item if they are an admin or if they uploaded the item - # and the item's state is one of: NEW, REJECTED. - if user and user_is_admin and cntrller == 'admin': - return True - if cntrller in [ 'tool' ]: - return user and user==item.user and ( item.is_new or item.is_rejected ) - return False - def can_purge( self, user, user_is_admin, cntrller ): - # The current user can purge the item if they are an admin. - return user and user_is_admin and cntrller == 'admin' - def can_upload_new_version( self, user, item ): - # The current user can upload a new version as long as the item's state is not NEW or WAITING. - if not user: - return False - versions = get_versions( item ) - state_ok = True - for version in versions: - if version.is_new or version.is_waiting: - state_ok = False - break - return state_ok - def can_view( self, user, user_is_admin, cntrller, item ): - # The current user can view the item if they are an admin or if they uploaded the item - # or if the item's state is APPROVED. - if user and user_is_admin and cntrller == 'admin': - return True - if cntrller in [ 'tool' ] and item.is_approved or item.is_archived or item.is_deleted: - return True - return user and user==item.user def can_push( self, user, repository ): - # TODO: handle this via the mercurial api. - if not user: - return False - # Read the repository's hgrc file - hgrc_file = os.path.abspath( os.path.join( repository.repo_path, ".hg", "hgrc" ) ) - config = ConfigParser.ConfigParser() - config.read( hgrc_file ) - for option in config.options( "web" ): - if option == 'allow_push': - allowed = config.get( "web", option ) - return user.username in allowed + if user: + return user.username in listify( repository.allow_push ) return False - def get_all_action_permissions( self, user, user_is_admin, cntrller, item ): - """Get all permitted actions on item for the current user""" - can_edit = self.can_edit( cntrller, user, user_is_admin, item ) - can_view = self.can_view( cntrller, user, user_is_admin, item ) - can_upload_new_version = self.can_upload_new_version( user, item ) - visible_versions = self.get_visible_versions( user, user_is_admin, cntrller, item ) - can_approve_or_reject = self.can_approve_or_reject( user, user_is_admin, cntrller, item ) - can_delete = self.can_delete( user, user_is_admin, cntrller, item ) - return can_edit, can_view, can_upload_new_version, can_delete, visible_versions, can_approve_or_reject - def get_visible_versions( self, user, user_is_admin, cntrller, item ): - # All previous versions of item can be displayed if the current user is an admin - # or they uploaded item. Otherwise, only versions whose state is APPROVED or - # ARCHIVED will be displayed. - if user and user_is_admin and cntrller == 'admin': - visible_versions = get_versions( item ) - elif cntrller in [ 'tool' ]: - visible_versions = [] - for version in get_versions( item ): - if version.is_approved or version.is_archived or version.user == user: - visible_versions.append( version ) - else: - visible_versions = [] - return visible_versions def get_permitted_actions( filter=None ): '''Utility method to return a subset of RBACAgent's permitted actions''' diff --git a/lib/galaxy/webapps/demo_sequencer/buildapp.py b/lib/galaxy/webapps/demo_sequencer/buildapp.py index 78d5761e83b..6f752c6361b 100644 --- a/lib/galaxy/webapps/demo_sequencer/buildapp.py +++ b/lib/galaxy/webapps/demo_sequencer/buildapp.py @@ -18,12 +18,12 @@ log = logging.getLogger( __name__ ) import config import galaxy.webapps.demo_sequencer.framework -def add_controllers( webapp, app ): +def add_ui_controllers( webapp, app ): """ Search for controllers in the 'galaxy.webapps.demo_sequencer.controllers' directory and add them to the webapp. """ - from galaxy.web.base.controller import BaseController + from galaxy.web.base.controller import BaseUIController from galaxy.web.base.controller import ControllerUnavailable import galaxy.webapps.demo_sequencer.controllers controller_dir = galaxy.webapps.demo_sequencer.controllers.__path__[0] @@ -37,8 +37,8 @@ def add_controllers( webapp, app ): # Look for a controller inside the modules for key in dir( module ): T = getattr( module, key ) - if isclass( T ) and T is not BaseController and issubclass( T, BaseController ): - webapp.add_controller( name, T( app ) ) + if isclass( T ) and T is not BaseUIController and issubclass( T, BaseUIController ): + webapp.add_ui_controller( name, T( app ) ) def app_factory( global_conf, **kwargs ): """Return a wsgi application serving the root object""" @@ -56,7 +56,7 @@ def app_factory( global_conf, **kwargs ): atexit.register( app.shutdown ) # Create the universe WSGI application webapp = galaxy.webapps.demo_sequencer.framework.WebApplication( app, session_cookie='galaxydemo_sequencersession' ) - add_controllers( webapp, app ) + add_ui_controllers( webapp, app ) # These two routes handle our simple needs at the moment webapp.add_route( '/:controller/:action', action='index' ) webapp.add_route( '/:action', controller='common', action='index' ) diff --git a/lib/galaxy/webapps/demo_sequencer/config.py b/lib/galaxy/webapps/demo_sequencer/config.py index 518e44ebcf4..e5d80248d76 100644 --- a/lib/galaxy/webapps/demo_sequencer/config.py +++ b/lib/galaxy/webapps/demo_sequencer/config.py @@ -49,8 +49,7 @@ class Configuration( object ): self.smtp_server = kwargs.get( 'smtp_server', None ) self.log_actions = string_as_bool( kwargs.get( 'log_actions', 'False' ) ) self.brand = kwargs.get( 'brand', None ) - self.wiki_url = kwargs.get( 'wiki_url', 'http://bitbucket.org/galaxy/galaxy-central/wiki/Home' ) - self.bugs_email = kwargs.get( 'bugs_email', None ) + self.wiki_url = kwargs.get( 'wiki_url', 'http://wiki.g2.bx.psu.edu/FrontPage' ) self.blog_url = kwargs.get( 'blog_url', None ) self.screencasts_url = kwargs.get( 'screencasts_url', None ) self.log_events = False diff --git a/lib/galaxy/webapps/demo_sequencer/controllers/common.py b/lib/galaxy/webapps/demo_sequencer/controllers/common.py index ce00a6a1d7a..9beaf1f389a 100644 --- a/lib/galaxy/webapps/demo_sequencer/controllers/common.py +++ b/lib/galaxy/webapps/demo_sequencer/controllers/common.py @@ -8,7 +8,7 @@ from urllib import quote_plus, unquote_plus import logging log = logging.getLogger( __name__ ) -class CommonController( BaseController ): +class CommonController( BaseUIController ): @web.expose def index( self, trans, **kwd ): redirect_action = util.restore_text( kwd.get( 'redirect_action', '' ) ) diff --git a/lib/galaxy/webapps/reports/buildapp.py b/lib/galaxy/webapps/reports/buildapp.py index 7bdaad9d9c1..192f143e3ab 100644 --- a/lib/galaxy/webapps/reports/buildapp.py +++ b/lib/galaxy/webapps/reports/buildapp.py @@ -20,12 +20,12 @@ import galaxy.model import galaxy.model.mapping import galaxy.web.framework -def add_controllers( webapp, app ): +def add_ui_controllers( webapp, app ): """ Search for controllers in the 'galaxy.webapps.controllers' module and add them to the webapp. """ - from galaxy.web.base.controller import BaseController + from galaxy.web.base.controller import BaseUIController from galaxy.web.base.controller import ControllerUnavailable import galaxy.webapps.reports.controllers controller_dir = galaxy.webapps.reports.controllers.__path__[0] @@ -39,8 +39,8 @@ def add_controllers( webapp, app ): # Look for a controller inside the modules for key in dir( module ): T = getattr( module, key ) - if isclass( T ) and T is not BaseController and issubclass( T, BaseController ): - webapp.add_controller( name, T( app ) ) + if isclass( T ) and T is not BaseUIController and issubclass( T, BaseUIController ): + webapp.add_ui_controller( name, T( app ) ) def app_factory( global_conf, **kwargs ): """Return a wsgi application serving the root object""" @@ -53,7 +53,7 @@ def app_factory( global_conf, **kwargs ): atexit.register( app.shutdown ) # Create the universe WSGI application webapp = galaxy.web.framework.WebApplication( app, session_cookie='galaxyreportssession' ) - add_controllers( webapp, app ) + add_ui_controllers( webapp, app ) # These two routes handle our simple needs at the moment webapp.add_route( '/:controller/:action', controller="root", action='index' ) webapp.add_route( '/:action', controller='root', action='index' ) diff --git a/lib/galaxy/webapps/reports/config.py b/lib/galaxy/webapps/reports/config.py index a2f81f74243..1a7508d2c6d 100644 --- a/lib/galaxy/webapps/reports/config.py +++ b/lib/galaxy/webapps/reports/config.py @@ -33,8 +33,7 @@ class Configuration( object ): self.sendmail_path = kwargs.get('sendmail_path',"/usr/sbin/sendmail") self.log_actions = string_as_bool( kwargs.get( 'log_actions', 'False' ) ) self.brand = kwargs.get( 'brand', None ) - self.wiki_url = kwargs.get( 'wiki_url', 'http://bitbucket.org/galaxy/galaxy-central/wiki/Home' ) - self.bugs_email = kwargs.get( 'bugs_email', None ) + self.wiki_url = kwargs.get( 'wiki_url', 'http://wiki.g2.bx.psu.edu/FrontPage' ) self.blog_url = kwargs.get( 'blog_url', None ) self.screencasts_url = kwargs.get( 'screencasts_url', None ) self.log_events = False diff --git a/lib/galaxy/webapps/reports/controllers/jobs.py b/lib/galaxy/webapps/reports/controllers/jobs.py index 64670c8055c..1121e5ce8ec 100644 --- a/lib/galaxy/webapps/reports/controllers/jobs.py +++ b/lib/galaxy/webapps/reports/controllers/jobs.py @@ -117,7 +117,7 @@ class SpecifiedDateListGrid( grids.Grid ): .join( model.User ) \ .enable_eagerloads( False ) -class Jobs( BaseController ): +class Jobs( BaseUIController ): specified_date_list_grid = SpecifiedDateListGrid() diff --git a/lib/galaxy/webapps/reports/controllers/root.py b/lib/galaxy/webapps/reports/controllers/root.py index 774687f08ee..5f73f002357 100644 --- a/lib/galaxy/webapps/reports/controllers/root.py +++ b/lib/galaxy/webapps/reports/controllers/root.py @@ -2,7 +2,7 @@ from galaxy.web.base.controller import * import logging log = logging.getLogger( __name__ ) -class Report( BaseController ): +class Report( BaseUIController ): @web.expose def index( self, trans, **kwd ): return trans.fill_template( '/webapps/reports/index.mako' ) diff --git a/lib/galaxy/webapps/reports/controllers/sample_tracking.py b/lib/galaxy/webapps/reports/controllers/sample_tracking.py index ef8ee32b9f7..73c47d6f53a 100644 --- a/lib/galaxy/webapps/reports/controllers/sample_tracking.py +++ b/lib/galaxy/webapps/reports/controllers/sample_tracking.py @@ -94,7 +94,7 @@ class SpecifiedDateListGrid( grids.Grid ): .join( model.User ) \ .enable_eagerloads( False ) -class SampleTracking( BaseController ): +class SampleTracking( BaseUIController ): specified_date_list_grid = SpecifiedDateListGrid() diff --git a/lib/galaxy/webapps/reports/controllers/system.py b/lib/galaxy/webapps/reports/controllers/system.py index 1c7f69b38ce..9672bdb7aed 100644 --- a/lib/galaxy/webapps/reports/controllers/system.py +++ b/lib/galaxy/webapps/reports/controllers/system.py @@ -6,7 +6,7 @@ from galaxy.model.orm import * import logging log = logging.getLogger( __name__ ) -class System( BaseController ): +class System( BaseUIController ): @web.expose def index( self, trans, **kwd ): params = util.Params( kwd ) @@ -112,7 +112,7 @@ class System( BaseController ): except: pass message = str( dataset_count ) + " datasets were deleted more than " + str( deleted_datasets_days ) + \ - " days ago, but have not yet been purged, disk space: " + str( disk_space ) + "." + " days ago, but have not yet been purged, disk space: " + nice_size( disk_space ) + "." else: message = "Enter the number of days." return str( deleted_datasets_days ), message diff --git a/lib/galaxy/webapps/reports/controllers/users.py b/lib/galaxy/webapps/reports/controllers/users.py index 536df6777ab..6b9c728c1c6 100644 --- a/lib/galaxy/webapps/reports/controllers/users.py +++ b/lib/galaxy/webapps/reports/controllers/users.py @@ -10,7 +10,7 @@ import sqlalchemy as sa import logging log = logging.getLogger( __name__ ) -class Users( BaseController ): +class Users( BaseUIController ): @web.expose def registered_users( self, trans, **kwd ): message = util.restore_text( kwd.get( 'message', '' ) ) @@ -116,3 +116,16 @@ class Users( BaseController ): users=users, not_logged_in_for_days=not_logged_in_for_days, message=message ) + + @web.expose + def user_disk_usage( self, trans, **kwd ): + message = util.restore_text( kwd.get( 'message', '' ) ) + user_cutoff = int( kwd.get( 'user_cutoff', 60 ) ) + # disk_usage isn't indexed + users = sorted( trans.sa_session.query( galaxy.model.User ).all(), key=operator.attrgetter( 'disk_usage' ), reverse=True ) + if user_cutoff: + users = users[:user_cutoff] + return trans.fill_template( '/webapps/reports/users_user_disk_usage.mako', + users=users, + user_cutoff=user_cutoff, + message=message ) diff --git a/lib/galaxy/webapps/reports/controllers/workflows.py b/lib/galaxy/webapps/reports/controllers/workflows.py index d360bca5ea7..143bb9cc01f 100644 --- a/lib/galaxy/webapps/reports/controllers/workflows.py +++ b/lib/galaxy/webapps/reports/controllers/workflows.py @@ -94,7 +94,7 @@ class SpecifiedDateListGrid( grids.Grid ): .join( model.User ) \ .enable_eagerloads( False ) -class Workflows( BaseController ): +class Workflows( BaseUIController ): specified_date_list_grid = SpecifiedDateListGrid() diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index f50bc3c2f84..e57fd0b864b 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -247,8 +247,20 @@ class ToolModule( WorkflowModule ): return data_inputs def get_data_outputs( self ): data_outputs = [] + data_inputs = None for name, tool_output in self.tool.outputs.iteritems(): - formats = [ tool_output.format ] + if tool_output.format_source != None: + formats = [ 'input' ] # default to special name "input" which remove restrictions on connections + if data_inputs == None: + data_inputs = self.get_data_inputs() + # find the input parameter referenced by format_source + for di in data_inputs: + # input names come prefixed with conditional and repeat names separated by '|' + # remove prefixes when comparing with format_source + if di['name'] != None and di['name'].split('|')[-1] == tool_output.format_source: + formats = di['extensions'] + else: + formats = [ tool_output.format ] for change_elem in tool_output.change_format: for when_elem in change_elem.findall( 'when' ): format = when_elem.get( 'format', None ) diff --git a/lib/galaxy_utils/sequence/fastq.py b/lib/galaxy_utils/sequence/fastq.py index 47732fe1c0e..ae993ab48ec 100644 --- a/lib/galaxy_utils/sequence/fastq.py +++ b/lib/galaxy_utils/sequence/fastq.py @@ -153,6 +153,8 @@ class fastqSequencingRead( SequencingRead ): rval.quality = reversed( rval.get_decimal_quality_scores() ) rval.quality = "%s " % " ".join( map( str, rval.quality ) ) return rval + def apply_galaxy_conventions( self ): + pass class fastqSangerRead( fastqSequencingRead ): format = 'sanger' @@ -206,7 +208,7 @@ class fastqCSSangerRead( fastqSequencingRead ): if self.has_adapter_base(): qual_len = len( self.get_ascii_quality_scores() ) seq_len = len( self.sequence ) - assert qual_len + 1 == seq_len, "Invalid FASTQ file: quality score length (%i) does not match sequence length (%i with adapter base)" % ( qual_len, seq_len ) + assert ( qual_len + 1 == seq_len ) or ( qual_len == seq_len ), "Invalid FASTQ file: quality score length (%i) does not match sequence length (%i with adapter base)" % ( qual_len, seq_len ) #SRA adds FAKE/DUMMY quality scores to the adapter base, we'll allow the reading of the Improper score here, but remove it in the Reader when "apply_galaxy_conventions" is set to True else: return fastqSequencingRead.assert_sequence_quality_lengths( self ) def get_sequence( self ): @@ -262,7 +264,12 @@ class fastqCSSangerRead( fastqSequencingRead ): elif new_adapter: rval.sequence = "%s%s" % ( new_adapter, rval.sequence ) return rval - + def apply_galaxy_conventions( self ): + if self.has_adapter_base() and len( self.sequence ) == len( self.get_ascii_quality_scores() ): #SRA adds FAKE/DUMMY quality scores to the adapter base, we remove them here + if self.is_ascii_encoded(): + self.quality = self.quality[1:] + else: + self.quality = " ".join( map( str, self.get_decimal_quality_scores()[1:] ) ) FASTQ_FORMATS = {} for format in [ fastqIlluminaRead, fastqSolexaRead, fastqSangerRead, fastqCSSangerRead ]: @@ -417,9 +424,10 @@ class fastqAggregator( object ): return column_stats class fastqReader( object ): - def __init__( self, fh, format = 'sanger' ): + def __init__( self, fh, format = 'sanger', apply_galaxy_conventions = False ): self.file = fh self.format = format + self.apply_galaxy_conventions = apply_galaxy_conventions def close( self ): return self.file.close() def next(self): @@ -438,7 +446,7 @@ class fastqReader( object ): while True: line = self.file.readline() if not line: - raise Exception( 'Invalid FASTQ file: could not parse second instance of sequence identifier.' ) + raise Exception( 'Invalid FASTQ file: could not find quality score of sequence identifier %s.' % rval.identifier ) line = line.rstrip( '\n\r' ) if line.startswith( '+' ) and ( len( line ) == 1 or line[1:].startswith( fastq_header[1:] ) ): rval.description = line @@ -450,6 +458,8 @@ class fastqReader( object ): break rval.append_quality( line ) rval.assert_sequence_quality_lengths() + if self.apply_galaxy_conventions: + rval.apply_galaxy_conventions() return rval def __iter__( self ): while True: @@ -494,13 +504,14 @@ class fastqVerboseErrorReader( fastqReader ): raise e class fastqNamedReader( object ): - def __init__( self, fh, format = 'sanger' ): + def __init__( self, fh, format = 'sanger', apply_galaxy_conventions = False ): self.file = fh self.format = format self.reader = fastqReader( self.file, self.format ) #self.last_offset = self.file.tell() self.offset_dict = {} self.eof = False + self.apply_galaxy_conventions = apply_galaxy_conventions def close( self ): return self.file.close() def get( self, sequence_id ): @@ -531,6 +542,8 @@ class fastqNamedReader( object ): if fastq_read.identifier not in self.offset_dict: self.offset_dict[ fastq_read.identifier ] = [] self.offset_dict[ fastq_read.identifier ].append( offset ) + if rval is not None and self.apply_galaxy_conventions: + rval.apply_galaxy_conventions() return rval def has_data( self ): #returns a string representation of remaining data, or empty string (False) if no data remaining @@ -547,7 +560,7 @@ class fastqNamedReader( object ): eof = True self.file.seek( offset ) if count: - rval = "There were %i known sequence reads not utilized. " + rval = "There were %i known sequence reads not utilized. " % count if not eof: rval = "%s%s" % ( rval, "An additional unknown number of reads exist in the input that were not utilized." ) return rval @@ -615,6 +628,16 @@ class fastqJoiner( object ): elif identifier[-1] == "2": identifier = "%s1" % identifier[:-1] return identifier + def is_first_mate( self, sequence_id ): + is_first = None + if not isinstance( sequence_id, basestring ): + sequence_id = sequence_id.identifier + if sequence_id[-2] == '/': + if sequence_id[-1] == "1": + is_first = True + else: + is_first = False + return is_first class fastqSplitter( object ): def split( self, fastq_read ): diff --git a/lib/galaxy_utils/sequence/vcf.py b/lib/galaxy_utils/sequence/vcf.py index 406a025b36a..491f3677276 100644 --- a/lib/galaxy_utils/sequence/vcf.py +++ b/lib/galaxy_utils/sequence/vcf.py @@ -1,6 +1,7 @@ #Dan Blankenberg -#See: http://1000genomes.org/wiki/doku.php?id=1000_genomes:analysis:vcf3.3 -#See: http://1000genomes.org/wiki/doku.php?id=1000_genomes:analysis:variant_call_format +# See http://www.1000genomes.org/wiki/Analysis/variant-call-format + +NOT_A_NUMBER = float( 'NaN' ) class VariantCall( object ): version = None @@ -40,7 +41,10 @@ class VariantCall33( VariantCall ): self.chrom, self.pos, self.id, self.ref, self.alt, self.qual, self.filter, self.info = self.fields[ :self.required_header_length ] self.pos = int( self.pos ) self.alt = self.alt.split( ',' ) - self.qual = float( self.qual ) + try: + self.qual = float( self.qual ) + except: + self.qual = NOT_A_NUMBER #Missing data can be denoted as a '.' if len( self.fields ) > self.required_header_length: self.format = self.fields[ self.required_header_length ].split( ':' ) for sample_value in self.fields[ self.required_header_length + 1: ]: @@ -51,9 +55,12 @@ class VariantCall40( VariantCall33 ): def __init__( self, vcf_line, metadata, sample_names ): VariantCall33.__init__( self, vcf_line, metadata, sample_names) +class VariantCall41( VariantCall40 ): + version = 'VCFv4.1' + #VCF Format version lookup dict VCF_FORMATS = {} -for format in [ VariantCall33, VariantCall40 ]: +for format in [ VariantCall33, VariantCall40, VariantCall41 ]: VCF_FORMATS[format.version] = format class Reader( object ): diff --git a/run.sh b/run.sh index 1c7c73814e4..b6804830d30 100755 --- a/run.sh +++ b/run.sh @@ -6,11 +6,13 @@ python ./scripts/check_python.py [ $? -ne 0 ] && exit 1 SAMPLES=" - external_service_types_conf.xml.sample datatypes_conf.xml.sample + external_service_types_conf.xml.sample reports_wsgi.ini.sample + shed_tool_conf.xml.sample tool_conf.xml.sample tool_data_table_conf.xml.sample + tool_sheds_conf.xml.sample universe_wsgi.ini.sample tool-data/shared/ucsc/builds.txt.sample tool-data/*.sample @@ -32,7 +34,7 @@ for arg in "$@"; do [ "$arg" = "--stop-daemon" ] && FETCH_EGGS=0; break done if [ $FETCH_EGGS -eq 1 ]; then - python ./scripts/check_eggs.py quiet + python ./scripts/check_eggs.py -q if [ $? -ne 0 ]; then echo "Some eggs are out of date, attempting to fetch..." python ./scripts/fetch_eggs.py diff --git a/scripts/api/common.py b/scripts/api/common.py index 6b17aaa2013..541fcad0710 100644 --- a/scripts/api/common.py +++ b/scripts/api/common.py @@ -88,6 +88,8 @@ def display( api_key, url, return_formatted=True ): print '------------------' for k, v in r.items(): print '%s: %s' % ( k, v ) + elif type( r ) == str: + print r else: print 'response is unknown type: %s' % type( r ) diff --git a/scripts/api/create.py b/scripts/api/create.py new file mode 100644 index 00000000000..06ad024bb1a --- /dev/null +++ b/scripts/api/create.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +""" +Generic POST/create script + +usage: create.py key url [key=value ...] +""" + +import os, sys +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import submit + +data = {} +for k, v in [ kwarg.split('=', 1) for kwarg in sys.argv[3:]]: + data[k] = v + +submit( sys.argv[1], sys.argv[2], data ) diff --git a/scripts/api/delete.py b/scripts/api/delete.py new file mode 100644 index 00000000000..a6b555faed4 --- /dev/null +++ b/scripts/api/delete.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +""" +Generic DELETE/delete script + +usage: delete.py key url +""" + +import os, sys +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import delete + +data = {} +for k, v in [ kwarg.split('=', 1) for kwarg in sys.argv[3:]]: + data[k] = v + +delete( sys.argv[1], sys.argv[2], data ) diff --git a/scripts/api/history_create_history.py b/scripts/api/history_create_history.py new file mode 100644 index 00000000000..2e09658f0ae --- /dev/null +++ b/scripts/api/history_create_history.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python + +import os, sys +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import submit + +try: + assert sys.argv[2] +except IndexError: + print 'usage: %s key url [name] ' % os.path.basename( sys.argv[0] ) + sys.exit( 1 ) +try: + data = {} + data[ 'name' ] = sys.argv[3] +except IndexError: + pass + +submit( sys.argv[1], sys.argv[2], data ) diff --git a/scripts/api/history_delete_history.py b/scripts/api/history_delete_history.py new file mode 100644 index 00000000000..55b5ec453a2 --- /dev/null +++ b/scripts/api/history_delete_history.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python + +import os, sys +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import delete + +try: + assert sys.argv[2] +except IndexError: + print 'usage: %s key url [purge (true/false)] ' % os.path.basename( sys.argv[0] ) + sys.exit( 1 ) +try: + data = {} + data[ 'purge' ] = sys.argv[3] +except IndexError: + pass + +delete( sys.argv[1], sys.argv[2], data ) diff --git a/scripts/api/import_library_dataset_to_history.py b/scripts/api/import_library_dataset_to_history.py new file mode 100644 index 00000000000..78594081a03 --- /dev/null +++ b/scripts/api/import_library_dataset_to_history.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +import os, sys +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import submit + +try: + assert sys.argv[3] + data = {} + data['from_ld_id'] = sys.argv[3] +except IndexError: + print 'usage: %s key url library_file_id' % os.path.basename( sys.argv[0] ) + print ' library_file_id is from /api/libraries//contents/' + sys.exit( 1 ) + +submit( sys.argv[1], sys.argv[2], data ) diff --git a/scripts/api/update.py b/scripts/api/update.py new file mode 100644 index 00000000000..a8bb198b7a0 --- /dev/null +++ b/scripts/api/update.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +""" +Generic PUT/update script + +usage: create.py key url [key=value ...] +""" + +import os, sys +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import update + +data = {} +for k, v in [ kwarg.split('=', 1) for kwarg in sys.argv[3:]]: + data[k] = v + +update( sys.argv[1], sys.argv[2], data ) diff --git a/scripts/check_eggs.py b/scripts/check_eggs.py index f397e8340ee..097700b23ec 100644 --- a/scripts/check_eggs.py +++ b/scripts/check_eggs.py @@ -3,27 +3,35 @@ usage: check_eggs.py """ import os, sys, logging +from optparse import OptionParser + +parser = OptionParser() +parser.add_option( '-c', '--config', dest='config', help='Path to Galaxy config file (universe_wsgi.ini)', default='universe_wsgi.ini' ) +parser.add_option( '-q', '--quiet', dest='quiet', action="store_true", help='Quiet (no output, only set return code)', default=False ) +( options, args ) = parser.parse_args() + +if not os.path.exists( options.config ): + print "Config file does not exist (see 'python %s --help'): %s" % ( sys.argv[0], options.config ) + sys.exit( 1 ) root = logging.getLogger() root.setLevel( 10 ) root.addHandler( logging.StreamHandler( sys.stdout ) ) +config_arg = '' +if options.config != 'universe_wsgi.ini': + config_arg = '-c %s' % options.config + lib = os.path.abspath( os.path.join( os.path.dirname( __file__ ), "..", "lib" ) ) sys.path.append( lib ) -try: - assert sys.argv[1] == 'quiet' - quiet = True -except: - quiet = False - from galaxy.eggs import Crate -c = Crate() +c = Crate( options.config ) if c.config_missing: - if not quiet: + if not options.quiet: print "Some of your Galaxy eggs are out of date. Please update them" print "by running:" - print " python scripts/fetch_eggs.py" + print " python scripts/fetch_eggs.py %s" % config_arg sys.exit( 1 ) sys.exit( 0 ) diff --git a/scripts/check_galaxy.py b/scripts/check_galaxy.py index 261761d0600..f4f418516c0 100755 --- a/scripts/check_galaxy.py +++ b/scripts/check_galaxy.py @@ -200,7 +200,7 @@ class Browser: tc.fv("1", "email", user) tc.fv("1", "password", pw) tc.fv("1", "confirm", pw) - tc.submit("Create") + tc.submit("Submit") tc.code(200) if len(tc.get_browser().get_all_forms()) > 0: p = userParser() @@ -271,7 +271,7 @@ class Browser: p.feed(tc.browser.get_html()) dids = p.dids for did in dids: - self.get("/root/delete?id=%s" % did) + self.get("/datasets/%s/delete" % did) def check_if_logged_in(self): self.get("/user?cntrller=user") @@ -283,16 +283,21 @@ class userParser(htmllib.HTMLParser): def __init__(self): htmllib.HTMLParser.__init__(self, formatter.NullFormatter()) self.in_span = False + self.in_div = False self.no_user = False self.bad_pw = False self.already_exists = False def start_span(self, attrs): self.in_span = True + def start_div(self, attrs): + self.in_div = True def end_span(self): self.in_span = False + def end_div(self): + self.in_div = False def handle_data(self, data): - if self.in_span: - if data == "No such user": + if self.in_span or self.in_div: + if data == "No such user (please note that login is case sensitive)": self.no_user = True elif data == "Invalid password": self.bad_pw = True diff --git a/scripts/check_python.py b/scripts/check_python.py index 2ef81df50ac..15017c9d2c0 100644 --- a/scripts/check_python.py +++ b/scripts/check_python.py @@ -8,7 +8,7 @@ contains instructions on how to force Galaxy to use a different version.""" % sy def check_python(): try: - assert sys.version_info[:2] >= ( 2, 4 ) and sys.version_info[:2] <= ( 2, 6 ) + assert sys.version_info[:2] >= ( 2, 4 ) and sys.version_info[:2] <= ( 2, 7 ) except AssertionError: print >>sys.stderr, msg raise diff --git a/scripts/cleanup_datasets/cleanup_datasets.py b/scripts/cleanup_datasets/cleanup_datasets.py index 0a4eec566f8..c0f68ef6b92 100644 --- a/scripts/cleanup_datasets/cleanup_datasets.py +++ b/scripts/cleanup_datasets/cleanup_datasets.py @@ -170,6 +170,7 @@ def purge_histories( app, cutoff_time, remove_from_disk, info_only = False, forc app.model.History.table.c.update_time < cutoff_time ) ) \ .options( eagerload( 'datasets' ) ) for history in histories: + print "### Processing history id %d (%s)" % (history.id, history.name) for dataset_assoc in history.datasets: _purge_dataset_instance( dataset_assoc, app, remove_from_disk, info_only = info_only ) #mark a DatasetInstance as deleted, clear associated files, and mark the Dataset as deleted if it is deletable if not info_only: @@ -182,6 +183,8 @@ def purge_histories( app, cutoff_time, remove_from_disk, info_only = False, forc history.purged = True app.sa_session.add( history ) app.sa_session.flush() + else: + print "History id %d will be purged (without 'info_only' mode)" % history.id history_count += 1 stop = time.time() print 'Purged %d histories.' % history_count @@ -310,17 +313,21 @@ def delete_datasets( app, cutoff_time, remove_from_disk, info_only = False, forc dataset_ids.extend( [ row.id for row in history_dataset_ids_query.execute() ] ) # Process each of the Dataset objects for dataset_id in dataset_ids: - print "######### Processing dataset id:", dataset_id dataset = app.sa_session.query( app.model.Dataset ).get( dataset_id ) - if dataset.id not in skip and _dataset_is_deletable( dataset ): - deleted_dataset_count += 1 - for dataset_instance in dataset.history_associations + dataset.library_associations: - # Mark each associated HDA as deleted - _purge_dataset_instance( dataset_instance, app, remove_from_disk, include_children=True, info_only=info_only, is_deletable=True ) - deleted_instance_count += 1 + if dataset.id in skip: + continue skip.append( dataset.id ) + print "######### Processing dataset id:", dataset_id + if not _dataset_is_deletable( dataset ): + print "Dataset is not deletable (shared between multiple histories/libraries, at least one is not deleted)" + continue + deleted_dataset_count += 1 + for dataset_instance in dataset.history_associations + dataset.library_associations: + # Mark each associated HDA as deleted + _purge_dataset_instance( dataset_instance, app, remove_from_disk, include_children=True, info_only=info_only, is_deletable=True ) + deleted_instance_count += 1 stop = time.time() - print "Examined %d datasets, marked %d as deleted and purged %d dataset instances" % ( len( skip ), deleted_dataset_count, deleted_instance_count ) + print "Examined %d datasets, marked %d datasets and %d dataset instances (HDA) as deleted" % ( len( skip ), deleted_dataset_count, deleted_instance_count ) print "Total elapsed time: ", stop - start print "##########################################" @@ -360,15 +367,24 @@ def _purge_dataset_instance( dataset_instance, app, remove_from_disk, include_ch # A dataset_instance is either a HDA or an LDDA. Purging a dataset instance marks the instance as deleted, # and marks the associated dataset as deleted if it is not associated with another active DatsetInstance. if not info_only: - print "Marking as deleted: ", dataset_instance.__class__.__name__, " id ", dataset_instance.id + print "Marking as deleted: %s id %d (for dataset id %d)" % \ + ( dataset_instance.__class__.__name__, dataset_instance.id, dataset_instance.dataset.id ) dataset_instance.mark_deleted( include_children = include_children ) dataset_instance.clear_associated_files() app.sa_session.add( dataset_instance ) app.sa_session.flush() app.sa_session.refresh( dataset_instance.dataset ) + else: + print "%s id %d (for dataset id %d) will be marked as deleted (without 'info_only' mode)" % \ + ( dataset_instance.__class__.__name__, dataset_instance.id, dataset_instance.dataset.id ) if is_deletable or _dataset_is_deletable( dataset_instance.dataset ): # Calling methods may have already checked _dataset_is_deletable, if so, is_deletable should be True _delete_dataset( dataset_instance.dataset, app, remove_from_disk, info_only=info_only, is_deletable=is_deletable ) + else: + if info_only: + print "Not deleting dataset ", dataset_instance.dataset.id, " (will be possibly deleted without 'info_only' mode)" + else: + print "Not deleting dataset %d (shared between multiple histories/libraries, at least one not deleted)" % dataset_instance.dataset.id #need to purge children here if include_children: for child in dataset_instance.children: @@ -396,8 +412,13 @@ def _delete_dataset( dataset, app, remove_from_disk, info_only=False, is_deletab .filter( app.model.MetadataFile.table.c.lda_id==ldda.id ): metadata_files.append( metadata_file ) for metadata_file in metadata_files: - print "The following metadata files attached to associations of Dataset '%s' have been purged:" % dataset.id - if not info_only: + op_description = "marked as deleted" + if remove_from_disk: + op_description = op_description + " and purged from disk" + if info_only: + print "The following metadata files attached to associations of Dataset '%s' will be %s (without 'info_only' mode):" % ( dataset.id, op_description ) + else: + print "The following metadata files attached to associations of Dataset '%s' have been %s:" % ( dataset.id, op_description ) if remove_from_disk: try: print "Removing disk file ", metadata_file.file_name @@ -411,10 +432,13 @@ def _delete_dataset( dataset, app, remove_from_disk, info_only=False, is_deletab app.sa_session.add( metadata_file ) app.sa_session.flush() print "%s" % metadata_file.file_name - print "Deleting dataset id", dataset.id - dataset.deleted = True - app.sa_session.add( dataset ) - app.sa_session.flush() + if not info_only: + print "Deleting dataset id", dataset.id + dataset.deleted = True + app.sa_session.add( dataset ) + app.sa_session.flush() + else: + print "Dataset %i will be deleted (without 'info_only' mode)" % ( dataset.id ) def _purge_dataset( app, dataset, remove_from_disk, info_only = False ): if dataset.deleted: @@ -429,10 +453,19 @@ def _purge_dataset( app, dataset, remove_from_disk, info_only = False ): # Remove associated extra files from disk if they exist if dataset.extra_files_path and os.path.exists( dataset.extra_files_path ): shutil.rmtree( dataset.extra_files_path ) #we need to delete the directory and its contents; os.unlink would always fail on a directory + usage_users = [] + for hda in dataset.history_associations: + if not hda.purged and hda.history.user is not None and hda.history.user not in usage_users: + usage_users.append( hda.history.user ) + for user in usage_users: + user.total_disk_usage -= dataset.total_size + app.sa_session.add( user ) print "Purging dataset id", dataset.id dataset.purged = True app.sa_session.add( dataset ) app.sa_session.flush() + else: + print "Dataset %i will be purged (without 'info_only' mode)" % (dataset.id) else: print "This dataset (%i) is not purgable, the file (%s) will not be removed.\n" % ( dataset.id, dataset.file_name ) except OSError, exc: diff --git a/scripts/dist-scramble.py b/scripts/dist-scramble.py index 26e620eb42d..727319b0c33 100644 --- a/scripts/dist-scramble.py +++ b/scripts/dist-scramble.py @@ -1,11 +1,10 @@ -""" -usage: dist-scramble.py [platform] - egg_name - The egg to scramble (as defined in eggs.ini) - platform - The platform to scramble on (as defined in - dist-eggs.ini). Leave blank for all. - Platform-inspecific eggs ignore this argument. -""" import os, sys, logging +from optparse import OptionParser + +parser = OptionParser() +parser.add_option( '-e', '--egg-name', dest='egg_name', help='Egg name (as defined in eggs.ini) to scramble (required)' ) +parser.add_option( '-p', '--platform', dest='platform', help='Scramble for a specific platform (by default, eggs are scrambled for all platforms, see dist-eggs.ini for platform names)' ) +( options, args ) = parser.parse_args() root = logging.getLogger() root.setLevel( 10 ) @@ -17,18 +16,20 @@ sys.path.append( lib ) from galaxy.eggs.dist import DistScrambleCrate, ScrambleFailure from galaxy.eggs import EggNotFetchable -if len( sys.argv ) > 3 or len( sys.argv ) < 2: - print __doc__ +if not options.egg_name: + print "ERROR: You must specify an egg to scramble (-e)" + parser.print_help() sys.exit( 1 ) -elif len( sys.argv ) == 3: - c = DistScrambleCrate( sys.argv[2] ) + +if options.platform: + c = DistScrambleCrate( None, options.platform ) else: - c = DistScrambleCrate() + c = DistScrambleCrate( None ) try: - eggs = c[sys.argv[1]] + eggs = c[options.egg_name] except: - print "error: %s not in eggs.ini" % sys.argv[1] + print "ERROR: %s not in eggs.ini" % options.egg_name sys.exit( 1 ) failed = [] for egg in eggs: diff --git a/scripts/fetch_eggs.py b/scripts/fetch_eggs.py index 0e0680ab514..ddd3c9695fc 100755 --- a/scripts/fetch_eggs.py +++ b/scripts/fetch_eggs.py @@ -1,15 +1,15 @@ -""" -usage: fetch_eggs.py [egg_name] [platform] - With no arguments, fetches all eggs necessary according to the - settings in universe_wsgi.ini. - egg_name - Fetch only this egg (as defined in eggs.ini) or 'all' for - all eggs (even those not required by your settings). - platform - Fetch eggs for a specific platform (if not provided, fetch - eggs for *this* platform). Useful for fetching eggs for cluster - nodes which are of a different architecture than the head node. - Platform name can be determined with the get_platforms.py script. -""" import os, sys, logging +from optparse import OptionParser + +parser = OptionParser() +parser.add_option( '-c', '--config', dest='config', help='Path to Galaxy config file (universe_wsgi.ini)', default='universe_wsgi.ini' ) +parser.add_option( '-e', '--egg-name', dest='egg_name', help='Egg name (as defined in eggs.ini) to fetch, or "all" for all eggs, even those not needed by your configuration' ) +parser.add_option( '-p', '--platform', dest='platform', help='Fetch for a specific platform (by default, eggs are fetched for *this* platform' ) +( options, args ) = parser.parse_args() + +if not os.path.exists( options.config ): + print "Config file does not exist (see 'python %s --help'): %s" % ( sys.argv[0], options.config ) + sys.exit( 1 ) root = logging.getLogger() root.setLevel( 10 ) @@ -21,18 +21,18 @@ sys.path.append( lib ) from galaxy.eggs import Crate, EggNotFetchable import pkg_resources +if options.platform: + c = Crate( options.config, platform = options.platform ) +else: + c = Crate( options.config ) try: - c = Crate( platform = sys.argv[2] ) -except: - c = Crate() -try: - if len( sys.argv ) == 1: + if not options.egg_name: c.resolve() # Only fetch eggs required by the config - elif sys.argv[1] == 'all': + elif options.egg_name == 'all': c.resolve( all=True ) # Fetch everything else: # Fetch a specific egg - name = sys.argv[1] + name = options.egg_name try: egg = c[name] except: @@ -41,12 +41,15 @@ try: dist = egg.resolve()[0] print "%s %s is installed at %s" % ( dist.project_name, dist.version, dist.location ) except EggNotFetchable, e: + config_arg = '' + if options.config != 'universe_wsgi.ini': + config_arg = '-c %s ' % options.config try: - assert sys.argv[1] != 'all' + assert options.egg_name != 'all' egg = e.eggs[0] print "%s %s couldn't be downloaded automatically. You can try" % ( egg.name, egg.version ) print "building it by hand with:" - print " python scripts/scramble.py %s" + print " python scripts/scramble.py %s-e %s" % ( config_arg, egg.name ) except ( AssertionError, IndexError ): print "One or more of the python eggs necessary to run Galaxy couldn't be" print "downloaded automatically. You can try building them by hand (all" @@ -54,6 +57,6 @@ except EggNotFetchable, e: print " python scripts/scramble.py" print "Or individually:" for egg in e.eggs: - print " python scripts/scramble.py %s" % egg.name + print " python scripts/scramble.py %s-e %s" % ( config_arg, egg.name ) sys.exit( 1 ) sys.exit( 0 ) diff --git a/scripts/helper.py b/scripts/helper.py new file mode 100644 index 00000000000..2c93d8e095f --- /dev/null +++ b/scripts/helper.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +""" +A command line helper for common operations performed by Galaxy maintainers. +Encodes and decodes IDs, returns Dataset IDs if provided an HDA or LDDA id, +returns the disk path of a dataset. +""" + +import os, sys +from ConfigParser import ConfigParser +from optparse import OptionParser + +default_config = os.path.abspath( os.path.join( os.path.dirname( __file__ ), '..', 'universe_wsgi.ini') ) + +parser = OptionParser() +parser.add_option( '-c', '--config', dest='config', help='Path to Galaxy config file (universe_wsgi.ini)', default=default_config ) +parser.add_option( '-e', '--encode-id', dest='encode_id', help='Encode an ID' ) +parser.add_option( '-d', '--decode-id', dest='decode_id', help='Decode an ID' ) +parser.add_option( '--hda', dest='hda_id', help='Display HistoryDatasetAssociation info' ) +parser.add_option( '--ldda', dest='ldda_id', help='Display LibraryDatasetDatasetAssociation info' ) +( options, args ) = parser.parse_args() + +try: + assert options.encode_id or options.decode_id or options.hda_id or options.ldda_id +except: + parser.print_help() + sys.exit( 1 ) + +options.config = os.path.abspath( options.config ) +os.chdir( os.path.dirname( options.config ) ) +sys.path.append( 'lib' ) + +from galaxy import eggs +import pkg_resources + +config = ConfigParser( dict( file_path = 'database/files', + id_secret = 'USING THE DEFAULT IS NOT SECURE!', + database_connection = 'sqlite:///database/universe.sqlite?isolation_level=IMMEDIATE' ) ) +config.read( os.path.basename( options.config ) ) + +from galaxy.web import security +from galaxy.model import mapping + +helper = security.SecurityHelper( id_secret = config.get( 'app:main', 'id_secret' ) ) +model = mapping.init( config.get( 'app:main', 'file_path' ), config.get( 'app:main', 'database_connection' ), create_tables = False ) + +if options.encode_id: + print 'Encoded "%s": %s' % ( options.encode_id, helper.encode_id( options.encode_id ) ) + +if options.decode_id: + print 'Decoded "%s": %s' % ( options.decode_id, helper.decode_id( options.decode_id ) ) + +if options.hda_id: + try: + hda_id = int( options.hda_id ) + except: + hda_id = int( helper.decode_id( options.hda_id ) ) + hda = model.context.current.query( model.HistoryDatasetAssociation ).get( hda_id ) + print 'HDA "%s" is Dataset "%s" at: %s' % ( hda.id, hda.dataset.id, hda.file_name ) + +if options.ldda_id: + try: + ldda_id = int( options.ldda_id ) + except: + ldda_id = int( helper.decode_id( options.ldda_id ) ) + ldda = model.context.current.query( model.HistoryDatasetAssociation ).get( ldda_id ) + print 'LDDA "%s" is Dataset "%s" at: %s' % ( ldda.id, ldda.dataset.id, ldda.file_name ) diff --git a/scripts/make_egg_packager.py b/scripts/make_egg_packager.py index dbe073ed731..d74b6a7132a 100644 --- a/scripts/make_egg_packager.py +++ b/scripts/make_egg_packager.py @@ -1,6 +1,16 @@ #!/usr/bin/env python import os, sys, logging, shutil +from optparse import OptionParser + +parser = OptionParser() +parser.add_option( '-c', '--config', dest='config', help='Path to Galaxy config file (universe_wsgi.ini)', default='universe_wsgi.ini' ) +parser.add_option( '-p', '--platform', dest='platform', help='Fetch for a specific platform (by default, eggs are fetched for *this* platform' ) +( options, args ) = parser.parse_args() + +if not os.path.exists( options.config ): + print "Config file does not exist (see 'python %s --help'): %s" % ( sys.argv[0], options.config ) + sys.exit( 1 ) root = logging.getLogger() root.setLevel( 10 ) @@ -13,12 +23,13 @@ from galaxy.eggs import Crate, EggNotFetchable, py import pkg_resources try: - platform = sys.argv[1] - c = Crate( platform = platform ) + assert options.platform + platform = options.platform + c = Crate( options.config, platform = platform ) print "Platform forced to '%s'" % platform except: platform = '-'.join( ( py, pkg_resources.get_platform() ) ) - c = Crate() + c = Crate( options.config ) print "Using Python interpreter at %s, Version %s" % ( sys.executable, sys.version ) print "This platform is '%s'" % platform print "Override with:" diff --git a/scripts/manage_db.py b/scripts/manage_db.py index fdb79435c5d..d3119736cf8 100644 --- a/scripts/manage_db.py +++ b/scripts/manage_db.py @@ -21,7 +21,15 @@ if sys.argv[-1] in [ 'community' ]: config_file = 'community_wsgi.ini' repo = 'lib/galaxy/webapps/community/model/migrate' else: + # Poor man's optparse config_file = 'universe_wsgi.ini' + if '-c' in sys.argv: + pos = sys.argv.index( '-c' ) + sys.argv.pop(pos) + config_file = sys.argv.pop( pos ) + if not os.path.exists( config_file ): + print "Galaxy config file does not exist (hint: use '-c config.ini' for non-standard locations): %s" % config_file + sys.exit( 1 ) repo = 'lib/galaxy/model/migrate' cp = SafeConfigParser() diff --git a/scripts/scramble.py b/scripts/scramble.py index 7ac09299c52..455cfaa380a 100644 --- a/scripts/scramble.py +++ b/scripts/scramble.py @@ -1,11 +1,14 @@ -""" -usage: scramble.py [egg_name] - With no arguments, scrambles all eggs necessary according to the - settings in universe_wsgi.ini. - egg_name - Scramble only this egg (as defined in eggs.ini) or 'all' - for all eggs (even those not required by your settings). -""" import os, sys, logging +from optparse import OptionParser + +parser = OptionParser() +parser.add_option( '-c', '--config', dest='config', help='Path to Galaxy config file (universe_wsgi.ini)', default='universe_wsgi.ini' ) +parser.add_option( '-e', '--egg-name', dest='egg_name', help='Egg name (as defined in eggs.ini) to fetch, or "all" for all eggs, even those not needed by your configuration' ) +( options, args ) = parser.parse_args() + +if not os.path.exists( options.config ): + print "Config file does not exist (see 'python %s --help'): %s" % ( sys.argv[0], options.config ) + sys.exit( 1 ) root = logging.getLogger() root.setLevel( 10 ) @@ -16,22 +19,25 @@ sys.path.append( lib ) from galaxy.eggs.scramble import ScrambleCrate, ScrambleFailure, EggNotFetchable -c = ScrambleCrate() +c = ScrambleCrate( options.config ) try: - if len( sys.argv ) == 1: + if not options.egg_name: eggs = c.scramble() - elif sys.argv[1] == 'all': + elif options.egg_name == 'all': c.scramble( all=True ) else: # Scramble a specific egg - name = sys.argv[1] + name = options.egg_name try: egg = c[name] except: print "error: %s not in eggs.ini" % name sys.exit( 1 ) for dependency in egg.dependencies: + config_arg = '' + if options.config != 'universe_wsgi.ini': + config_arg = '-c %s' % options.config print "Checking %s dependency: %s" % ( egg.name, dependency ) try: c[dependency].require() @@ -39,7 +45,7 @@ try: degg = e.eggs[0] print "%s build dependency %s %s couldn't be downloaded" % ( egg.name, degg.name, degg.version ) print "automatically. You can try building it by hand with:" - print " python scripts/scramble.py %s" % degg.name + print " python scripts/scramble.py %s-e %s" % ( config_arg, degg.name ) sys.exit( 1 ) egg.scramble() sys.exit( 0 ) diff --git a/scripts/scramble/lib/scramble_lib.py b/scripts/scramble/lib/scramble_lib.py index 8249b4f0cec..607b865b67b 100644 --- a/scripts/scramble/lib/scramble_lib.py +++ b/scripts/scramble/lib/scramble_lib.py @@ -26,7 +26,7 @@ def get_deps(): depf = open( '.galaxy_deps', 'r' ) except: return [] - c = eggs.Crate() + c = eggs.Crate( None ) for dep in depf: c[dep.strip()].require() diff --git a/scripts/set_dataset_sizes.py b/scripts/set_dataset_sizes.py new file mode 100644 index 00000000000..a39cf5e244b --- /dev/null +++ b/scripts/set_dataset_sizes.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +import os, sys +from ConfigParser import ConfigParser +from optparse import OptionParser + +default_config = os.path.abspath( os.path.join( os.path.dirname( __file__ ), '..', 'universe_wsgi.ini') ) + +parser = OptionParser() +parser.add_option( '-c', '--config', dest='config', help='Path to Galaxy config file (universe_wsgi.ini)', default=default_config ) +( options, args ) = parser.parse_args() + +def init(): + + options.config = os.path.abspath( options.config ) + os.chdir( os.path.dirname( options.config ) ) + sys.path.append( 'lib' ) + + from galaxy import eggs + import pkg_resources + + config = ConfigParser( dict( file_path = 'database/files', + database_connection = 'sqlite:///database/universe.sqlite?isolation_level=IMMEDIATE' ) ) + config.read( os.path.basename( options.config ) ) + + from galaxy.model import mapping + + return mapping.init( config.get( 'app:main', 'file_path' ), config.get( 'app:main', 'database_connection' ), create_tables = False ) + +if __name__ == '__main__': + print 'Loading Galaxy model...' + model = init() + sa_session = model.context.current + + set = 0 + dataset_count = sa_session.query( model.Dataset ).count() + print 'Processing %i datasets...' % dataset_count + percent = 0 + print 'Completed %i%%' % percent, + sys.stdout.flush() + for i, dataset in enumerate( sa_session.query( model.Dataset ).enable_eagerloads( False ).yield_per( 1000 ) ): + if dataset.total_size is None: + dataset.set_total_size() + set += 1 + if not set % 1000: + sa_session.flush() + new_percent = int( float(i) / dataset_count * 100 ) + if new_percent != percent: + percent = new_percent + print '\rCompleted %i%%' % percent, + sys.stdout.flush() + sa_session.flush() + print 'Completed 100%%' diff --git a/scripts/set_user_disk_usage.py b/scripts/set_user_disk_usage.py new file mode 100644 index 00000000000..0471ae2daf6 --- /dev/null +++ b/scripts/set_user_disk_usage.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python + +import os, sys +from ConfigParser import ConfigParser +from optparse import OptionParser + +default_config = os.path.abspath( os.path.join( os.path.dirname( __file__ ), '..', 'universe_wsgi.ini') ) + +parser = OptionParser() +parser.add_option( '-c', '--config', dest='config', help='Path to Galaxy config file (universe_wsgi.ini)', default=default_config ) +parser.add_option( '-u', '--username', dest='username', help='Username of user to update', default='all' ) +parser.add_option( '-e', '--email', dest='email', help='Email address of user to update', default='all' ) +parser.add_option( '--dry-run', dest='dryrun', help='Dry run (show changes but do not save to database)', action='store_true', default=False ) +( options, args ) = parser.parse_args() + +def init(): + + options.config = os.path.abspath( options.config ) + if options.username == 'all': + options.username = None + if options.email == 'all': + options.email = None + + os.chdir( os.path.dirname( options.config ) ) + sys.path.append( 'lib' ) + + from galaxy import eggs + import pkg_resources + + config = ConfigParser( dict( file_path = 'database/files', + database_connection = 'sqlite:///database/universe.sqlite?isolation_level=IMMEDIATE' ) ) + config.read( os.path.basename( options.config ) ) + + from galaxy.model import mapping + + return mapping.init( config.get( 'app:main', 'file_path' ), config.get( 'app:main', 'database_connection' ), create_tables = False ) + +def quotacheck( sa_session, users ): + sa_session.refresh( user ) + current = user.get_disk_usage() + print user.username, '<' + user.email + '> old usage:', str( current ) + ',', + new = user.calculate_disk_usage() + sa_session.refresh( user ) + # usage changed while calculating, do it again + if user.get_disk_usage() != current: + print 'usage changed while calculating, trying again...' + return quotacheck( sa_session, user ) + # yes, still a small race condition between here and the flush + if new == current: + print 'no change' + else: + print 'new usage:', new + if not options.dryrun: + user.set_disk_usage( new ) + sa_session.add( user ) + sa_session.flush() + +if __name__ == '__main__': + print 'Loading Galaxy model...' + model = init() + sa_session = model.context.current + + if not options.username and not options.email: + user_count = sa_session.query( model.User ).count() + print 'Processing %i users...' % user_count + for i, user in enumerate( sa_session.query( model.User ).enable_eagerloads( False ).yield_per( 1000 ) ): + print '%3i%%' % int( float(i) / user_count * 100 ), + quotacheck( sa_session, user ) + print '100% complete' + sys.exit( 0 ) + elif options.username: + user = sa_session.query( model.User ).enable_eagerloads( False ).filter_by( username=options.username ).first() + elif options.email: + user = sa_session.query( model.User ).enable_eagerloads( False ).filter_by( email=options.email ).first() + if not user: + print 'User not found' + sys.exit( 1 ) + quotacheck( sa_session, user ) diff --git a/scripts/tool_shed/migrate_tools_to_repositories.py b/scripts/tool_shed/migrate_tools_to_repositories.py index 2948adf08fa..37068de94c3 100644 --- a/scripts/tool_shed/migrate_tools_to_repositories.py +++ b/scripts/tool_shed/migrate_tools_to_repositories.py @@ -8,6 +8,8 @@ associated with them, and migrates old tool shed stuff to new tool shed stuff. ====== CRITICAL ======= +0. This script must be run on a repo updated to changeset: 5621:4618be57481b + 1. Before running this script, make sure the following config setting is set in community_wsgi.ini # Enable next-gen tool shed features @@ -16,10 +18,6 @@ enable_next_gen_tool_shed = True 2. This script requires the Galaxy instance to use Postgres for database storage. To run this script, use "sh migrate_tools_to_repositories.sh" from this directory - -TODO: This script currently creates hg repos under the name of the user running the script. When -we get the hgweb stuff working, see if we can correct this, creating repos under the user name of the -user that uploaded the tool archive. ''' import sys, os, subprocess, ConfigParser, shutil, tarfile, tempfile @@ -61,12 +59,12 @@ def get_versions( app, item ): this_item = item while item.newer_version: if item.newer_version.state in valid_states: - versions.insert( 0, item.newer_version ) + versions.append( item.newer_version ) item = item.newer_version item = this_item while item.older_version: if item.older_version[ 0 ].state in valid_states: - versions.append( item.older_version[ 0 ] ) + versions.insert( 0, item.older_version[ 0 ] ) item = item.older_version[ 0 ] return versions diff --git a/shed_tool_conf.xml.sample b/shed_tool_conf.xml.sample new file mode 100644 index 00000000000..31ca79b8421 --- /dev/null +++ b/shed_tool_conf.xml.sample @@ -0,0 +1,3 @@ + + + diff --git a/static/gmaj/gmaj.jar b/static/gmaj/gmaj.jar index cbb6cf5d11b..8c685ca5d33 100644 Binary files a/static/gmaj/gmaj.jar and b/static/gmaj/gmaj.jar differ diff --git a/static/images/delete_icon_grey.png b/static/images/delete_icon_grey.png new file mode 100644 index 00000000000..d7c4b80bd6e Binary files /dev/null and b/static/images/delete_icon_grey.png differ diff --git a/static/images/fugue/arrow-transition-270-bw.png b/static/images/fugue/arrow-transition-270-bw.png new file mode 100644 index 00000000000..f1994aefa39 Binary files /dev/null and b/static/images/fugue/arrow-transition-270-bw.png differ diff --git a/static/images/fugue/arrow-transition-bw.png b/static/images/fugue/arrow-transition-bw.png new file mode 100644 index 00000000000..d846a36ec49 Binary files /dev/null and b/static/images/fugue/arrow-transition-bw.png differ diff --git a/static/images/tracks/block.png b/static/images/tracks/block.png new file mode 100644 index 00000000000..73d06c2bb24 Binary files /dev/null and b/static/images/tracks/block.png differ diff --git a/static/june_2007_style/base.css.tmpl b/static/june_2007_style/base.css.tmpl index 8d87a573fbc..998165bb0d7 100644 --- a/static/june_2007_style/base.css.tmpl +++ b/static/june_2007_style/base.css.tmpl @@ -829,6 +829,10 @@ div.permissionContainer { -sprite-group: history-buttons; -sprite-image: delete_icon_dark.png; } +.icon-button.delete_disabled { + -sprite-group: history-buttons; + -sprite-image: delete_icon_grey.png; +} .icon-button.edit { -sprite-group: history-buttons; -sprite-image: pencil_icon.png; diff --git a/static/june_2007_style/blue/base.css b/static/june_2007_style/blue/base.css index 2b21517d9ff..a6caed06955 100644 --- a/static/june_2007_style/blue/base.css +++ b/static/june_2007_style/blue/base.css @@ -143,9 +143,10 @@ div.permissionContainer{padding-left:20px;} .icon-button.display_disabled{background:url(history-buttons.png) no-repeat 0px -52px;} .icon-button.delete{background:url(history-buttons.png) no-repeat 0px -78px;} .icon-button.delete:hover{background:url(history-buttons.png) no-repeat 0px -104px;} -.icon-button.edit{background:url(history-buttons.png) no-repeat 0px -130px;} -.icon-button.edit:hover{background:url(history-buttons.png) no-repeat 0px -156px;} -.icon-button.edit_disabled{background:url(history-buttons.png) no-repeat 0px -182px;} +.icon-button.delete_disabled{background:url(history-buttons.png) no-repeat 0px -130px;} +.icon-button.edit{background:url(history-buttons.png) no-repeat 0px -156px;} +.icon-button.edit:hover{background:url(history-buttons.png) no-repeat 0px -182px;} +.icon-button.edit_disabled{background:url(history-buttons.png) no-repeat 0px -208px;} .icon-button.tag{background:url(fugue.png) no-repeat 0px -0px;} .icon-button.tags{background:url(fugue.png) no-repeat 0px -26px;} .icon-button.tag--plus{background:url(fugue.png) no-repeat 0px -52px;} diff --git a/static/june_2007_style/blue/history-buttons.png b/static/june_2007_style/blue/history-buttons.png index e967d498b89..8ba6bc80c78 100644 Binary files a/static/june_2007_style/blue/history-buttons.png and b/static/june_2007_style/blue/history-buttons.png differ diff --git a/static/june_2007_style/blue/panel_layout.css b/static/june_2007_style/blue/panel_layout.css index 1620c11374b..298ee40af41 100644 --- a/static/june_2007_style/blue/panel_layout.css +++ b/static/june_2007_style/blue/panel_layout.css @@ -1,4 +1,3 @@ -body,html{overflow:hidden;margin:0;padding:0;width:100%;height:100%;} body{font:75% "Lucida Grande",verdana,arial,helvetica,sans-serif;background:#eee;} .unselectable{user-select:none;-moz-user-select:none;-webkit-user-select:none;} #background{position:absolute;background:#eee;z-index:-1;top:0;left:0;margin:0;padding:0;width:100%;height:100%;} @@ -35,8 +34,14 @@ div.unified-panel-body{position:absolute;top:2em;bottom:0;width:100%;margin-top: .panel-info-message{background-image:url(info_small.png);background-color:#CCCCFF;} #masthead{position:absolute;top:0;left:0;width:100%;min-width:900px;height:32px;background:#2C3143;color:#fff;border-bottom:solid #444 1px;z-index:15000;padding:0;} #masthead a{color:#eeeeee;text-decoration:none;} -#masthead .title{font-family:verdana;padding:3px 10px;font-size:175%;font-weight:bold;} +#masthead .title{font-family:verdana;padding:3px 10px;font-size:175%;font-weight:bold;z-index:-1;} #masthead a:hover{text-decoration:underline;} +.quota-meter-container{position:absolute;top:0;right:0;height:32px;} +.quota-meter{position:absolute;top:8px;right:8px;height:16px;width:100px;background-color:#C1C9E5;;} +.quota-meter-bar{position:absolute;top:0;left:0;height:16px;background-color:#969DB3;;} +.quota-meter-bar-warn{background-color:#FFB400;;} +.quota-meter-bar-error{background-color:#FF4343;;} +.quota-meter-text{position:absolute;top:50%;left:0;width:100px;height:16px;margin-top:-6px;text-align:center;z-index:9001;color:#000;;} .tab-group{margin:0;padding:0 10px;height:100%;white-space:nowrap;cursor:default;background:transparent;} .tab-group .tab{background:#2C3143;position:relative;float:left;margin:0;padding:0 1em;height:32px;line-height:32px;text-align:left;} .tab-group .tab .submenu{display:none;position:absolute;z-index:16000;left:0;top:32px;padding:1em;margin:-1em;padding-top:0;margin-top:0;background-color:rgba(0,0,0,0.5);-moz-border-radius:0 0 1em 1em;-webkit-border-bottom-right-radius:1em;-webkit-border-bottom-left-radius:1em;} diff --git a/static/june_2007_style/blue/trackster.css b/static/june_2007_style/blue/trackster.css index 077866f050b..240576b1da7 100644 --- a/static/june_2007_style/blue/trackster.css +++ b/static/june_2007_style/blue/trackster.css @@ -19,13 +19,16 @@ .viewport-canvas{width:100%;height:100px;} .yaxislabel{color:#777;z-index:100;} .line-track .track-content{border-top:1px solid #eee;border-bottom:1px solid #eee;} +.group-handle{cursor:move;float:left;background:#eee url('/static/images/tracks/block.png');width:12px;height:12px;} +.group{min-height:20px;border-top:2px solid #888;border-bottom:2px solid #888;} .track{background:white;} .track-header{text-align:left;padding:4px 0px;color:#666;} .track-header .menubutton{margin-left:0px;} .track-content{overflow:hidden;text-align:center;border-top:1px solid #eee;border-bottom:2px solid #eee;background:#eee url('/static/images/tracks/diag_bg.gif');min-height:16px;} .label-track .track-content{background:white;} .track-tile{background:white;} -.track-tile canvas{position:relative;z-index:100;border:solid white;border-width:2px 0px 0px 0px;} +.track-tile canvas{position:relative;z-index:100;} +.tile-message{border-bottom:solid 1px red;text-align:center;color:red;background-color:white;} .track.error .track-content{background-color:#ECB4AF;background-image:none;} .track.nodata .track-content{background-color:#eee;background-image:none;} .track.pending .track-content{background-color:white;background-image:none;} @@ -38,12 +41,24 @@ .top-labeltrack{position:relative;border-bottom:solid #999 1px;} .nav-labeltrack{border-top:solid #999 1px;border-bottom:solid #333 1px;} input{font:10px verdana;} -.dynamic-tool,.filters{width:410px;margin-left:0.25em;padding-bottom:0.5em;} +.dynamic-tool,.filters{margin-left:0.25em;padding-bottom:0.5em;} +.dynamic-tool{width:410px;} +.filters>.sliders,.display-controls{float:left;margin:1em;} +.sliders{width:410px;} +.display-controls{border-left:solid 2px #DDDDDD;padding-left:1em} .slider-row{margin-top:0.4em;margin-left:1em;} -.slider-label{float:left;font-weight:bold;} +.elt-label{float:left;font-weight:bold;margin-right:1em;} .slider{float:right;width:200px;position:relative;} .tool-name{font-size:110%;font-weight:bold;} .param-row{margin-top:0.2em;margin-left:1em;} .param-label{float:left;font-weight:bold;padding-top:0.2em;} .child-track-icon{background:url('../images/fugue/arrow-000-small-bw.png') no-repeat;width:30px;cursor:move;} .track-resize{background:white url('../images/visualization/draggable_vertical.png') no-repeat top center;position:absolute;right:3px;bottom:-4px;width:14px;height:7px;border:solid #999 1px;z-index:100;} +.bookmark{background:white;border:solid #999 1px;border-right:none;margin:0.5em;margin-right:0;padding:0.5em;} +.bookmark .position{font-weight:bold;} +.delete-icon-container{float:right;} +.icon{display:inline-block;width:16px;height:16px;} +.icon.more-down{background:url('../images/fugue/arrow-transition-270-bw.png') no-repeat 0px 0px;} +.icon.more-across{background:url('../images/fugue/arrow-transition-bw.png') no-repeat 0px 0px;} +.intro{padding:1em;} +.intro > .action-button{background-color:#CCC;padding:1em;} diff --git a/static/june_2007_style/blue_colors.ini b/static/june_2007_style/blue_colors.ini index 3c6b9631458..b0fd5ba5482 100644 --- a/static/june_2007_style/blue_colors.ini +++ b/static/june_2007_style/blue_colors.ini @@ -59,6 +59,12 @@ masthead_text=#eeeeee masthead_bg_hatch=- masthead_link=#eeeeee masthead_active_tab_bg=#222532 +# Quota meter +quota_meter_bg=#C1C9E5; +quota_meter_bar=#969DB3; +quota_meter_warn_bar=#FFB400; +quota_meter_error_bar=#FF4343; +quota_meter_text=#000; # ---- Layout ----------------------------------------------------------------- # Overall background color (including space between panels) layout_bg=#eee diff --git a/static/june_2007_style/masthead.css.tmpl b/static/june_2007_style/masthead.css.tmpl index a6d3b7ea5c7..f80fcfbef08 100644 --- a/static/june_2007_style/masthead.css.tmpl +++ b/static/june_2007_style/masthead.css.tmpl @@ -59,4 +59,4 @@ span.link-group span.active-link margin-left: -3px; margin-right: -3px; padding-bottom: 10px; margin-bottom: -10px; -} \ No newline at end of file +} diff --git a/static/june_2007_style/panel_layout.css.tmpl b/static/june_2007_style/panel_layout.css.tmpl index 6a6070a3da2..614079a91a4 100644 --- a/static/june_2007_style/panel_layout.css.tmpl +++ b/static/june_2007_style/panel_layout.css.tmpl @@ -1,11 +1,3 @@ -body, html { - overflow: hidden; - margin: 0; - padding: 0; - width: 100%; - height: 100%; -} - body { font: 75% "Lucida Grande",verdana,arial,helvetica,sans-serif; background: ${layout_bg}; @@ -259,6 +251,7 @@ div.unified-panel-body { padding: 3px 10px; font-size: 175%; font-weight: bold; + z-index: -1; } } @@ -266,6 +259,56 @@ div.unified-panel-body { text-decoration: underline; } +.quota-meter-container +{ + position: absolute; + top: 0; + right: 0; + height: 32px; +} + +.quota-meter +{ + position: absolute; + top: 8px; + right: 8px; + height: 16px; + width: 100px; + background-color: $quota_meter_bg; +} + +.quota-meter-bar +{ + position: absolute; + top: 0; + left: 0; + height: 16px; + background-color: $quota_meter_bar; +} + +.quota-meter-bar-warn +{ + background-color: $quota_meter_warn_bar; +} + +.quota-meter-bar-error +{ + background-color: $quota_meter_error_bar; +} + +.quota-meter-text +{ + position: absolute; + top: 50%; + left: 0; + width: 100px; + height: 16px; + margin-top: -6px; + text-align: center; + z-index: 9001; + color: $quota_meter_text; +} + ## Tabs .tab-group { diff --git a/static/june_2007_style/trackster.css.tmpl b/static/june_2007_style/trackster.css.tmpl index 722804991b1..2b58db5bb43 100644 --- a/static/june_2007_style/trackster.css.tmpl +++ b/static/june_2007_style/trackster.css.tmpl @@ -126,7 +126,19 @@ border-top: 1px solid #eee; border-bottom: 1px solid #eee; } - + +.group-handle { + cursor: move; + float: left; + background: #eee url('/static/images/tracks/block.png'); + width: 12px; + height: 12px; +} +.group { + min-height: 20px; + border-top: 2px solid #888; + border-bottom: 2px solid #888; +} .track { /* border-top: solid #DDDDDD 1px; */ /* border-bottom: solid #DDDDDD 1px; */ @@ -163,8 +175,13 @@ .track-tile canvas { position: relative; z-index: 100; - border: solid white; - border-width: 2px 0px 0px 0px; +} + +.tile-message { + border-bottom: solid 1px red; + text-align: center; + color: red; + background-color: white; } .track.error .track-content { @@ -226,17 +243,31 @@ input { font: 10px verdana; } .dynamic-tool, .filters { - width: 410px; margin-left: 0.25em; padding-bottom:0.5em; } +.dynamic-tool { + width:410px; +} +.filters > .sliders, .display-controls { + float: left; + margin: 1em; +} +.sliders{ + width: 410px; +} +.display-controls{ + border-left: solid 2px #DDDDDD; + padding-left: 1em +} .slider-row { margin-top: 0.4em; margin-left: 1em; } -.slider-label { +.elt-label { float: left; font-weight: bold; + margin-right: 1em; } .slider { float: right; @@ -271,4 +302,37 @@ input { border: solid #999 1px; z-index: 100; } +.bookmark { + background:white; + border:solid #999 1px; + border-right:none; + margin:0.5em; + margin-right:0; + padding:0.5em; +} +.bookmark .position { + font-weight:bold; +} +.delete-icon-container { + float:right; +} + +.icon { + display:inline-block; + width:16px; + height:16px; +} +.icon.more-down { + background:url('../images/fugue/arrow-transition-270-bw.png') no-repeat 0px 0px; +} +.icon.more-across { + background: url('../images/fugue/arrow-transition-bw.png') no-repeat 0px 0px; +} +.intro { + padding: 1em; +} +.intro > .action-button { + background-color: #CCC; + padding: 1em; +} diff --git a/static/scripts/galaxy.base.js b/static/scripts/galaxy.base.js index 6b086c3bc8f..fa0fe391dd2 100644 --- a/static/scripts/galaxy.base.js +++ b/static/scripts/galaxy.base.js @@ -330,7 +330,94 @@ function replace_big_select_inputs(min_length, max_length) { }); } -// Edit and save text asynchronously. +/** + * Returns editable text element. Element is a div with text: (a) when user clicks on text, a textbox/area + * enables user to edit text; (b) when user presses enter key, element's text is set. + */ +// TODO: use this function to implement async_save_text (implemented below). +function get_editable_text_elt(text, use_textarea, num_cols, num_rows, on_finish) { + // Set defaults if necessary. + if (num_cols === undefined) { + num_cols = 30; + } + if (num_rows === undefined) { + num_rows = 4; + } + + // Create div for element. + var container = $("
    ").addClass("editable-text").text(text).click(function() { + // If there's already an input element, editing is active, so do nothing. + if ($(this).children(":input").length > 0) { + return; + } + + container.removeClass("editable-text"); + + // Handler for setting element text. + var set_text = function(new_text, do_on_finish) { + container.find(":input").remove(); + + if (new_text != "") { + container.text(new_text); + } + else { + // No text; need a line so that there is a click target. + container.html("
    "); + } + container.addClass("editable-text"); + + if (do_on_finish && on_finish) { + on_finish(new_text); + } + }; + + // Create input element(s) for editing. + var cur_text = container.text(), + input_elt, button_elt; + + if (use_textarea) { + input_elt = $("").attr({rows:i,cols:c}).text($.trim(k))}else{j=$("").attr({value:$.trim(k),size:c})}j.attr("id","renaming-active");j.blur(function(){$(this).remove();l.show();if(b){b(j)}});j.keyup(function(n){if(n.keyCode===27){$(this).trigger("blur")}else{if(n.keyCode===13){var m={};m[a]=$(this).val();$(this).trigger("blur");$.ajax({url:e,data:m,error:function(){alert("Text editing for elt "+f+" failed")},success:function(o){if(o!==""){l.text(o)}else{l.html("None")}if(b){b(j)}}})}}});if(g){g(j)}l.hide();j.insertAfter(l);j.focus();j.select();return})}function init_history_items(d,a,c){var b=function(){try{var e=$.jStorage.get("history_expand_state");if(e){for(var g in e){$("#"+g+" div.historyItemBody").show()}}}catch(f){$.jStorage.deleteKey("history_expand_state")}if($.browser.mozilla){$("div.historyItemBody").each(function(){if(!$(this).is(":visible")){$(this).find("pre.peek").css("overflow","hidden")}})}d.each(function(){var j=this.id,h=$(this).children("div.historyItemBody"),i=h.find("pre.peek");$(this).find(".historyItemTitleBar > .historyItemTitle").wrap("").click(function(){var k;if(h.is(":visible")){if($.browser.mozilla){i.css("overflow","hidden")}h.slideUp("fast");if(!c){k=$.jStorage.get("history_expand_state");if(k){delete k[j];$.jStorage.set("history_expand_state",k)}}}else{h.slideDown("fast",function(){if($.browser.mozilla){i.css("overflow","auto")}});if(!c){k=$.jStorage.get("history_expand_state");if(!k){k={}}k[j]=true;$.jStorage.set("history_expand_state",k)}}return false})});$("#top-links > a.toggle").click(function(){var h=$.jStorage.get("history_expand_state");if(!h){h={}}$("div.historyItemBody:visible").each(function(){if($.browser.mozilla){$(this).find("pre.peek").css("overflow","hidden")}$(this).slideUp("fast");if(h){delete h[$(this).parent().attr("id")]}});$.jStorage.set("history_expand_state",h)}).show()};b()}function commatize(b){b+="";var a=/(\d+)(\d{3})/;while(a.test(b)){b=b.replace(a,"$1,$2")}return b}function reset_tool_search(a){var c=$("#galaxy_tools").contents();if(c.length===0){c=$(document)}$(this).removeClass("search_active");c.find(".toolTitle").removeClass("search_match");c.find(".toolSectionBody").hide();c.find(".toolTitle").show();c.find(".toolPanelLabel").show();c.find(".toolSectionWrapper").each(function(){if($(this).attr("id")!="recently_used_wrapper"){$(this).show()}else{if($(this).hasClass("user_pref_visible")){$(this).show()}}});c.find("#search-no-results").hide();c.find("#search-spinner").hide();if(a){var b=c.find("#tool-search-query");b.val("search tools");b.css("font-style","italic")}}var GalaxyAsync=function(a){this.url_dict={};this.log_action=(a===undefined?false:a)};GalaxyAsync.prototype.set_func_url=function(a,b){this.url_dict[a]=b};GalaxyAsync.prototype.set_user_pref=function(a,b){var c=this.url_dict[arguments.callee];if(c===undefined){return false}$.ajax({url:c,data:{pref_name:a,pref_value:b},error:function(){return false},success:function(){return true}})};GalaxyAsync.prototype.log_user_action=function(c,b,d){if(!this.log_action){return}var a=this.url_dict[arguments.callee];if(a===undefined){return false}$.ajax({url:a,data:{action:c,context:b,params:d},error:function(){return false},success:function(){return true}})};$(document).ready(function(){$("select[refresh_on_change='true']").change(function(){var a=$(this),e=a.val(),d=false,c=a.attr("refresh_on_change_values");if(c){c=c.split(",");var b=a.attr("last_selected_value");if($.inArray(e,c)===-1&&$.inArray(b,c)===-1){return}}$(window).trigger("refresh_on_change");$(document).trigger("convert_to_values");a.get(0).form.submit()});$("a[confirm]").click(function(){return confirm($(this).attr("confirm"))});if($.fn.tipsy){$(".tooltip").tipsy({gravity:"s"})}make_popup_menus();replace_big_select_inputs(20,1500);$("a").click(function(){var b=$(this);var c=(parent.frames&&parent.frames.galaxy_main);if((b.attr("target")=="galaxy_main")&&(!c)){var a=b.attr("href");if(a.indexOf("?")==-1){a+="?"}else{a+="&"}a+="use_panels=True";b.attr("href",a);b.attr("target","_self")}return b})}); \ No newline at end of file +if(!Array.indexOf){Array.prototype.indexOf=function(c){for(var b=0,a=this.length;b");var f=b.data("menu_options");if(obj_length(f)<=0){$("
  • No Options.
    • ").appendTo(g)}$.each(f,function(j,i){if(i){$("
  • ").html(j).click(i).appendTo(g)}else{$("
  • ").html(j).appendTo(g)}});var h=$("
    ");h.append(g).append("
    ").appendTo("body");var e=d.pageX-h.width()/2;e=Math.min(e,$(document).scrollLeft()+$(window).width()-$(h).width()-20);e=Math.max(e,$(document).scrollLeft()+20);h.css({top:d.pageY-15,left:e})},10);setTimeout(function(){var f=function(h){$(h).bind("click.close_popup",function(){$(".popmenu-wrapper").remove();h.unbind("click.close_popup")})};f($(window.document));f($(window.top.document));for(var e=window.top.frames.length;e--;){var g=$(window.top.frames[e].document);f(g)}},50);return false})}function make_popup_menus(){jQuery("div[popupmenu]").each(function(){var a={};var c=$(this);c.find("a").each(function(){var f=$(this),h=f.get(0);var d=h.getAttribute("confirm"),e=h.getAttribute("href"),g=h.getAttribute("target");if(!e){a[f.text()]=null}else{a[f.text()]=function(){if(!d||confirm(d)){var i;if(g=="_parent"){window.parent.location=e}else{if(g=="_top"){window.top.location=e}else{if(g=="demo"){if(i==undefined||i.closed){i=window.open(e,g);i.creator=self}}else{window.location=e}}}}}}});var b=$("#"+c.attr("popupmenu"));b.find("a").bind("click",function(d){d.stopPropagation();return true});make_popupmenu(b,a);b.addClass("popup");c.remove()})}function naturalSort(j,h){var p=/(-?[0-9\.]+)/g,k=j.toString().toLowerCase()||"",g=h.toString().toLowerCase()||"",l=String.fromCharCode(0),n=k.replace(p,l+"$1"+l).split(l),e=g.replace(p,l+"$1"+l).split(l),d=(new Date(k)).getTime(),o=d?(new Date(g)).getTime():null;if(o){if(do){return 1}}}var m,f;for(var i=0,c=Math.max(n.length,e.length);if){return 1}}}return 0}function replace_big_select_inputs(a,b){if(!jQuery().autocomplete){return}if(a===undefined){a=20}if(b===undefined){b=3000}$("select").each(function(){var d=$(this);var g=d.find("option").length;if((gb)){return}if(d.attr("multiple")===true){return}if(d.hasClass("no-autocomplete")){return}var m=d.attr("value");var c=$("");c.attr("size",40);c.attr("name",d.attr("name"));c.attr("id",d.attr("id"));c.click(function(){var n=$(this).val();$(this).val("Loading...");$(this).showAllInCache();$(this).val(n);$(this).select()});var e=[];var i={};d.children("option").each(function(){var o=$(this).text();var n=$(this).attr("value");e.push(o);i[o]=n;i[n]=n;if(n==m){c.attr("value",o)}});if(m===""||m==="?"){c.attr("value","Click to Search or Select")}if(d.attr("name")=="dbkey"){e=e.sort(naturalSort)}var f={selectFirst:false,autoFill:false,mustMatch:false,matchContains:true,max:b,minChars:0,hideForLessThanMinChars:false};c.autocomplete(e,f);d.replaceWith(c);var k=function(){var o=c.attr("value");var n=i[o];if(n!==null&&n!==undefined){c.attr("value",n)}else{if(m!==""){c.attr("value",m)}else{c.attr("value","?")}}};c.parents("form").submit(function(){k()});$(document).bind("convert_to_values",function(){k()});if(d.attr("refresh_on_change")=="true"){var h=d.attr("refresh_on_change_values"),l=d.attr("last_selected_value");if(h!==undefined){h=h.split(",")}var j=function(){var n=i[c.attr("value")];if(l!==n&&n!==null&&n!==undefined){if(h!==undefined&&$.inArray(n,h)===-1&&$.inArray(l,h)===-1){return}c.attr("value",n);$(window).trigger("refresh_on_change");c.parents("form").submit()}};c.bind("result",j);c.keyup(function(n){if(n.keyCode===13){j()}});c.keydown(function(n){if(n.keyCode===13){return false}})}})}function get_editable_text_elt(f,e,d,c,b){if(d===undefined){d=30}if(c===undefined){c=4}var a=$("
    ").addClass("editable-text").text(f).click(function(){if($(this).children(":input").length>0){return}a.removeClass("editable-text");var h=function(l,k){a.find(":input").remove();if(l!=""){a.text(l)}else{a.html("
    ")}a.addClass("editable-text");if(k&&b){b(l)}};var g=a.text(),j,i;if(e){j=$("").attr({rows:i,cols:c}).text($.trim(k))}else{j=$("").attr({value:$.trim(k),size:c})}j.attr("id","renaming-active");j.blur(function(){$(this).remove();l.show();if(b){b(j)}});j.keyup(function(n){if(n.keyCode===27){$(this).trigger("blur")}else{if(n.keyCode===13){var m={};m[a]=$(this).val();$(this).trigger("blur");$.ajax({url:e,data:m,error:function(){alert("Text editing for elt "+f+" failed")},success:function(o){if(o!==""){l.text(o)}else{l.html("None")}if(b){b(j)}}})}}});if(g){g(j)}l.hide();j.insertAfter(l);j.focus();j.select();return})}function init_history_items(d,a,c){var b=function(){try{var e=$.jStorage.get("history_expand_state");if(e){for(var g in e){$("#"+g+" div.historyItemBody").show()}}}catch(f){$.jStorage.deleteKey("history_expand_state")}if($.browser.mozilla){$("div.historyItemBody").each(function(){if(!$(this).is(":visible")){$(this).find("pre.peek").css("overflow","hidden")}})}d.each(function(){var j=this.id,h=$(this).children("div.historyItemBody"),i=h.find("pre.peek");$(this).find(".historyItemTitleBar > .historyItemTitle").wrap("").click(function(){var k;if(h.is(":visible")){if($.browser.mozilla){i.css("overflow","hidden")}h.slideUp("fast");if(!c){k=$.jStorage.get("history_expand_state");if(k){delete k[j];$.jStorage.set("history_expand_state",k)}}}else{h.slideDown("fast",function(){if($.browser.mozilla){i.css("overflow","auto")}});if(!c){k=$.jStorage.get("history_expand_state");if(!k){k={}}k[j]=true;$.jStorage.set("history_expand_state",k)}}return false})});$("#top-links > a.toggle").click(function(){var h=$.jStorage.get("history_expand_state");if(!h){h={}}$("div.historyItemBody:visible").each(function(){if($.browser.mozilla){$(this).find("pre.peek").css("overflow","hidden")}$(this).slideUp("fast");if(h){delete h[$(this).parent().attr("id")]}});$.jStorage.set("history_expand_state",h)}).show()};b()}function commatize(b){b+="";var a=/(\d+)(\d{3})/;while(a.test(b)){b=b.replace(a,"$1,$2")}return b}function reset_tool_search(a){var c=$("#galaxy_tools").contents();if(c.length===0){c=$(document)}$(this).removeClass("search_active");c.find(".toolTitle").removeClass("search_match");c.find(".toolSectionBody").hide();c.find(".toolTitle").show();c.find(".toolPanelLabel").show();c.find(".toolSectionWrapper").each(function(){if($(this).attr("id")!="recently_used_wrapper"){$(this).show()}else{if($(this).hasClass("user_pref_visible")){$(this).show()}}});c.find("#search-no-results").hide();c.find("#search-spinner").hide();if(a){var b=c.find("#tool-search-query");b.val("search tools");b.css("font-style","italic")}}var GalaxyAsync=function(a){this.url_dict={};this.log_action=(a===undefined?false:a)};GalaxyAsync.prototype.set_func_url=function(a,b){this.url_dict[a]=b};GalaxyAsync.prototype.set_user_pref=function(a,b){var c=this.url_dict[arguments.callee];if(c===undefined){return false}$.ajax({url:c,data:{pref_name:a,pref_value:b},error:function(){return false},success:function(){return true}})};GalaxyAsync.prototype.log_user_action=function(c,b,d){if(!this.log_action){return}var a=this.url_dict[arguments.callee];if(a===undefined){return false}$.ajax({url:a,data:{action:c,context:b,params:d},error:function(){return false},success:function(){return true}})};$(document).ready(function(){$("select[refresh_on_change='true']").change(function(){var a=$(this),e=a.val(),d=false,c=a.attr("refresh_on_change_values");if(c){c=c.split(",");var b=a.attr("last_selected_value");if($.inArray(e,c)===-1&&$.inArray(b,c)===-1){return}}$(window).trigger("refresh_on_change");$(document).trigger("convert_to_values");a.get(0).form.submit()});$(":checkbox[refresh_on_change='true']").click(function(){var a=$(this),e=a.val(),d=false,c=a.attr("refresh_on_change_values");if(c){c=c.split(",");var b=a.attr("last_selected_value");if($.inArray(e,c)===-1&&$.inArray(b,c)===-1){return}}$(window).trigger("refresh_on_change");a.get(0).form.submit()});$("a[confirm]").click(function(){return confirm($(this).attr("confirm"))});if($.fn.tipsy){$(".tooltip").tipsy({gravity:"s"})}make_popup_menus();replace_big_select_inputs(20,1500);$("a").click(function(){var b=$(this);var c=(parent.frames&&parent.frames.galaxy_main);if((b.attr("target")=="galaxy_main")&&(!c)){var a=b.attr("href");if(a.indexOf("?")==-1){a+="?"}else{a+="&"}a+="use_panels=True";b.attr("href",a);b.attr("target","_self")}return b})}); \ No newline at end of file diff --git a/static/scripts/packed/trackster.js b/static/scripts/packed/trackster.js index f1c415af3ee..07737a929f9 100644 --- a/static/scripts/packed/trackster.js +++ b/static/scripts/packed/trackster.js @@ -1 +1 @@ -var class_module=function(b,a){var c=function(){var f=arguments[0];for(var e=1;ec){a=AFTER}else{if(f<=c){a=CONTAINED_BY}else{a=OVERLAP_END}}}return a};var is_overlap=function(c,b){var a=compute_overlap(c,b);return(a!==BEFORE&&a!==AFTER)};var trackster_module=function(f,T){var n=f("class").extend,p=f("slotting"),I=f("painters");var Z=function(aa,ab){this.document=aa;this.default_font=ab!==undefined?ab:"9px Monaco, Lucida Console, monospace";this.dummy_canvas=this.new_canvas();this.dummy_context=this.dummy_canvas.getContext("2d");this.dummy_context.font=this.default_font;this.char_width_px=this.dummy_context.measureText("A").width;this.patterns={};this.load_pattern("right_strand","/visualization/strand_right.png");this.load_pattern("left_strand","/visualization/strand_left.png");this.load_pattern("right_strand_inv","/visualization/strand_right_inv.png");this.load_pattern("left_strand_inv","/visualization/strand_left_inv.png")};n(Z.prototype,{load_pattern:function(aa,ae){var ab=this.patterns,ac=this.dummy_context,ad=new Image();ad.src=image_path+ae;ad.onload=function(){ab[aa]=ac.createPattern(ad,"repeat")}},get_pattern:function(aa){return this.patterns[aa]},new_canvas:function(){var aa=this.document.createElement("canvas");if(window.G_vmlCanvasManager){G_vmlCanvasManager.initElement(aa)}aa.manager=this;return aa}});var C=function(aa,ab){aa.bind("drag",{handle:ab,relative:true},function(af,ag){var ae=$(this).parent();var ad=ae.children();var ac;for(ac=0;ac=this.num_elements){var aa=this.key_ary.shift();delete this.obj_cache[aa]}this.key_ary.push(ab)}this.obj_cache[ab]=ac;return ac},move_key_to_end:function(ab,aa){this.key_ary.splice(aa,1);this.key_ary.push(ab)},clear:function(){this.obj_cache={};this.key_ary=[]},size:function(){return this.key_ary.length}});var N=function(ab,aa,ac){c.call(this,ab);this.track=aa;this.subset=(ac!==undefined?ac:true)};n(N.prototype,c.prototype,{load_data:function(ai,aj,ae,ah,ab,ag){var ad={chrom:ai,low:aj,high:ae,mode:ah,resolution:ab,dataset_id:this.track.dataset_id,hda_ldda:this.track.hda_ldda};$.extend(ad,ag);if(this.track.filters_manager){var ak=[];var aa=this.track.filters_manager.filters;for(var af=0;af1){return}return N.prototype.load_data.call(this,ac,aa,ae,af,ab,ad)}});var Y=function(aa,ad,ac,ab,ae){this.container=aa;this.chrom=null;this.vis_id=ac;this.dbkey=ab;this.title=ad;this.tracks=[];this.label_tracks=[];this.max_low=0;this.max_high=0;this.num_tracks=0;this.track_id_counter=0;this.zoom_factor=3;this.min_separation=30;this.has_changes=false;this.init(ae);this.canvas_manager=new Z(aa.get(0).ownerDocument);this.reset()};n(Y.prototype,{init:function(ad){var ac=this.container,aa=this;this.top_container=$("
    ").addClass("top-container").appendTo(ac);this.content_div=$("
    ").addClass("content").css("position","relative").appendTo(ac);this.bottom_container=$("
    ").addClass("bottom-container").appendTo(ac);this.top_labeltrack=$("
    ").addClass("top-labeltrack").appendTo(this.top_container);this.viewport_container=$("
    ").addClass("viewport-container").addClass("viewport-container").appendTo(this.content_div);this.intro_div=$("
    ").addClass("intro").text("Select a chrom from the dropdown below").hide();this.nav_labeltrack=$("
    ").addClass("nav-labeltrack").appendTo(this.bottom_container);this.nav_container=$("
    ").addClass("nav-container").prependTo(this.top_container);this.nav=$("
    ").addClass("nav").appendTo(this.nav_container);this.overview=$("
    ").addClass("overview").appendTo(this.bottom_container);this.overview_viewport=$("
    ").addClass("overview-viewport").appendTo(this.overview);this.overview_close=$("Close Overview").addClass("overview-close").hide().appendTo(this.overview_viewport);this.overview_highlight=$("
    ").addClass("overview-highlight").hide().appendTo(this.overview_viewport);this.overview_box_background=$("
    ").addClass("overview-boxback").appendTo(this.overview_viewport);this.overview_box=$("
    ").addClass("overview-box").appendTo(this.overview_viewport);this.default_overview_height=this.overview_box.height();this.nav_controls=$("
    ").addClass("nav-controls").appendTo(this.nav);this.chrom_select=$("").addClass("nav-input").hide().bind("keyup focusout",ab).appendTo(this.nav_controls);this.location_span=$("").addClass("location").appendTo(this.nav_controls);this.location_span.bind("click",function(){aa.location_span.hide();aa.chrom_select.hide();aa.nav_input.val(aa.chrom+":"+aa.low+"-"+aa.high);aa.nav_input.css("display","inline-block");aa.nav_input.select();aa.nav_input.focus()});if(this.vis_id!==undefined){this.hidden_input=$("").attr("type","hidden").val(this.vis_id).appendTo(this.nav_controls)}this.zo_link=$("").click(function(){aa.zoom_out();aa.redraw()}).appendTo(this.nav_controls);this.zi_link=$("").click(function(){aa.zoom_in();aa.redraw()}).appendTo(this.nav_controls);this.load_chroms({low:0},ad);this.chrom_select.bind("change",function(){aa.change_chrom(aa.chrom_select.val())});this.intro_div.show();this.content_div.bind("click",function(ae){$(this).find("input").trigger("blur")});this.content_div.bind("dblclick",function(ae){aa.zoom_in(ae.pageX,this.viewport_container)});this.overview_box.bind("dragstart",function(ae,af){this.current_x=af.offsetX}).bind("drag",function(ae,ag){var ah=ag.offsetX-this.current_x;this.current_x=ag.offsetX;var af=Math.round(ah/aa.viewport_container.width()*(aa.max_high-aa.max_low));aa.move_delta(-af)});this.overview_close.bind("click",function(){for(var af=0,ae=aa.tracks.length;afaa.viewport_container.width()-16){return false}}).bind("dragstart",function(ae,af){af.original_low=aa.low;af.current_height=ae.clientY;af.current_x=af.offsetX}).bind("drag",function(ag,ai){var ae=$(this);var aj=ai.offsetX-ai.current_x;var af=ae.scrollTop()-(ag.clientY-ai.current_height);ae.scrollTop(af);ai.current_height=ag.clientY;ai.current_x=ai.offsetX;var ah=Math.round(aj/aa.viewport_container.width()*(aa.high-aa.low));aa.move_delta(ah)}).bind("mousewheel",function(ag,ai,af,ae){if(af){var ah=Math.round(-af/aa.viewport_container.width()*(aa.high-aa.low));aa.move_delta(ah)}});this.top_labeltrack.bind("dragstart",function(ae,af){return $("
    ").css({height:aa.content_div.height()+aa.top_labeltrack.height()+aa.nav_labeltrack.height()+1,top:"0px",position:"absolute","background-color":"#ccf",opacity:0.5,"z-index":1000}).appendTo($(this))}).bind("drag",function(ai,aj){$(aj.proxy).css({left:Math.min(ai.pageX,aj.startX),width:Math.abs(ai.pageX-aj.startX)});var af=Math.min(ai.pageX,aj.startX)-aa.container.offset().left,ae=Math.max(ai.pageX,aj.startX)-aa.container.offset().left,ah=(aa.high-aa.low),ag=aa.viewport_container.width();aa.update_location(Math.round(af/ag*ah)+aa.low,Math.round(ae/ag*ah)+aa.low)}).bind("dragend",function(aj,ak){var af=Math.min(aj.pageX,ak.startX),ae=Math.max(aj.pageX,ak.startX),ah=(aa.high-aa.low),ag=aa.viewport_container.width(),ai=aa.low;aa.low=Math.round(af/ag*ah)+ai;aa.high=Math.round(ae/ag*ah)+ai;$(ak.proxy).remove();aa.redraw()});this.add_label_track(new X(this,this.top_labeltrack));this.add_label_track(new X(this,this.nav_labeltrack));$(window).bind("resize",function(){aa.resize_window()});$(document).bind("redraw",function(){aa.redraw()});this.reset();$(window).trigger("resize")},update_location:function(aa,ab){this.location_span.text(commatize(aa)+" - "+commatize(ab));this.nav_input.val(this.chrom+":"+commatize(aa)+"-"+commatize(ab))},load_chroms:function(ab,ac){ab.num=t;$.extend(ab,(this.vis_id!==undefined?{vis_id:this.vis_id}:{dbkey:this.dbkey}));var aa=this;$.ajax({url:chrom_url,data:ab,dataType:"json",success:function(ae){if(ae.chrom_info.length===0){alert("Invalid chromosome: "+ab.chrom);return}if(ae.reference){aa.add_label_track(new x(aa))}aa.chrom_data=ae.chrom_info;var ah='';for(var ag=0,ad=aa.chrom_data.length;ag'+af+""}if(ae.prev_chroms){ah+='"}if(ae.next_chroms){ah+='"}aa.chrom_select.html(ah);if(ac){ac()}aa.chrom_start_index=ae.start_index},error:function(){alert("Could not load chroms for this dbkey:",aa.dbkey)}})},change_chrom:function(ae,ab,ag){if(!ae||ae==="None"){return}var ad=this;if(ae==="previous"){ad.load_chroms({low:this.chrom_start_index-t});return}if(ae==="next"){ad.load_chroms({low:this.chrom_start_index+t});return}var af=$.grep(ad.chrom_data,function(ai,aj){return ai.chrom===ae})[0];if(af===undefined){ad.load_chroms({chrom:ae},function(){ad.change_chrom(ae,ab,ag)});return}else{if(ae!==ad.chrom){ad.chrom=ae;if(!ad.chrom){ad.intro_div.show()}else{ad.intro_div.hide()}ad.chrom_select.val(ad.chrom);ad.max_high=af.len-1;ad.reset();ad.redraw(true);for(var ah=0,aa=ad.tracks.length;ahaa.max_high){aa.high=aa.max_high;aa.low=aa.max_high-ab}else{aa.high-=ac;aa.low-=ac}}aa.redraw()},add_track:function(aa){aa.view=this;aa.track_id=this.track_id_counter;this.tracks.push(aa);if(aa.init){aa.init()}aa.container_div.attr("id","track_"+aa.track_id);C(aa.container_div,".draghandle");this.track_id_counter+=1;this.num_tracks+=1},add_label_track:function(aa){aa.view=this;this.label_tracks.push(aa)},remove_track:function(aa){this.has_changes=true;aa.container_div.fadeOut("slow",function(){$(this).remove()});delete this.tracks[this.tracks.indexOf(aa)];this.num_tracks-=1},reset:function(){this.low=this.max_low;this.high=this.max_high;this.viewport_container.find(".yaxislabel").remove()},redraw:function(ah){var ag=this.high-this.low,af=this.low,ab=this.high;if(afthis.max_high){ab=this.max_high}if(this.high!==0&&ag").addClass("dynamic-tool").hide();this.parent_div.bind("drag",function(aq){aq.stopPropagation()}).bind("click",function(aq){aq.stopPropagation()}).bind("dblclick",function(aq){aq.stopPropagation()});var al=$("
    ").appendTo(this.parent_div).text(this.name);var aj=this.params;var ah=this;$.each(this.params,function(ar,av){var au=$("
    ").addClass("param-row").appendTo(ah.parent_div);var aq=$("
    ").addClass("param-label").text(av.label).appendTo(au);var at=$("
    ").addClass("slider").html(av.html).appendTo(au);at.find(":input").val(av.value);$("
    ").appendTo(au)});this.parent_div.find("input").click(function(){$(this).select()});var ap=$("
    ").addClass("param-row").appendTo(this.parent_div);var af=$("").attr("value","Run on complete dataset").appendTo(ap);var aa=$("").attr("value","Run on visible region").css("margin-left","3em").appendTo(ap);var ah=this;aa.click(function(){ah.run_on_region()});af.click(function(){ah.run_on_dataset()})};n(o.prototype,{get_param_values_dict:function(){var aa={};this.parent_div.find(":input").each(function(){var ab=$(this).attr("name"),ac=$(this).val();aa[ab]=JSON.stringify(ac)});return aa},get_param_values:function(){var ab=[];var aa={};this.parent_div.find(":input").each(function(){var ac=$(this).attr("name"),ad=$(this).val();if(ac){ab[ab.length]=ad}});return ab},run_on_dataset:function(){var aa=this;aa.run({dataset_id:this.track.original_dataset_id,tool_id:aa.name},function(ab){show_modal(aa.name+" is Running",aa.name+" is running on the complete dataset. Tool outputs are in dataset's history.",{Close:hide_modal})})},run_on_region:function(){var aa={dataset_id:this.track.original_dataset_id,chrom:this.track.view.chrom,low:this.track.view.low,high:this.track.view.high,tool_id:this.name},ac=this.track,ab=aa.tool_id+ac.tool_region_and_parameters_str(aa.chrom,aa.low,aa.high),ad;if(ac.track_type==="FeatureTrack"){ad=new Q(ab,view,ac.hda_ldda,undefined,{},{},ac);ad.change_mode(ac.mode)}this.track.add_track(ad);ad.content_div.text("Starting job.");this.run(aa,function(ae){ad.dataset_id=ae.dataset_id;ad.content_div.text("Running job.");ad.init()})},run:function(ab,ac){$.extend(ab,this.get_param_values_dict());var aa=function(){$.getJSON(rerun_tool_url,ab,function(ad){if(ad==="no converter"){new_track.container_div.addClass("error");new_track.content_div.text(G)}else{if(ad.error){new_track.container_div.addClass("error");new_track.content_div.text(v+ad.message)}else{if(ad==="pending"){new_track.container_div.addClass("pending");new_track.content_div.text("Converting input data so that it can be easily reused.");setTimeout(aa,2000)}else{ac(ad)}}}})};aa()}});var K=function(ab,aa,ac,ad){this.name=ab;this.label=aa;this.html=ac;this.value=ad};var g=function(ac,ab,ae,af,ad,aa){K.call(this,ac,ab,ae,af);this.min=ad;this.max=aa};var h=function(ab,aa,ac,ad){this.name=ab;this.index=aa;this.tool_id=ac;this.tool_exp_name=ad};var R=function(ab,aa,ac,ad){h.call(this,ab,aa,ac,ad);this.low=-Number.MAX_VALUE;this.high=Number.MAX_VALUE;this.min=Number.MAX_VALUE;this.max=-Number.MAX_VALUE;this.slider=null;this.slider_label=null};n(R.prototype,{applies_to:function(aa){if(aa.length>this.index){return true}return false},keep:function(aa){if(!this.applies_to(aa)){return true}var ab=parseInt(aa[this.index]);return(isNaN(ab)||(ab>=this.low&&ab<=this.high))},update_attrs:function(ab){var aa=false;if(!this.applies_to(ab)){return aa}if(ab[this.index]this.max){this.max=Math.ceil(ab[this.index]);aa=true}return aa},update_ui_elt:function(){var ac=function(af,ad){var ae=ad-af;return(ae<=2?0.01:1)};var ab=this.slider.slider("option","min"),aa=this.slider.slider("option","max");if(this.minaa){this.slider.slider("option","min",this.min);this.slider.slider("option","max",this.max);this.slider.slider("option","step",ac(this.min,this.max));this.slider.slider("option","values",[this.min,this.max])}}});var W=function(ac,al){this.track=ac;this.filters=[];for(var ag=0;ag").attr("size",input_size).attr("maxlength",input_size).attr("value",ar).appendTo(ap).focus().select().click(function(at){at.stopPropagation()}).blur(function(){$(this).remove();ap.text(ar)}).keyup(function(ax){if(ax.keyCode===27){$(this).trigger("blur")}else{if(ax.keyCode===13){var av=aq.slider("option","min"),at=aq.slider("option","max"),aw=function(ay){return(isNaN(ay)||ay>at||ay").addClass("filters").hide();this.parent_div.bind("drag",function(ao){ao.stopPropagation()}).bind("click",function(ao){ao.stopPropagation()}).bind("dblclick",function(ao){ao.stopPropagation()}).bind("keydown",function(ao){ao.stopPropagation()});var ae=this;$.each(this.filters,function(av,ap){var ar=$("
    ").addClass("slider-row").appendTo(ae.parent_div);var ao=$("
    ").addClass("slider-label").appendTo(ar);var ax=$("").addClass("slider-name").text(ap.name+" ").appendTo(ao);var aq=$("");var at=$("").addClass("slider-value").appendTo(ao).append("[").append(aq).append("]");var aw=$("
    ").addClass("slider").appendTo(ar);ap.control_element=$("
    ").attr("id",ap.name+"-filter-control").appendTo(aw);var au=[0,0];ap.control_element.slider({range:true,min:Number.MAX_VALUE,max:-Number.MIN_VALUE,values:[0,0],slide:function(ay,az){au=az.values;aq.text(az.values[0]+"-"+az.values[1]);setTimeout(function(){if(az.values[0]==au[0]&&az.values[1]==au[1]){var aA=az.values;aq.text(aA[0]+"-"+aA[1]);ap.low=aA[0];ap.high=aA[1];ae.track.draw(true,true)}},50)},change:function(ay,az){ap.control_element.slider("option","slide").call(ap.control_element,ay,az)}});ap.slider=ap.control_element;ap.slider_label=aq;aj(at,aq,ap.control_element);$("
    ").appendTo(ar)});if(this.filters.length!=0){var am=$("
    ").addClass("param-row").appendTo(this.parent_div);var ah=$("").attr("value","Run on complete dataset").appendTo(am);var af=this;ah.click(function(){af.run_on_dataset()})}};n(W.prototype,{reset_filters:function(){for(var aa=0;aa= "+aa.low}if(aa.max!=aa.high){ab=ai(ac,aa.tool_id,[]);ab[ab.length]=aa.tool_exp_name+" <= "+aa.high}}}var ag=[];for(var aj in ac){ag[ag.length]=[aj,ac[aj]]}var ah=ag.length;(function af(aq,an){var al=an[0],am=al[0],ap=al[1],ao="("+ap.join(") and (")+")",ak={cond:ao,input:aq,target_dataset_id:aq,tool_id:am},an=an.slice(1);$.getJSON(run_tool_url,ak,function(ar){if(ar.error){show_modal("Filter Dataset","Error running tool "+am,{Close:hide_modal})}else{if(an.length===0){show_modal("Filtering Dataset","Filter(s) are running on the complete dataset. Outputs are in dataset's history.",{Close:hide_modal})}else{af(ar.dataset_id,an)}}})})(this.track.dataset_id,ag)}});var V=function(aa){this.track=aa.track;this.params=aa.params;this.values={};if(aa.saved_values){this.restore_values(aa.saved_values)}this.onchange=aa.onchange};n(V.prototype,{restore_values:function(aa){var ab=this;$.each(this.params,function(ac,ad){if(aa[ad.key]!==undefined){ab.values[ad.key]=aa[ad.key]}else{ab.values[ad.key]=ad.default_value}})},build_form:function(){var ab=this;var aa=$("
    ");$.each(this.params,function(af,ad){if(!ad.hidden){var ac="param_"+af;var ak=$("
    ").appendTo(aa);ak.append($("
    ").hide();var ae=$("
    ").appendTo(ai);var aj=$("
    ").appendTo(ae).farbtastic({width:100,height:100,callback:ag,color:ah});$("
    ").append(ag).append(ai).appendTo(ak).bind("click",function(al){ai.css({left:$(this).position().left+($(ag).width()/2)-60,top:$(this).position().top+$(this.height)}).show();$(document).bind("click.color-picker",function(){ai.hide();$(document).unbind("click.color-picker")});al.stopPropagation()})}else{ak.append($("").attr("id",ac).attr("name",ac).val(ab.values[ad.key]))}}}});return aa},update_from_form:function(aa){var ac=this;var ab=false;$.each(this.params,function(ad,af){if(!af.hidden){var ag="param_"+ad;var ae=aa.find("#"+ag).val();if(af.type==="float"){ae=parseFloat(ae)}else{if(af.type==="int"){ae=parseInt(ae)}else{if(af.type==="bool"){ae=aa.find("#"+ag).is(":checked")}}}if(ae!==ac.values[af.key]){ac.values[af.key]=ae;ab=true}}});if(ab){this.onchange()}}});var b=function(ac,ab,aa){this.index=ac;this.resolution=ab;this.canvas=$("
    ").append(aa)};var l=function(ac,ab,aa,ad){b.call(this,ac,ab,aa);this.max_val=ad};var L=function(ac,ab,aa){b.call(this,ac,ab,aa)};var j=function(ab,aa,ae,ac,ad){this.name=ab;this.view=aa;this.parent_element=ae;this.data_url=(ac?ac:default_data_url);this.data_url_extra_params={};this.data_query_wait=(ad?ad:H);this.dataset_check_url=converted_datasets_state_url;this.container_div=$("
    ").addClass("track").css("position","relative");if(!this.hidden){this.header_div=$("
    ").appendTo(this.container_div);if(this.view.editor){this.drag_div=$("
    ").appendTo(this.header_div)}this.name_div=$("
    ").appendTo(this.header_div);this.name_div.text(this.name);this.name_div.attr("id",this.name.replace(/\s+/g,"-").replace(/[^a-zA-Z0-9\-]/g,"").toLowerCase())}this.content_div=$("
    ").appendTo(this.container_div);this.parent_element.append(this.container_div)};n(j.prototype,{init:function(){var aa=this;aa.enabled=false;aa.tile_cache.clear();aa.data_cache.clear();aa.initial_canvas=undefined;aa.content_div.css("height","auto");aa.container_div.removeClass("nodata error pending");if(!aa.dataset_id){return}$.getJSON(converted_datasets_state_url,{hda_ldda:aa.hda_ldda,dataset_id:aa.dataset_id,chrom:aa.view.chrom},function(ab){if(!ab||ab==="error"||ab.kind==="error"){aa.container_div.addClass("error");aa.content_div.text(m);if(ab.message){var ad=aa.view.tracks.indexOf(aa);var ac=$(" ").text("View error").bind("click",function(){show_modal("Trackster Error","
    "+ab.message+"
    ",{Close:hide_modal})});aa.content_div.append(ac)}}else{if(ab==="no converter"){aa.container_div.addClass("error");aa.content_div.text(G)}else{if(ab==="no data"||(ab.data!==undefined&&(ab.data===null||ab.data.length===0))){aa.container_div.addClass("nodata");aa.content_div.text(B)}else{if(ab==="pending"){aa.container_div.addClass("pending");aa.content_div.text(q);setTimeout(function(){aa.init()},aa.data_query_wait)}else{if(ab.status==="data"){if(ab.valid_chroms){aa.valid_chroms=ab.valid_chroms;aa.make_name_popup_menu()}aa.content_div.text(U);if(aa.view.chrom){aa.content_div.text("");aa.content_div.css("height",aa.height_px+"px");aa.enabled=true;$.when(aa.predraw_init()).done(function(){aa.container_div.removeClass("nodata error pending");aa.draw()})}}}}}}})},predraw_init:function(){},update_name:function(aa){this.old_name=this.name;this.name=aa;this.name_div.text(this.name)},revert_name:function(){this.name=this.old_name;this.name_div.text(this.name)}});var J=function(ah,af,ai){var ab=this,aj=ab.view;this.filters_manager=(ah!==undefined?new W(this,ah):undefined);this.filters_available=false;this.filters_visible=false;this.tool=(af!==undefined&&obj_length(af)>0?new o(this,af):undefined);this.parent_track=ai;this.child_tracks=[];if(ab.hidden){return}if(this.parent_track){this.header_div.find(".draghandle").removeClass("draghandle").addClass("child-track-icon").addClass("icon-button");this.parent_element.addClass("child-track");this.tool=undefined}ab.child_tracks_container=$("
    ").addClass("child-tracks-container").hide();ab.container_div.append(ab.child_tracks_container);if(this.filters_manager){this.filters_div=this.filters_manager.parent_div;this.header_div.after(this.filters_div)}if(this.tool){this.dynamic_tool_div=this.tool.parent_div;this.header_div.after(this.dynamic_tool_div)}if(ab.display_modes!==undefined){if(ab.mode_div===undefined){ab.mode_div=$("
    ").appendTo(ab.header_div);var ac=(ab.track_config&&ab.track_config.values.mode?ab.track_config.values.mode:ab.display_modes[0]);ab.mode=ac;ab.mode_div.text(ac);var aa={};for(var ad=0,ag=ab.display_modes.length;ad0){var ae=(ab.filters_div.is(":visible")?"Hide filters":"Show filters");aa[ae]=function(){ab.filters_visible=(ab.filters_div.is(":visible"));if(ab.filters_visible){ab.filters_manager.reset_filters()}ab.filters_div.toggle();ab.make_name_popup_menu()}}if(ab.tool){var ae=(ab.dynamic_tool_div.is(":visible")?"Hide tool":"Show tool");aa[ae]=function(){if(!ab.dynamic_tool_div.is(":visible")){ab.update_name(ab.name+ab.tool_region_and_parameters_str())}else{menu_option_text="Show dynamic tool";ab.revert_name()}ab.dynamic_tool_div.toggle();ab.make_name_popup_menu()}}if(ab.valid_chroms){aa["List chrom/contigs with data"]=function(){show_modal("Chrom/contigs with data","

    "+ab.valid_chroms.join("
    ")+"

    ",{Close:function(){hide_modal()}})}}var ac=view;var ad=function(){$("#no-tracks").show()};if(this.parent_track){ac=this.parent_track;ad=function(){}}aa.Remove=function(){ac.remove_track(ab);if(ac.num_tracks===0){ad()}};make_popupmenu(ab.name_div,aa)},draw:function(aa,ac){var au=this.view.low,ag=this.view.high,ai=ag-au,ak=this.view.container.width(),ae=ak/ai,al=this.view.resolution,ad=$("
    "),am=function(aw,ax,av){return aw+"_"+ax+"_"+av};if(!ac){this.content_div.children().remove()}this.content_div.append(ad);this.max_height=0;var ao=Math.floor(au/al/M);var af=[];var ap=0;while((ao*M*al)=aD;ax--){var aw=$(ay[ax]);if(az){aw.remove()}else{if(aw.children().length!==0){az=true}}}}if(aj.track_type=="FeatureTrack"&&aj.mode=="Histogram"){var aC=-1;for(var ax=0;axaC){aC=aF}}for(var ax=0;ax=ad.view.low){var an=(ab?undefined:ad.tile_cache.get(ai));if(an){ad.show_tile(an,aj,ag,am);ak[ak.length]=an}else{$.when(ad.data_cache.get_data(view.chrom,ag,al,ad.mode,ae,ad.data_url_extra_params)).then(function(ao){n(ao,af);if(view.reference_track&&am>view.canvas_manager.char_width_px){$.when(view.reference_track.data_cache.get_data(view.chrom,ag,al,ad.mode,ae,view.reference_track.data_url_extra_params)).then(function(ap){ah(aa,ao,ae,ac,aj,am,ap)})}else{ah(aa,ao,ae,ac,aj,am)}})}}},50)},show_tile:function(ad,ag,ae,ah){var ab=this;var ac=this.view.high-this.view.low,af=(ae-this.view.low)*ah;if(this.left_offset){af-=this.left_offset}var aa=ad.canvas;aa.css({position:"absolute",top:0,left:af,height:""});ag.append(aa);ab.max_height=Math.max(ab.max_height,aa.height());ab.content_div.css("height",ab.max_height+"px");ag.children().css("height",ab.max_height+"px")},set_overview:function(){var aa=this.view;if(this.initial_canvas&&this.is_overview){aa.overview_close.show();aa.overview_viewport.append(this.initial_canvas);aa.overview_highlight.show().height(this.initial_canvas.height());aa.overview_viewport.height(this.initial_canvas.height()+aa.overview_box.height())}$(window).trigger("resize")},tool_region_and_parameters_str:function(ac,aa,ad){var ab=this,ae=(ac!==undefined&&aa!==undefined&&ad!==undefined?ac+":"+aa+"-"+ad:"all");return" - region=["+ae+"], parameters=["+ab.tool.get_param_values().join(", ")+"]"},add_track:function(aa){aa.track_id=this.track_id+"_"+this.child_tracks.length;aa.container_div.attr("id","track_"+aa.track_id);this.child_tracks_container.append(aa.container_div);C(aa.container_div,".child-track-icon");if(!$(this.child_tracks_container).is(":visible")){this.child_tracks_container.show()}this.child_tracks.push(aa);this.view.has_changes=true},remove_track:function(aa){aa.container_div.fadeOut("slow",function(){$(this).remove()})}});var X=function(aa,ab){this.track_type="LabelTrack";this.hidden=true;j.call(this,null,aa,ab);this.container_div.addClass("label-track")};n(X.prototype,j.prototype,{draw:function(){var ac=this.view,ad=ac.high-ac.low,ag=Math.floor(Math.pow(10,Math.floor(Math.log(ad)/Math.log(10)))),aa=Math.floor(ac.low/ag)*ag,ae=this.view.container.width(),ab=$("
    ");while(aa"+commatize(aa)+"
    ").css({position:"absolute",left:af-1}));aa+=ag}this.content_div.children(":first").remove();this.content_div.append(ab)}});var x=function(aa){this.track_type="ReferenceTrack";this.hidden=true;j.call(this,null,aa,aa.top_labeltrack);J.call(this);aa.reference_track=this;this.left_offset=200;this.height_px=12;this.container_div.addClass("reference-track");this.content_div.css("background","none");this.content_div.css("min-height","0px");this.content_div.css("border","none");this.data_url=reference_url;this.data_url_extra_params={dbkey:aa.dbkey};this.data_cache=new E(y,this,false);this.tile_cache=new c(r)};n(x.prototype,J.prototype,{draw_tile:function(ai,af,ab,ak){var ae=this,ac=M*af;if(ak>this.view.canvas_manager.char_width_px){if(ai===null){ae.content_div.css("height","0px");return}var ad=this.view.canvas_manager.new_canvas();var aj=ad.getContext("2d");ad.width=Math.ceil(ac*ak+ae.left_offset);ad.height=ae.height_px;aj.font=aj.canvas.manager.default_font;aj.textAlign="center";for(var ag=0,ah=ai.length;ag");$(aa.container_div).hover(function(){ad=true;ab.show()},function(){ad=false;if(!ac){ab.hide()}});ab.hide().bind("dragstart",function(ae,af){ac=true;af.original_height=$(aa.content_div).height()}).bind("drag",function(af,ag){var ae=Math.min(Math.max(ag.original_height+ag.deltaY,aa.min_height_px),aa.max_height_px);$(aa.content_div).css("height",ae);aa.height_px=ae;aa.draw(true)}).bind("dragend",function(ae,af){aa.tile_cache.clear();ac=false;if(!ad){ab.hide()}aa.track_config.values.height=aa.height_px}).appendTo(aa.container_div)},predraw_init:function(){var aa=this,ab=aa.view.tracks.indexOf(aa);aa.vertical_range=undefined;return $.getJSON(aa.data_url,{stats:true,chrom:aa.view.chrom,low:null,high:null,hda_ldda:aa.hda_ldda,dataset_id:aa.dataset_id},function(ac){aa.container_div.addClass("line-track");var ae=ac.data;if(isNaN(parseFloat(aa.prefs.min_value))||isNaN(parseFloat(aa.prefs.max_value))){aa.prefs.min_value=ae.min;aa.prefs.max_value=ae.max;$("#track_"+ab+"_minval").val(aa.prefs.min_value);$("#track_"+ab+"_maxval").val(aa.prefs.max_value)}aa.vertical_range=aa.prefs.max_value-aa.prefs.min_value;aa.total_frequency=ae.total_frequency;aa.container_div.find(".yaxislabel").remove();var af=$("
    ").addClass("yaxislabel").attr("id","linetrack_"+ab+"_minval").text(u(aa.prefs.min_value));var ad=$("
    ").addClass("yaxislabel").attr("id","linetrack_"+ab+"_maxval").text(u(aa.prefs.max_value));ad.css({position:"absolute",top:"24px",left:"10px"});ad.prependTo(aa.container_div);af.css({position:"absolute",bottom:"2px",left:"10px"});af.prependTo(aa.container_div)})},draw_tile:function(ak,ae,ab,aj){if(this.vertical_range===undefined){return}var af=ab*M*ae,ad=M*ae,aa=Math.ceil(ad*aj),ah=this.height_px;var ac=this.view.canvas_manager.new_canvas();ac.width=aa,ac.height=ah;var ai=ac.getContext("2d");var ag=new I.LinePainter(ak.data,af,af+ad,this.prefs,this.mode);ag.draw(ai,aa,ah);return new b(ad,ae,ac)}});var e=function(aa,af,ae,ai,ah,ac,ad,ag){var ab=this;this.track_type="FeatureTrack";this.display_modes=["Auto","Histogram","Dense","Squish","Pack"];this.track_config=new V({track:this,params:[{key:"block_color",label:"Block color",type:"color",default_value:"#444"},{key:"label_color",label:"Label color",type:"color",default_value:"black"},{key:"show_counts",label:"Show summary counts",type:"bool",default_value:true},{key:"mode",type:"string",default_value:this.mode,hidden:true},],saved_values:ah,onchange:function(){ab.tile_cache.clear();ab.draw()}});this.prefs=this.track_config.values;j.call(this,aa,af,af.viewport_container);J.call(this,ac,ad,ag);this.height_px=0;this.container_div.addClass("feature-track");this.hda_ldda=ae;this.dataset_id=ai;this.original_dataset_id=ai;this.show_labels_scale=0.001;this.showing_details=false;this.summary_draw_height=30;this.inc_slots={};this.start_end_dct={};this.tile_cache=new c(d);this.data_cache=new N(20,this);this.left_offset=200;this.painter=I.LinkedFeaturePainter};n(e.prototype,J.prototype,{update_auto_mode:function(aa){if(this.mode=="Auto"){if(aa=="no_detail"){aa="feature spans"}else{if(aa=="summary_tree"){aa="coverage histogram"}}this.mode_div.text("Auto ("+aa+")")}},incremental_slots:function(ae,ab,ad){var ac=this.view.canvas_manager.dummy_context,aa=this.inc_slots[ae];if(!aa||(aa.mode!==ad)){aa=new (p.FeatureSlotter)(ae,ad==="Pack",w,function(af){return ac.measureText(af)});aa.mode=ad;this.inc_slots[ae]=aa}return aa.slot_features(ab)},get_summary_tree_data:function(ae,ah,ac,ap){if(ap>ac-ah){ap=ac-ah}var al=Math.floor((ac-ah)/ap),ao=[],ad=0;var af=0,ag=0,ak,an=0,ai=[],am,aj;var ab=function(at,ar,au,aq){at[0]=ar+au*aq;at[1]=ar+(au+1)*aq};while(anad){ad=am[1]}an++}return{max:ad,delta:al,data:ao}},draw_tile:function(an,aw,aA,aj,ad){var at=this,aC=aA*M*aw,ab=(aA+1)*M*aw,ap=ab-aC,au=Math.ceil(ap*aj),ar=this.mode,aG=25,ae=this.left_offset,ao,af;if(ar==="Auto"){if(an.dataset_type==="summary_tree"){ar=an.dataset_type}else{if(an.extra_info==="no_detail"){ar="no_detail"}else{var aF=an.data;if(this.view.high-this.view.low>F){ar="Squish"}else{ar="Pack"}}}this.update_auto_mode(ar)}if(ar==="summary_tree"||ar==="Histogram"){af=this.summary_draw_height;this.container_div.find(".yaxislabel").remove();var aa=$("
    ").addClass("yaxislabel");aa.text(an.max);aa.css({position:"absolute",top:"24px",left:"10px",color:this.prefs.label_color});aa.prependTo(this.container_div);var ac=this.view.canvas_manager.new_canvas();ac.width=au+ae;ac.height=af+O;if(an.dataset_type!="summary_tree"){var ak=this.get_summary_tree_data(an.data,aC,ab,200);if(an.max){ak.max=an.max}an=ak}var aD=new I.SummaryTreePainter(an,aC,ab,this.prefs);var av=ac.getContext("2d");av.translate(ae,O);aD.draw(av,au,af);return new l(aA,aw,ac,an.max)}var ao,ah=1;if(ar==="no_detail"||ar==="Squish"||ar==="Pack"){ah=this.incremental_slots(aj,an.data,ar);ao=this.inc_slots[aj].slots}var ai=[];if(an.data){var al=this.filters_manager.filters;for(var ax=0,az=an.data.length;axB[0]&&G=0){g-=v;j="left"}else{k+=v;j="right"}}var t=q(g,k);if(t>=0){if(h[t]===undefined){h[t]=[]}h[t].push([g,k]);s[o]=t;n=Math.max(n,t)}else{}}return n+1}})};var painters_module=function(j,w){var t=j("class").extend;var o=function(H,z,F,y,E,C){if(C===undefined){C=4}var B=y-z;var A=E-F;var D=Math.floor(Math.sqrt(B*B+A*A)/C);var I=B/D;var G=A/D;var x;for(x=0;x1){B=Math.ceil((S[1][0]-S[0][0])*A)}else{B=10}for(var N=0,O=S.length;NC){P=true;D=C}}if(H==="Histogram"){D=Math.round(D/I*x);M.fillRect(Q,T,B,-D)}else{if(H==="Intensity"){D=255-Math.floor((D-G)/I*255);M.fillStyle="rgb("+D+","+D+","+D+")";M.fillRect(Q,0,B,x)}else{D=Math.round(x-(D-G)/I*x);if(E){M.lineTo(Q,D)}else{E=true;if(H==="Filled"){M.moveTo(Q,x);M.lineTo(Q,D)}else{M.moveTo(Q,D)}}}}M.fillStyle=this.prefs.overflow_color;if(P||F){var R;if(H==="Histogram"||H==="Intensity"){R=B}else{Q-=2;R=4}if(P){M.fillRect(Q,0,R,3)}if(F){M.fillRect(Q,x-3,R,3)}}M.fillStyle=this.prefs.color}if(H==="Filled"){if(E){M.lineTo(Q,T);M.lineTo(0,T)}M.fill()}else{M.stroke()}M.restore()};var n=function(z,B,x,y,A){m.call(this,z,B,x,y,A)};n.prototype.default_prefs={block_color:"#FFF",connector_color:"#FFF"};t(n.prototype,{get_required_height:function(y){var x=y_scale=this.get_row_height(),z=this.mode;if(z==="no_detail"||z==="Squish"||z==="Pack"){x=y*y_scale}return x+Math.max(Math.round(y_scale/2),5)},draw:function(J,A,I,F){var D=this.data,G=this.view_start,K=this.view_end;J.save();J.fillStyle=this.prefs.block_color;J.textAlign="right";var N=this.view_end-this.view_start,M=A/N,z=this.get_row_height();for(var C=0,E=D.length;CG)&&(this.mode=="Dense"||H!==null)){this.draw_element(J,this.mode,L,H,G,K,M,z,A)}}J.restore()}});var d=10,h=3,l=5,v=10,f=1,r=3,e=3,a=9,k=2,g="#ccc";var q=function(z,B,x,y,A){n.call(this,z,B,x,y,A)};t(q.prototype,n.prototype,{get_row_height:function(){var y=this.mode,x;if(y==="Dense"){x=d}else{if(y==="no_detail"){x=h}else{if(y==="Squish"){x=l}else{x=v}}}return x},draw_element:function(J,C,R,E,L,ab,af,ag,x){var O=R[0],ad=R[1],V=R[2],M=R[3],W=Math.floor(Math.max(0,(ad-L)*af)),K=Math.ceil(Math.min(x,Math.max(0,(V-L)*af))),U=(C==="Dense"?0:(0+E))*ag,I,Z,N=null,ah=null,A=this.prefs.block_color,Y=this.prefs.label_color;if(C=="Dense"){E=1}if(C==="no_detail"){J.fillStyle=A;J.fillRect(W,U+5,K-W,f)}else{var H=R[4],T=R[5],X=R[6],B=R[7];if(T&&X){N=Math.floor(Math.max(0,(T-L)*af));ah=Math.ceil(Math.min(x,Math.max(0,(X-L)*af)))}var ae,P;if(C==="Squish"||C==="Dense"){ae=1;P=e}else{ae=5;P=a}if(!B){if(R.strand){if(R.strand==="+"){J.fillStyle=J.canvas.manager.get_pattern("right_strand_inv")}else{if(R.strand==="-"){J.fillStyle=J.canvas.manager.get_pattern("left_strand_inv")}}}else{J.fillStyle=A}J.fillRect(W,U,K-W,P)}else{var G,Q;if(C==="Squish"||C==="Dense"){J.fillStyle=g;G=U+Math.floor(e/2)+1;Q=1}else{if(H){var G=U;var Q=P;if(H==="+"){J.fillStyle=J.canvas.manager.get_pattern("right_strand")}else{if(H==="-"){J.fillStyle=J.canvas.manager.get_pattern("left_strand")}}}else{J.fillStyle=g;G+=(e/2)+1;Q=1}}J.fillRect(W,G,K-W,Q);for(var ac=0,z=B.length;acS){continue}J.fillStyle=A;J.fillRect(y,U+(P-ae)/2+1,S-y,ae);if(N!==undefined&&X>T&&!(y>ah||SL){J.fillStyle=Y;if(L===0&&W-J.measureText(M).width<0){J.textAlign="left";J.fillText(M,K+k,U+8)}else{J.textAlign="right";J.fillText(M,W-k,U+8)}J.fillStyle=A}}}});var b=function(z,B,x,y,A){n.call(this,z,B,x,y,A)};t(b.prototype,n.prototype,{draw_element:function(Q,L,F,B,T,z,I,R,O){var F=data[i],H=F[0],P=F[1],A=F[2],K=F[3],D=Math.floor(Math.max(0,(P-T)*I)),G=Math.ceil(Math.min(O,Math.max(0,(A-T)*I))),C=(L==="Dense"?0:(0+B))*R,x,U,y=null,J=null;if(no_label){Q.fillStyle=block_color;Q.fillRect(D+left_offset,C+5,G-D,1)}else{var S=F[4],N=F[5],E=F[6];x=9;U=1;Q.fillRect(D+left_offset,C,G-D,x);if(L!=="Dense"&&K!==undefined&&P>T){Q.fillStyle=label_color;if(T===0&&D-Q.measureText(K).width<0){Q.textAlign="left";Q.fillText(K,G+2+left_offset,C+8)}else{Q.textAlign="right";Q.fillText(K,D-2+left_offset,C+8)}Q.fillStyle=block_color}var M=S+" / "+N;if(P>T&&Q.measureText(M).width<(G-D)){Q.fillStyle="white";Q.textAlign="center";Q.fillText(M,left_offset+D+(G-D)/2,C+8);Q.fillStyle=block_color}}}});var s=function(A,C,x,z,B,y){n.call(this,A,C,x,z,B);this.ref_seq=y};s.prototype.default_prefs=t({},n.prototype.default_prefs,{show_insertions:false});t(s.prototype,n.prototype,{get_row_height:function(){var x,y=this.mode;if(y==="Dense"){x=d}else{if(y==="Squish"){x=l}else{x=v;if(this.prefs.show_insertions){x*=2}}}return x},draw_read:function(T,O,K,Y,z,S,H,E,D){T.textAlign="center";var R=this,y=[Y,z],N=0,U=0,Q=0;ref_seq=this.ref_seq,char_width_px=T.canvas.manager.char_width_px;var ad=[];if((O==="Pack"||this.mode==="Auto")&&E!==undefined&&K>char_width_px){Q=Math.round(K/2)}if(!H){H=[[0,E.length]]}for(var L=0,W=H.length;L0){T.fillStyle=this.prefs.block_color;T.fillRect(ac-Q,D+1,G-ac,9);T.fillStyle=g;for(var aa=0,x=P.length;aa=Y&&F+aa<=z){var ab=Math.floor(Math.max(0,(F+aa-Y)*K));T.fillText(P[aa],ab,D+9)}}}else{T.fillStyle=this.prefs.block_color;T.fillRect(ac,D+4,G-ac,e)}}U+=M;N+=M;break;case"N":T.fillStyle=g;T.fillRect(ac-Q,D+5,G-ac,1);N+=M;break;case"D":T.fillStyle="red";T.fillRect(ac-Q,D+4,G-ac,3);N+=M;break;case"P":break;case"I":var X=ac-Q;if(is_overlap([F,F+M],y)){var P=E.slice(U,U+M);if(this.prefs.show_insertions){var C=ac-(G-ac)/2;if((O==="Pack"||this.mode==="Auto")&&E!==undefined&&K>char_width_px){T.fillStyle="yellow";T.fillRect(C-Q,D-9,G-ac,9);ad[ad.length]={type:"triangle",data:[X,D+4,5]};T.fillStyle=g;switch(seq_tile_overlap){case (OVERLAP_START):P=P.slice(Y-F);break;case (OVERLAP_END):P=P.slice(0,F-z);break;case (CONTAINED_BY):break;case (CONTAINS):P=P.slice(Y-F,F-z);break}for(var aa=0,x=P.length;aachar_width_px){ad[ad.length]={type:"text",data:[P.length,X,D+9]}}else{}}}U+=M;break;case"X":U+=M;break}}T.fillStyle="yellow";var Z,B,ae;for(var V=0;VQ.canvas.manager.char_width_px){var N=Math.round(H/2)}Q.fillStyle=U;if(D[5] instanceof Array){var M=Math.floor(Math.max(0,(D[4][0]-T)*H)),K=Math.ceil(Math.min(O,Math.max(0,(D[4][1]-T)*H))),J=Math.floor(Math.max(0,(D[5][0]-T)*H)),x=Math.ceil(Math.min(O,Math.max(0,(D[5][1]-T)*H)));if(D[4][1]>=T&&D[4][0]<=y&&D[4][2]){this.draw_read(Q,L,H,T,y,D[4][0],D[4][2],D[4][3],B)}if(D[5][1]>=T&&D[5][0]<=y&&D[5][2]){this.draw_read(Q,L,H,T,y,D[5][0],D[5][2],D[5][3],B)}if(J>K){Q.fillStyle=g;o(Q,K-N,B+5,J-N,B+5)}}else{Q.fillStyle=U;this.draw_read(Q,L,H,T,y,P,D[4],D[5],B)}if(L==="Pack"&&P>T){Q.fillStyle=this.prefs.label_color;var S=1;if(S===0&&C-Q.measureText(I).width<0){Q.textAlign="left";Q.fillText(I,E+k-N,B+8)}else{Q.textAlign="right";Q.fillText(I,C-k-N,B+8)}Q.fillStyle=U}}});w.SummaryTreePainter=u;w.LinePainter=c;w.LinkedFeaturePainter=q;w.ReadPainter=s;w.VariantPainter=b};(function(d){var c={};var b=function(e){return c[e]};var a=function(f,g){var e={};g(b,e);c[f]=e};a("class",class_module);a("slotting",slotting_module);a("painters",painters_module);a("trackster",trackster_module);for(key in c.trackster){d[key]=c.trackster[key]}})(window); \ No newline at end of file +var class_module=function(b,a){var c=function(){var f=arguments[0];for(var e=1;ec){a=AFTER}else{if(f<=c){a=CONTAINED_BY}else{a=OVERLAP_END}}}return a};var is_overlap=function(c,b){var a=compute_overlap(c,b);return(a!==BEFORE&&a!==AFTER)};var trackster_module=function(f,aa){var q=f("class").extend,t=f("slotting"),N=f("painters");var ag=function(ah,ai){this.document=ah;this.default_font=ai!==undefined?ai:"9px Monaco, Lucida Console, monospace";this.dummy_canvas=this.new_canvas();this.dummy_context=this.dummy_canvas.getContext("2d");this.dummy_context.font=this.default_font;this.char_width_px=this.dummy_context.measureText("A").width;this.patterns={};this.load_pattern("right_strand","/visualization/strand_right.png");this.load_pattern("left_strand","/visualization/strand_left.png");this.load_pattern("right_strand_inv","/visualization/strand_right_inv.png");this.load_pattern("left_strand_inv","/visualization/strand_left_inv.png")};q(ag.prototype,{load_pattern:function(ah,al){var ai=this.patterns,aj=this.dummy_context,ak=new Image();ak.src=image_path+al;ak.onload=function(){ai[ah]=aj.createPattern(ak,"repeat")}},get_pattern:function(ah){return this.patterns[ah]},new_canvas:function(){var ah=this.document.createElement("canvas");if(window.G_vmlCanvasManager){G_vmlCanvasManager.initElement(ah)}ah.manager=this;return ah}});var o={};var m=function(ah,ai){o[ah.attr("id")]=ai};var n=function(ah,aj,al,ak){al=".group";var ai={};o[ah.attr("id")]=ak;ah.bind("drag",{handle:"."+aj,relative:true},function(au,av){var at=$(this);var ay=$(this).parent(),ap=ay.children(),ar=o[$(this).attr("id")],ao,an,aw,am,aq;an=$(this).parents(al);if(an.length!==0){aw=an.position().top;am=aw+an.outerHeight();if(av.offsetYam){$(this).insertAfter(an);var ax=o[an.attr("id")];ax.remove_drawable(ar);ax.container.add_drawable(ar);return}}}an=null;for(aq=0;aq=aw&&av.offsetY<=am){if(av.offsetY-aw0?aq-1:aq))}}}).bind("dragstart",function(){ai["border-top"]=ah.css("border-top");ai["border-bottom"]=ah.css("border-bottom");$(this).css({"border-top":"1px solid blue","border-bottom":"1px solid blue"})}).bind("dragend",function(){$(this).css(ai)})};aa.moveable=n;var af=16,I=9,F=20,V=I+2,B=100,K=12000,T=200,E=5,w=10,M=5000,x=100,p="There was an error in indexing this dataset. ",L="A converter for this dataset is not installed. Please check your datatypes_conf.xml file.",G="No data for this chrom/contig.",u="Currently indexing... please wait",z="Tool cannot be rerun: ",a="Loading data...",ab="Ready for display",d=10,v=5,D=5;function y(ah){return Math.round(ah*1000)/1000}var c=function(ah){this.num_elements=ah;this.clear()};q(c.prototype,{get:function(ai){var ah=this.key_ary.indexOf(ai);if(ah!==-1){if(this.obj_cache[ai].stale){this.key_ary.splice(ah,1);delete this.obj_cache[ai]}else{this.move_key_to_end(ai,ah)}}return this.obj_cache[ai]},set:function(ai,aj){if(!this.obj_cache[ai]){if(this.key_ary.length>=this.num_elements){var ah=this.key_ary.shift();delete this.obj_cache[ah]}this.key_ary.push(ai)}this.obj_cache[ai]=aj;return aj},move_key_to_end:function(ai,ah){this.key_ary.splice(ah,1);this.key_ary.push(ai)},clear:function(){this.obj_cache={};this.key_ary=[]},size:function(){return this.key_ary.length}});var U=function(ai,ah,aj){c.call(this,ai);this.track=ah;this.subset=(aj!==undefined?aj:true)};q(U.prototype,c.prototype,{load_data:function(aq,al,ao,ai,an){var ap=this.track.view.chrom,ak={chrom:ap,low:aq,high:al,mode:ao,resolution:ai,dataset_id:this.track.dataset_id,hda_ldda:this.track.hda_ldda};$.extend(ak,an);if(this.track.filters_manager){var ar=[];var ah=this.track.filters_manager.filters;for(var am=0;am1){return}return U.prototype.load_data.call(this,aj,ah,al,am,ai,ak)}});var r=function(ak,ai,ah,aj,al){this.name=ak;this.view=ai;this.container=ah;this.drag_handle_class=al;this.config=new H({track:this,params:[{key:"name",label:"Name",type:"text",default_value:ak}],saved_values:aj,onchange:function(){this.track.set_name(this.track.config.values.name)}});this.prefs=this.config.values};q(r.prototype,{init:function(){},request_draw:function(){},_draw:function(){},to_json:function(){},make_name_popup_menu:function(){},set_name:function(ah){this.old_name=this.name;this.name=ah;this.name_div.text(this.name)},revert_name:function(){this.name=this.old_name;this.name_div.text(this.name)},remove:function(){this.container.remove_drawable(this);this.container_div.fadeOut("slow",function(){$(this).remove();view.update_intro_div();view.has_changes=true})}});var A=function(al,ak,ai,ah,aj,am){r.call(this,ak,ai,ah,aj,am);this.obj_type=al;this.drawables=[]};q(A.prototype,r.prototype,{init:function(){for(var ah=0;ah").addClass("group").attr("id","group_"+al).appendTo(this.container.content_div);this.header_div=$("
    ").addClass("track-header").appendTo(this.container_div);this.header_div.append($("
    ").addClass(this.drag_handle_class));this.name_div=$("
    ").addClass("group-name menubutton popup").text(this.name).appendTo(this.header_div);this.content_div=$("
    ").addClass("content-div").attr("id","group_"+al+"_content_div").appendTo(this.container_div);m(this.container_div,this);m(this.content_div,this);n(this.container_div,this.drag_handle_class,".group",this);this.make_name_popup_menu()};q(S.prototype,r.prototype,A.prototype,{make_name_popup_menu:function(){var ai=this;var ah={};ah["Edit configuration"]=function(){var al=function(){hide_modal();$(window).unbind("keypress.check_enter_esc")},aj=function(){ai.config.update_from_form($(".dialog-box"));hide_modal();$(window).unbind("keypress.check_enter_esc")},ak=function(am){if((am.keyCode||am.which)===27){al()}else{if((am.keyCode||am.which)===13){aj()}}};$(window).bind("keypress.check_enter_esc",ak);show_modal("Configure Group",ai.config.build_form(),{Cancel:al,OK:aj})};ah.Remove=function(){ai.remove()};make_popupmenu(ai.name_div,ah)}});var ae=function(ah,ak,aj,ai){A.call(this,"View");this.container=ah;this.chrom=null;this.vis_id=aj;this.dbkey=ai;this.title=ak;this.tracks=this.drawables;this.label_tracks=[];this.tracks_to_be_redrawn=[];this.max_low=0;this.max_high=0;this.zoom_factor=3;this.min_separation=30;this.has_changes=false;this.load_chroms_deferred=null;this.init();this.canvas_manager=new ag(ah.get(0).ownerDocument);this.reset()};q(ae.prototype,A.prototype,{init:function(){var aj=this.container,ah=this;this.top_container=$("
    ").addClass("top-container").appendTo(aj);this.browser_content_div=$("
    ").addClass("content").css("position","relative").appendTo(aj);this.bottom_container=$("
    ").addClass("bottom-container").appendTo(aj);this.top_labeltrack=$("
    ").addClass("top-labeltrack").appendTo(this.top_container);this.viewport_container=$("
    ").addClass("viewport-container").attr("id","viewport-container").appendTo(this.browser_content_div);this.content_div=this.viewport_container;m(this.viewport_container,ah);this.intro_div=$("
    ").addClass("intro");var ak=$("
    ").text("Add Datasets to Visualization").addClass("action-button").appendTo(this.intro_div).click(function(){add_tracks()});this.nav_labeltrack=$("
    ").addClass("nav-labeltrack").appendTo(this.bottom_container);this.nav_container=$("
    ").addClass("nav-container").prependTo(this.top_container);this.nav=$("
    ").addClass("nav").appendTo(this.nav_container);this.overview=$("
    ").addClass("overview").appendTo(this.bottom_container);this.overview_viewport=$("
    ").addClass("overview-viewport").appendTo(this.overview);this.overview_close=$("Close Overview").addClass("overview-close").hide().appendTo(this.overview_viewport);this.overview_highlight=$("
    ").addClass("overview-highlight").hide().appendTo(this.overview_viewport);this.overview_box_background=$("
    ").addClass("overview-boxback").appendTo(this.overview_viewport);this.overview_box=$("
    ").addClass("overview-box").appendTo(this.overview_viewport);this.default_overview_height=this.overview_box.height();this.nav_controls=$("
    ").addClass("nav-controls").appendTo(this.nav);this.chrom_select=$("").addClass("nav-input").hide().bind("keyup focusout",ai).appendTo(this.nav_controls);this.location_span=$("").addClass("location").appendTo(this.nav_controls);this.location_span.click(function(){ah.location_span.hide();ah.chrom_select.hide();ah.nav_input.val(ah.chrom+":"+ah.low+"-"+ah.high);ah.nav_input.css("display","inline-block");ah.nav_input.select();ah.nav_input.focus()});if(this.vis_id!==undefined){this.hidden_input=$("").attr("type","hidden").val(this.vis_id).appendTo(this.nav_controls)}this.zo_link=$("").click(function(){ah.zoom_out();ah.request_redraw()}).appendTo(this.nav_controls);this.zi_link=$("").click(function(){ah.zoom_in();ah.request_redraw()}).appendTo(this.nav_controls);this.load_chroms_deferred=this.load_chroms({low:0});this.chrom_select.bind("change",function(){ah.change_chrom(ah.chrom_select.val())});this.browser_content_div.click(function(al){$(this).find("input").trigger("blur")});this.browser_content_div.bind("dblclick",function(al){ah.zoom_in(al.pageX,this.viewport_container)});this.overview_box.bind("dragstart",function(al,am){this.current_x=am.offsetX}).bind("drag",function(al,an){var ao=an.offsetX-this.current_x;this.current_x=an.offsetX;var am=Math.round(ao/ah.viewport_container.width()*(ah.max_high-ah.max_low));ah.move_delta(-am)});this.overview_close.click(function(){ah.reset_overview()});this.viewport_container.bind("draginit",function(al,am){if(al.clientX>ah.viewport_container.width()-16){return false}}).bind("dragstart",function(al,am){am.original_low=ah.low;am.current_height=al.clientY;am.current_x=am.offsetX}).bind("drag",function(an,ap){var al=$(this);var aq=ap.offsetX-ap.current_x;var am=al.scrollTop()-(an.clientY-ap.current_height);al.scrollTop(am);ap.current_height=an.clientY;ap.current_x=ap.offsetX;var ao=Math.round(aq/ah.viewport_container.width()*(ah.high-ah.low));ah.move_delta(ao)}).bind("mousewheel",function(an,ap,am,al){if(am){var ao=Math.round(-am/ah.viewport_container.width()*(ah.high-ah.low));ah.move_delta(ao)}});this.top_labeltrack.bind("dragstart",function(al,am){return $("
    ").css({height:ah.browser_content_div.height()+ah.top_labeltrack.height()+ah.nav_labeltrack.height()+1,top:"0px",position:"absolute","background-color":"#ccf",opacity:0.5,"z-index":1000}).appendTo($(this))}).bind("drag",function(ap,aq){$(aq.proxy).css({left:Math.min(ap.pageX,aq.startX),width:Math.abs(ap.pageX-aq.startX)});var am=Math.min(ap.pageX,aq.startX)-ah.container.offset().left,al=Math.max(ap.pageX,aq.startX)-ah.container.offset().left,ao=(ah.high-ah.low),an=ah.viewport_container.width();ah.update_location(Math.round(am/an*ao)+ah.low,Math.round(al/an*ao)+ah.low)}).bind("dragend",function(aq,ar){var am=Math.min(aq.pageX,ar.startX),al=Math.max(aq.pageX,ar.startX),ao=(ah.high-ah.low),an=ah.viewport_container.width(),ap=ah.low;ah.low=Math.round(am/an*ao)+ap;ah.high=Math.round(al/an*ao)+ap;$(ar.proxy).remove();ah.request_redraw()});this.add_label_track(new ad(this,{content_div:this.top_labeltrack}));this.add_label_track(new ad(this,{content_div:this.nav_labeltrack}));$(window).bind("resize",function(){ah.resize_window()});$(document).bind("redraw",function(){ah.redraw()});this.reset();$(window).trigger("resize");this.update_intro_div()},update_intro_div:function(){if(this.num_tracks===0){this.intro_div.appendTo(this.viewport_container)}else{this.intro_div.remove()}},update_location:function(ah,ai){this.location_span.text(commatize(ah)+" - "+commatize(ai));this.nav_input.val(this.chrom+":"+commatize(ah)+"-"+commatize(ai))},load_chroms:function(aj){aj.num=x;$.extend(aj,(this.vis_id!==undefined?{vis_id:this.vis_id}:{dbkey:this.dbkey}));var ah=this,ai=$.Deferred();$.ajax({url:chrom_url,data:aj,dataType:"json",success:function(al){if(al.chrom_info.length===0){alert("Invalid chromosome: "+aj.chrom);return}if(al.reference){ah.add_label_track(new C(ah))}ah.chrom_data=al.chrom_info;var ao='';for(var an=0,ak=ah.chrom_data.length;an'+am+""}if(al.prev_chroms){ao+='"}if(al.next_chroms){ao+='"}ah.chrom_select.html(ao);ah.chrom_start_index=al.start_index;ai.resolve(al)},error:function(){alert("Could not load chroms for this dbkey:",ah.dbkey)}});return ai},change_chrom:function(al,ai,an){if(!al||al==="None"){return}var ak=this;if(al==="previous"){ak.load_chroms({low:this.chrom_start_index-x});return}if(al==="next"){ak.load_chroms({low:this.chrom_start_index+x});return}var am=$.grep(ak.chrom_data,function(ap,aq){return ap.chrom===al})[0];if(am===undefined){ak.load_chroms({chrom:al},function(){ak.change_chrom(al,ai,an)});return}else{if(al!==ak.chrom){ak.chrom=al;ak.chrom_select.val(ak.chrom);ak.max_high=am.len-1;ak.reset();ak.request_redraw(true);for(var ao=0,ah=ak.tracks.length;aoah.max_high){ah.high=ah.max_high;ah.low=ah.max_high-ai}else{ah.high-=aj;ah.low-=aj}}ah.request_redraw()},add_drawable:function(ah){A.prototype.add_drawable.call(this,ah);ah.init();this.has_changes=true;this.update_intro_div()},add_label_track:function(ah){ah.view=this;ah.init();this.label_tracks.push(ah)},remove_drawable:function(aj,ai){A.prototype.remove_drawable.call(this,aj);if(ai){var ah=this;aj.container_div.fadeOut("slow",function(){$(this).remove();ah.update_intro_div()});this.has_changes=true}},reset:function(){this.low=this.max_low;this.high=this.max_high;this.viewport_container.find(".yaxislabel").remove()},request_redraw:function(ap,ah,ao,ai){var an=this,al=(ai?[ai]:an.tracks),aj;var ai;for(var am=0;amthis.max_high){ak=this.max_high}var aq=this.high-this.low;if(this.high!==0&&aq").addClass("dynamic-tool").hide();this.parent_div.bind("drag",function(ay){ay.stopPropagation()}).click(function(ay){ay.stopPropagation()}).bind("dblclick",function(ay){ay.stopPropagation()});var at=$("
    ").appendTo(this.parent_div).text(this.name);var aq=this.params;var ao=this;$.each(this.params,function(az,aC){var aB=$("
    ").addClass("param-row").appendTo(ao.parent_div);var ay=$("
    ").addClass("param-label").text(aC.label).appendTo(aB);var aA=$("
    ").addClass("slider").html(aC.html).appendTo(aB);aA.find(":input").val(aC.value);$("
    ").appendTo(aB)});this.parent_div.find("input").click(function(){$(this).select()});var ax=$("
    ").addClass("param-row").appendTo(this.parent_div);var am=$("").attr("value","Run on complete dataset").appendTo(ax);var ah=$("").attr("value","Run on visible region").css("margin-left","3em").appendTo(ax);var ao=this;ah.click(function(){ao.run_on_region()});am.click(function(){ao.run_on_dataset()})};q(s.prototype,{get_param_values_dict:function(){var ah={};this.parent_div.find(":input").each(function(){var ai=$(this).attr("name"),aj=$(this).val();ah[ai]=JSON.stringify(aj)});return ah},get_param_values:function(){var ai=[];var ah={};this.parent_div.find(":input").each(function(){var aj=$(this).attr("name"),ak=$(this).val();if(aj){ai[ai.length]=ak}});return ai},run_on_dataset:function(){var ah=this;ah.run({dataset_id:this.track.original_dataset_id,tool_id:ah.name},null,function(ai){show_modal(ah.name+" is Running",ah.name+" is running on the complete dataset. Tool outputs are in dataset's history.",{Close:hide_modal})})},run_on_region:function(){var ai={dataset_id:this.track.original_dataset_id,chrom:this.track.view.chrom,low:this.track.view.low,high:this.track.view.high,tool_id:this.name},al=this.track,aj=ai.tool_id+al.tool_region_and_parameters_str(ai.chrom,ai.low,ai.high),ah,am;if(al.container===view){var ak=new S(this.name,this.track.view,this.track.container);al.container.add_drawable(ak);al.container.remove_drawable(al);ak.add_drawable(al);al.container_div.appendTo(ak.content_div);ah=ak}else{ah=al.container}if(al instanceof e){am=new X(aj,view,ah,"hda");am.change_mode(al.mode);ah.add_drawable(am)}am.content_div.text("Starting job.");this.run(ai,am,function(an){am.dataset_id=an.dataset_id;am.content_div.text("Running job.");am.init()})},run:function(ai,aj,ak){$.extend(ai,this.get_param_values_dict());var ah=function(){$.getJSON(rerun_tool_url,ai,function(al){if(al==="no converter"){aj.container_div.addClass("error");aj.content_div.text(L)}else{if(al.error){aj.container_div.addClass("error");aj.content_div.text(z+al.message)}else{if(al==="pending"){aj.container_div.addClass("pending");aj.content_div.text("Converting input data so that it can be used quickly with tool.");setTimeout(ah,2000)}else{ak(al)}}}})};ah()}});var P=function(ai,ah,aj,ak){this.name=ai;this.label=ah;this.html=aj;this.value=ak};var g=function(aj,ai,al,am,ak,ah){P.call(this,aj,ai,al,am);this.min=ak;this.max=ah};var h=function(ai,ah,aj,ak){this.name=ai;this.index=ah;this.tool_id=aj;this.tool_exp_name=ak};var Y=function(ai,ah,aj,ak){h.call(this,ai,ah,aj,ak);this.low=-Number.MAX_VALUE;this.high=Number.MAX_VALUE;this.min=Number.MAX_VALUE;this.max=-Number.MAX_VALUE;this.container=null;this.slider=null;this.slider_label=null};q(Y.prototype,{applies_to:function(ah){if(ah.length>this.index){return true}return false},keep:function(ah){if(!this.applies_to(ah)){return true}var ai=parseFloat(ah[this.index]);return(isNaN(ai)||(ai>=this.low&&ai<=this.high))},update_attrs:function(ai){var ah=false;if(!this.applies_to(ai)){return ah}if(ai[this.index]this.max){this.max=Math.ceil(ai[this.index]);ah=true}return ah},update_ui_elt:function(){if(this.min!=this.max){this.container.show()}else{this.container.hide()}var aj=function(am,ak){var al=ak-am;return(al<=2?0.01:1)};var ai=this.slider.slider("option","min"),ah=this.slider.slider("option","max");if(this.minah){this.slider.slider("option","min",this.min);this.slider.slider("option","max",this.max);this.slider.slider("option","step",aj(this.min,this.max));this.slider.slider("option","values",[this.min,this.max])}}});var ac=function(ar,ay){this.track=ar;this.filters=[];for(var at=0;at").attr("size",input_size).attr("maxlength",input_size).attr("value",aD).appendTo(aB).focus().select().click(function(aE){aE.stopPropagation()}).blur(function(){$(this).remove();aB.text(aD)}).keyup(function(aI){if(aI.keyCode===27){$(this).trigger("blur")}else{if(aI.keyCode===13){var aG=aC.slider("option","min"),aE=aC.slider("option","max"),aH=function(aJ){return(isNaN(aJ)||aJ>aE||aJ").addClass("filters").hide();this.parent_div.bind("drag",function(aA){aA.stopPropagation()}).click(function(aA){aA.stopPropagation()}).bind("dblclick",function(aA){aA.stopPropagation()}).bind("keydown",function(aA){aA.stopPropagation()});var av=$("
    ").addClass("sliders").appendTo(this.parent_div);var ap=this;$.each(this.filters,function(aD,aF){aF.container=$("
    ").addClass("slider-row").appendTo(av);var aE=$("
    ").addClass("elt-label").appendTo(aF.container);var aC=$("").addClass("slider-name").text(aF.name+" ").appendTo(aE);var aB=$("");var aH=$("").addClass("slider-value").appendTo(aE).append("[").append(aB).append("]");var aA=$("
    ").addClass("slider").appendTo(aF.container);aF.control_element=$("
    ").attr("id",aF.name+"-filter-control").appendTo(aA);var aG=[0,0];aF.control_element.slider({range:true,min:Number.MAX_VALUE,max:-Number.MIN_VALUE,values:[0,0],slide:function(aJ,aK){var aI=aK.values;aB.text(aI[0]+"-"+aI[1]);aF.low=aI[0];aF.high=aI[1];ap.track.request_draw(true,true)},change:function(aI,aJ){aF.control_element.slider("option","slide").call(aF.control_element,aI,aJ)}});aF.slider=aF.control_element;aF.slider_label=aB;al(aH,aB,aF.control_element);$("
    ").appendTo(aF.container)});if(this.filters.length!==0){var am=$("
    ").addClass("param-row").appendTo(av);var ao=$("").attr("value","Run on complete dataset").appendTo(am);var aj=this;ao.click(function(){aj.run_on_dataset()})}var aq=$("
    ").addClass("display-controls").appendTo(this.parent_div),an=$("").addClass("elt-label").text("Transparency:").appendTo(aq),ai=$("').attr("id",aj).attr("name",aj).attr("checked",ao))}else{if(ak.type==="text"){ar.append($('').attr("id",aj).val(ao).click(function(){$(this).select()}))}else{if(ak.type==="color"){var an=$("").attr("id",aj).attr("name",aj).val(ao);var ap=$("
    ").hide();var al=$("
    ").appendTo(ap);var aq=$("
    ").appendTo(al).farbtastic({width:100,height:100,callback:an,color:ao});$("
    ").append(an).append(ap).appendTo(ar).bind("click",function(at){ap.css({left:$(this).position().left+($(an).width()/2)-60,top:$(this).position().top+$(this.height)}).show();$(document).bind("click.color-picker",function(){ap.hide();$(document).unbind("click.color-picker")});at.stopPropagation()})}else{ar.append($("").attr("id",aj).attr("name",aj).val(ao))}}}}});return ah},update_from_form:function(ah){var aj=this;var ai=false;$.each(this.params,function(ak,am){if(!am.hidden){var an="param_"+ak;var al=ah.find("#"+an).val();if(am.type==="float"){al=parseFloat(al)}else{if(am.type==="int"){al=parseInt(al)}else{if(am.type==="bool"){al=ah.find("#"+an).is(":checked")}}}if(al!==aj.values[am.key]){aj.values[am.key]=al;ai=true}}});if(ai){this.onchange()}}});var b=function(aj,ai,ah,ak){this.index=aj;this.low=aj*T*ai;this.high=(aj+1)*T*ai;this.resolution=ai;this.canvas=$("
    ").append(ah);this.data=ak;this.stale=false};var l=function(aj,ai,ah,ak,al){b.call(this,aj,ai,ah,ak);this.max_val=al};var R=function(ak,aj,ai,an,am,ah){b.call(this,ak,aj,ai,an);this.message=am;this.feature_mapper=ah;var al=this;$(this.canvas).mousemove(function(ap){var ao=al.feature_mapper.get_feature_data(ap.offsetX,ap.offsetY)})};var j=function(ak,ai,ah,aj,al,am){r.call(this,ak,ai,ah,{},"draghandle");this.data_url=(al?al:default_data_url);this.data_url_extra_params={};this.data_query_wait=(am?am:M);this.dataset_check_url=converted_datasets_state_url;if(!j.id_counter){j.id_counter=0}this.container_div=$("
    ").addClass("track").attr("id","track_"+j.id_counter++).css("position","relative");if(!this.hidden){this.header_div=$("
    ").appendTo(this.container_div);if(this.view.editor){this.drag_div=$("
    ").addClass(this.drag_handle_class).appendTo(this.header_div)}this.name_div=$("
    ").appendTo(this.header_div);this.name_div.text(this.name);this.name_div.attr("id",this.name.replace(/\s+/g,"-").replace(/[^a-zA-Z0-9\-]/g,"").toLowerCase())}this.content_div=$("
    ").appendTo(this.container_div);this.container.content_div.append(this.container_div)};q(j.prototype,r.prototype,{get_type:function(){if(this instanceof ad){return"LabelTrack"}else{if(this instanceof C){return"ReferenceTrack"}else{if(this instanceof k){return"LineTrack"}else{if(this instanceof Z){return"ReadTrack"}else{if(this instanceof X){return"ToolDataFeatureTrack"}else{if(this instanceof W){return"VcfTrack"}else{if(this instanceof e){return"FeatureTrack"}}}}}}}return""},init:function(){var ah=this;ah.enabled=false;ah.tile_cache.clear();ah.data_manager.clear();ah.initial_canvas=undefined;ah.content_div.css("height","auto");ah.container_div.removeClass("nodata error pending");if(!ah.dataset_id){return}$.getJSON(converted_datasets_state_url,{hda_ldda:ah.hda_ldda,dataset_id:ah.dataset_id,chrom:ah.view.chrom},function(ai){if(!ai||ai==="error"||ai.kind==="error"){ah.container_div.addClass("error");ah.content_div.text(p);if(ai.message){var ak=ah.view.tracks.indexOf(ah);var aj=$(" ").text("View error").click(function(){show_modal("Trackster Error","
    "+ai.message+"
    ",{Close:hide_modal})});ah.content_div.append(aj)}}else{if(ai==="no converter"){ah.container_div.addClass("error");ah.content_div.text(L)}else{if(ai==="no data"||(ai.data!==undefined&&(ai.data===null||ai.data.length===0))){ah.container_div.addClass("nodata");ah.content_div.text(G)}else{if(ai==="pending"){ah.container_div.addClass("pending");ah.content_div.text(u);setTimeout(function(){ah.init()},ah.data_query_wait)}else{if(ai.status==="data"){if(ai.valid_chroms){ah.valid_chroms=ai.valid_chroms;ah.make_name_popup_menu()}ah.content_div.text(ab);if(ah.view.chrom){ah.content_div.text("");ah.content_div.css("height",ah.height_px+"px");ah.enabled=true;$.when(ah.predraw_init()).done(function(){ah.container_div.removeClass("nodata error pending");ah.request_draw()})}}}}}}})},predraw_init:function(){},});var O=function(ao,am){var ai=this,ap=ai.view;n(ai.container_div,ai.drag_handle_class,".group",ai);this.filters_manager=new ac(this,(ao!==undefined?ao:{}));this.filters_available=false;this.filters_visible=false;this.tool=(am!==undefined&&obj_length(am)>0?new s(this,am):undefined);this.is_overview=false;if(ai.hidden){return}if(this.filters_manager){this.filters_div=this.filters_manager.parent_div;this.header_div.after(this.filters_div)}if(this.tool){this.dynamic_tool_div=this.tool.parent_div;this.header_div.after(this.dynamic_tool_div)}if(ai.display_modes!==undefined){if(ai.mode_div===undefined){ai.mode_div=$("
    ").appendTo(ai.header_div);var aj=(ai.config&&ai.config.values.mode?ai.config.values.mode:ai.display_modes[0]);ai.mode=aj;ai.mode_div.text(aj);var ah={};for(var ak=0,an=ai.display_modes.length;ak0){var aj=(ai.filters_div.is(":visible")?"Hide filters":"Show filters");ah[aj]=function(){ai.filters_visible=(ai.filters_div.is(":visible"));if(ai.filters_visible){ai.filters_manager.reset_filters()}ai.filters_div.toggle();ai.make_name_popup_menu()}}if(ai.tool){var aj=(ai.dynamic_tool_div.is(":visible")?"Hide tool":"Show tool");ah[aj]=function(){if(!ai.dynamic_tool_div.is(":visible")){ai.set_name(ai.name+ai.tool_region_and_parameters_str())}else{menu_option_text="Show dynamic tool";ai.revert_name()}ai.dynamic_tool_div.toggle();ai.make_name_popup_menu()}}ah.Remove=function(){ai.remove()};make_popupmenu(ai.name_div,ah)},set_is_overview:function(ah){this.is_overview=ah;this.make_name_popup_menu()},get_overview_tile:function(){var ah=this;view=ah.view,resolution=Math.pow(E,Math.ceil(Math.log((view.max_high-view.max_low)/T)/Math.log(E))),view_width=view.container.width(),w_scale=view_width/(view.max_high-view.max_low),overview_tile=$.Deferred();$.when(ah.data_manager.get_data(view.max_low,view.max_high,"Auto",resolution,ah.data_url_extra_params)).then(function(ai){var ak=ah._gen_tile_cache_key(view_width,w_scale,0),am=ah.tile_cache.get(ak);if(!am){am=ah.draw_tile(ai,"Auto",resolution,0,w_scale);ah.tile_cache.set(ak,am)}var ap=$(am.canvas.find("canvas")),al=ap.clone(),ao=ap.get(0).getContext("2d"),aj=al.get(0).getContext("2d"),an=ao.getImageData(0,0,ao.canvas.width,ao.canvas.height);aj.putImageData(an,-ah.left_offset,(am.data.dataset_type==="summary_tree"?V:0));new_tile=new b(-1,resolution,al);overview_tile.resolve(new_tile)});return overview_tile},_gen_tile_cache_key:function(ai,aj,ah){return ai+"_"+aj+"_"+ah},request_draw:function(ai,ah){this.view.request_redraw(false,ai,ah,this)},_draw:function(aj,ar){if(!this.enabled){return}if(!(this instanceof C)&&(!this.dataset_id)){return}var aq=this.view.low,an=this.view.high,ao=an-aq,ak=this.view.container.width(),av=ak/ao,am=this.view.resolution,au=$("
    ");if(!ar){this.content_div.children().remove()}this.content_div.append(au);this.max_height=0;var ai=Math.floor(aq/am/T);var ap=true;var at=[];var ah=0;while((ai*T*am)view.canvas_manager.char_width_px){aw=view.reference_track.data_manager.get_data(ap,ax,al.mode,an,view.reference_track.data_url_extra_params);if(ar(aw)){am=false}}if(am){q(ah,ao);var aq=al.draw_tile(ah,al.mode,an,ak,ay,aw);if(aq!==undefined){al.tile_cache.set(at,aq);al.show_tile(aq,au,ay)}return aq}$.when(ah,aw).then(function(){view.request_redraw()});return null},show_tile:function(ao,aq,ar){var aj=this,ai=ao.canvas,an=ai;if(ao.message){var at=$("
    "),ap=$("
    ").addClass("tile-message").text(ao.message).css({height:F-1,width:ao.canvas.width}).appendTo(at),al=$("").addClass("icon more-down").appendTo(ap),ah=$("").addClass("icon more-across").appendTo(ap);at.append(ai);an=at;al.click(function(){ao.stale=true;aj.data_manager.get_more_data(ao.low,ao.high,aj.mode,ao.resolution,{},aj.data_manager.DEEP_DATA_REQ);aj.request_draw()}).dblclick(function(au){au.stopPropagation()});ah.click(function(){ao.stale=true;aj.data_manager.get_more_data(ao.low,ao.high,aj.mode,ao.resolution,{},aj.data_manager.BROAD_DATA_REQ);aj.request_draw()}).dblclick(function(au){au.stopPropagation()})}var am=this.view.high-this.view.low,ak=(ao.low-this.view.low)*ar;if(this.left_offset){ak-=this.left_offset}an.css({position:"absolute",top:0,left:ak,height:""});aq.append(an);aj.max_height=Math.max(aj.max_height,an.height());aj.content_div.css("height",aj.max_height+"px");aq.children().css("height",aj.max_height+"px")},_get_tile_bounds:function(ah,ai){var ak=ah*T*ai,al=T*ai,aj=(ak+al<=this.view.max_high?ak+al:this.view.max_high);return[ak,aj]},tool_region_and_parameters_str:function(aj,ah,ak){var ai=this,al=(aj!==undefined&&ah!==undefined&&ak!==undefined?aj+":"+ah+"-"+ak:"all");return" - region=["+al+"], parameters=["+ai.tool.get_param_values().join(", ")+"]"}});var ad=function(ai,ah){this.hidden=true;j.call(this,"label",ai,ah,{});this.container_div.addClass("label-track")};q(ad.prototype,j.prototype,{init:function(){this.enabled=true},_draw:function(){var aj=this.view,ak=aj.high-aj.low,an=Math.floor(Math.pow(10,Math.floor(Math.log(ak)/Math.log(10)))),ah=Math.floor(aj.low/an)*an,al=this.view.container.width(),ai=$("
    ");while(ah"+commatize(ah)+"
    ").css({position:"absolute",left:am-1}));ah+=an}this.content_div.children(":first").remove();this.content_div.append(ai)}});var C=function(ah){this.hidden=true;j.call(this,"reference",ah,{content_div:ah.top_labeltrack},{});O.call(this);ah.reference_track=this;this.left_offset=200;this.height_px=12;this.container_div.addClass("reference-track");this.content_div.css("background","none");this.content_div.css("min-height","0px");this.content_div.css("border","none");this.data_url=reference_url;this.data_url_extra_params={dbkey:ah.dbkey};this.data_manager=new J(D,this,false);this.tile_cache=new c(v)};q(C.prototype,r.prototype,O.prototype,{init:function(){this.enabled=true},draw_tile:function(ar,an,am,ai,at){var al=this,aj=T*am;if(at>this.view.canvas_manager.char_width_px){if(ar===null){al.content_div.css("height","0px");return}var ak=this.view.canvas_manager.new_canvas();var aq=ak.getContext("2d");ak.width=Math.ceil(aj*at+al.left_offset);ak.height=al.height_px;aq.font=aq.canvas.manager.default_font;aq.textAlign="center";ar=ar.data;for(var ao=0,ap=ar.length;ao");$(ah.container_div).hover(function(){ak=true;ai.show()},function(){ak=false;if(!aj){ai.hide()}});ai.hide().bind("dragstart",function(al,am){aj=true;am.original_height=$(ah.content_div).height()}).bind("drag",function(am,an){var al=Math.min(Math.max(an.original_height+an.deltaY,ah.min_height_px),ah.max_height_px);$(ah.content_div).css("height",al);ah.height_px=al;ah.request_draw(true)}).bind("dragend",function(al,am){ah.tile_cache.clear();aj=false;if(!ak){ai.hide()}ah.config.values.height=ah.height_px}).appendTo(ah.container_div)},predraw_init:function(){var ah=this,ai=ah.view.tracks.indexOf(ah);ah.vertical_range=undefined;return $.getJSON(ah.data_url,{stats:true,chrom:ah.view.chrom,low:null,high:null,hda_ldda:ah.hda_ldda,dataset_id:ah.dataset_id},function(aj){ah.container_div.addClass("line-track");var al=aj.data;if(isNaN(parseFloat(ah.prefs.min_value))||isNaN(parseFloat(ah.prefs.max_value))){ah.prefs.min_value=al.min;ah.prefs.max_value=al.max;$("#track_"+ai+"_minval").val(ah.prefs.min_value);$("#track_"+ai+"_maxval").val(ah.prefs.max_value)}ah.vertical_range=ah.prefs.max_value-ah.prefs.min_value;ah.total_frequency=al.total_frequency;ah.container_div.find(".yaxislabel").remove();var am=$("
    ").addClass("yaxislabel").attr("id","linetrack_"+ai+"_minval").text(y(ah.prefs.min_value));var ak=$("
    ").addClass("yaxislabel").attr("id","linetrack_"+ai+"_maxval").text(y(ah.prefs.max_value));ak.css({position:"absolute",top:"24px",left:"10px"});ak.prependTo(ah.container_div);am.css({position:"absolute",bottom:"2px",left:"10px"});am.prependTo(ah.container_div)})},draw_tile:function(au,am,al,aj,at){if(this.vertical_range===undefined){return}var ah=this._get_tile_bounds(aj,al),an=ah[0],ar=ah[1],ai=Math.ceil((ar-an)*at),ap=this.height_px;var ak=this.view.canvas_manager.new_canvas();ak.width=ai,ak.height=ap;var aq=ak.getContext("2d");var ao=new N.LinePainter(au.data,an,ar,this.prefs,am);ao.draw(aq,ai,ap);return new b(aj,al,ak,au.data)}});var e=function(ah,an,ai,am,ap,ao,ak,al){var aj=this;this.display_modes=["Auto","Histogram","Dense","Squish","Pack"];j.call(this,ah,an,ai,ao);O.call(this,ak,al);this.config=new H({track:this,params:[{key:"name",label:"Name",type:"text",default_value:ah},{key:"block_color",label:"Block color",type:"color",default_value:"#444"},{key:"label_color",label:"Label color",type:"color",default_value:"black"},{key:"show_counts",label:"Show summary counts",type:"bool",default_value:true},{key:"mode",type:"string",default_value:this.mode,hidden:true},],saved_values:ao,onchange:function(){aj.set_name(aj.prefs.name);aj.tile_cache.clear();aj.request_draw()}});this.prefs=this.config.values;this.height_px=0;this.container_div.addClass("feature-track");this.hda_ldda=am;this.dataset_id=ap;this.original_dataset_id=ap;this.show_labels_scale=0.001;this.showing_details=false;this.summary_draw_height=30;this.inc_slots={};this.start_end_dct={};this.tile_cache=new c(d);this.data_manager=new U(20,this);this.left_offset=200;this.painter=N.LinkedFeaturePainter};q(e.prototype,r.prototype,O.prototype,{postdraw_actions:function(ax,ah,ay,aw){O.prototype.postdraw_actions.call(this,ax,aw);var ak=this;if(aw){var am=ak.content_div.children();var an=false;for(var al=am.length-1,ar=0;al>=ar;al--){var aj=$(am[al]);if(an){aj.remove()}else{if(aj.children().length!==0){an=true}}}}if(ak.mode=="Histogram"){var aq=-1;for(var al=0;alaq){aq=av}}for(var al=0;alaj-ao){ax=aj-ao}var at=Math.floor((aj-ao)/ax),aw=[],ak=0;var am=0,an=0,ar,av=0,ap=[],au,aq;var ai=function(aA,az,aB,ay){aA[0]=az+aB*ay;aA[1]=az+(aB+1)*ay};while(avak){ak=au[1]}av++}return{max:ak,delta:at,data:aw}},draw_tile:function(aw,az,aE,aI,ar,ak){var aB=this,am=aB._get_tile_bounds(aI,aE),aL=am[0],ai=am[1],ay=ai-aL,aC=Math.ceil(ay*ar),aP=25,al=this.left_offset,ax,an;if(az==="Auto"){if(aw.dataset_type==="summary_tree"){az=aw.dataset_type}else{if(aw.extra_info==="no_detail"){az="no_detail"}else{var aO=aw.data;if(this.view.high-this.view.low>K){az="Squish"}else{az="Pack"}}}this.update_auto_mode(az)}if(az==="summary_tree"||az==="Histogram"){an=this.summary_draw_height;this.container_div.find(".yaxislabel").remove();var ah=$("
    ").addClass("yaxislabel");ah.text(aw.max);ah.css({position:"absolute",top:"24px",left:"10px",color:this.prefs.label_color});ah.prependTo(this.container_div);var aj=this.view.canvas_manager.new_canvas();aj.width=aC+al;aj.height=an+V;if(aw.dataset_type!="summary_tree"){var at=this.get_summary_tree_data(aw.data,aL,ai,200);if(aw.max){at.max=aw.max}aw=at}var aM=new N.SummaryTreePainter(aw,aL,ai,this.prefs);var aD=aj.getContext("2d");aD.translate(al,V);aM.draw(aD,aC,an);return new l(aI,aE,aj,aw.data,aw.max)}var ax,ap=1;if(az==="no_detail"||az==="Squish"||az==="Pack"){ap=this.incremental_slots(ar,aw.data,az);ax=this.inc_slots[ar].slots}var aq=[];if(aw.data){var au=this.filters_manager.filters;for(var aF=0,aH=aw.data.length;aFB[0]&&G=0){g-=v;j="left"}else{k+=v;j="right"}}var t=q(g,k);if(t>=0){if(h[t]===undefined){h[t]=[]}h[t].push([g,k]);s[o]=t;n=Math.max(n,t)}else{}}return n+1}})};var painters_module=function(k,y){var v=k("class").extend;var q=function(J,B,H,A,G,E){if(E===undefined){E=4}var D=A-B;var C=G-H;var F=Math.floor(Math.sqrt(D*D+C*C)/E);var K=D/F;var I=C/F;var z;for(z=0;z1){C=Math.ceil((T[1][0]-T[0][0])*B)}else{C=10}for(var O=0,P=T.length;OD){Q=true;E=D}}if(I==="Histogram"){E=Math.round(E/J*z);N.fillRect(R,U,C,-E)}else{if(I==="Intensity"){E=255-Math.floor((E-H)/J*255);N.fillStyle="rgb("+E+","+E+","+E+")";N.fillRect(R,0,C,z)}else{E=Math.round(z-(E-H)/J*z);if(F){N.lineTo(R,E)}else{F=true;if(I==="Filled"){N.moveTo(R,z);N.lineTo(R,E)}else{N.moveTo(R,E)}}}}N.fillStyle=this.prefs.overflow_color;if(Q||G){var S;if(I==="Histogram"||I==="Intensity"){S=C}else{R-=2;S=4}if(Q){N.fillRect(R,0,S,3)}if(G){N.fillRect(R,z-3,S,3)}}N.fillStyle=this.prefs.color}if(I==="Filled"){if(F){N.lineTo(R,U);N.lineTo(0,U)}N.fill()}else{N.stroke()}N.restore()};var o=function(z){this.feature_positions={};this.slot_height=z;this.translation=0};o.prototype.map_feature_data=function(A,C,z,B){if(!this.feature_positions[C]){this.feature_positions[C]=[]}this.feature_positions[C].push({data:A,x_start:z,x_end:B})};o.prototype.get_feature_data=function(z,D){var C=Math.floor(D/this.slot_height),B;if(!this.feature_positions[C]){return null}z+=this.translation;for(var A=0;A=B.x_start&&z<=B.x_end){return B.data}}};var p=function(C,E,z,B,D,A){n.call(this,C,E,z,B,D);this.alpha_generator=(A?A:new g())};p.prototype.default_prefs={block_color:"#FFF",connector_color:"#FFF"};v(p.prototype,{get_required_height:function(A){var z=y_scale=this.get_row_height(),B=this.mode;if(B==="no_detail"||B==="Squish"||B==="Pack"){z=A*y_scale}return z+Math.max(Math.round(y_scale/2),5)},draw:function(L,J,H,G){var R=this.data,E=this.view_start,N=this.view_end;L.save();L.fillStyle=this.prefs.block_color;L.textAlign="right";var I=this.view_end-this.view_start,F=J/I,M=this.get_row_height(),Q=new o(M),C;for(var O=0,P=R.length;OE)&&(this.mode=="Dense"||A!==null)){C=this.draw_element(L,this.mode,B,A,E,N,F,M,J);Q.map_feature_data(B,A,C[0],C[1])}}L.restore();return Q},draw_element:function(F,B,H,D,C,E,G,A,z){console.log("WARNING: Unimplemented function.");return[0,0]}});var d=10,j=3,m=5,x=10,f=1,t=3,e=3,a=9,l=2,h="#ccc";var s=function(C,E,z,B,D,A){p.call(this,C,E,z,B,D,A)};v(s.prototype,p.prototype,{get_row_height:function(){var A=this.mode,z;if(A==="Dense"){z=d}else{if(A==="no_detail"){z=j}else{if(A==="Squish"){z=m}else{z=x}}}return z},draw_element:function(L,E,T,G,N,ae,ai,ak,z){var Q=T[0],ag=T[1],Y=T[2],O=T[3],Z=Math.floor(Math.max(0,(ag-N)*ai)),M=Math.ceil(Math.min(z,Math.max(0,(Y-N)*ai))),X=Z,aj=M,W=(E==="Dense"?0:(0+G))*ak,K,ac,P=null,al=null,C=this.prefs.block_color,ab=this.prefs.label_color;L.globalAlpha=this.alpha_generator.gen_alpha(T);if(E=="Dense"){G=1}if(E==="no_detail"){L.fillStyle=C;L.fillRect(Z,W+5,M-Z,f)}else{var J=T[4],V=T[5],aa=T[6],D=T[7];if(V&&aa){P=Math.floor(Math.max(0,(V-N)*ai));al=Math.ceil(Math.min(z,Math.max(0,(aa-N)*ai)))}var ah,R;if(E==="Squish"||E==="Dense"){ah=1;R=e}else{ah=5;R=a}if(!D){if(T.strand){if(T.strand==="+"){L.fillStyle=L.canvas.manager.get_pattern("right_strand_inv")}else{if(T.strand==="-"){L.fillStyle=L.canvas.manager.get_pattern("left_strand_inv")}}}else{L.fillStyle=C}L.fillRect(Z,W,M-Z,R)}else{var I,S;if(E==="Squish"||E==="Dense"){L.fillStyle=h;I=W+Math.floor(e/2)+1;S=1}else{if(J){var I=W;var S=R;if(J==="+"){L.fillStyle=L.canvas.manager.get_pattern("right_strand")}else{if(J==="-"){L.fillStyle=L.canvas.manager.get_pattern("left_strand")}}}else{L.fillStyle=h;I+=(e/2)+1;S=1}}L.fillRect(Z,I,M-Z,S);for(var af=0,B=D.length;afU){continue}L.fillStyle=C;L.fillRect(A,W+(R-ah)/2+1,U-A,ah);if(P!==undefined&&aa>V&&!(A>al||UN){L.fillStyle=ab;if(N===0&&Z-L.measureText(O).width<0){L.textAlign="left";L.fillText(O,M+l,W+8);aj+=L.measureText(O).width+l}else{L.textAlign="right";L.fillText(O,Z-l,W+8);X-=L.measureText(O).width+l}}}L.globalAlpha=1;return[X,aj]}});var b=function(C,E,z,B,D,A){p.call(this,C,E,z,B,D,A)};v(b.prototype,p.prototype,{draw_element:function(S,N,H,D,V,B,K,T,Q){var H=data[i],J=H[0],R=H[1],C=H[2],M=H[3],F=Math.floor(Math.max(0,(R-V)*K)),I=Math.ceil(Math.min(Q,Math.max(0,(C-V)*K))),E=(N==="Dense"?0:(0+D))*T,z,W,A=null,L=null;if(no_label){S.fillStyle=block_color;S.fillRect(F+left_offset,E+5,I-F,1)}else{var U=H[4],P=H[5],G=H[6];z=9;W=1;S.fillRect(F+left_offset,E,I-F,z);if(N!=="Dense"&&M!==undefined&&R>V){S.fillStyle=label_color;if(V===0&&F-S.measureText(M).width<0){S.textAlign="left";S.fillText(M,I+2+left_offset,E+8)}else{S.textAlign="right";S.fillText(M,F-2+left_offset,E+8)}S.fillStyle=block_color}var O=U+" / "+P;if(R>V&&S.measureText(O).width<(I-F)){S.fillStyle="white";S.textAlign="center";S.fillText(O,left_offset+F+(I-F)/2,E+8);S.fillStyle=block_color}}return[F,I]}});var u=function(D,F,z,C,E,A,B){p.call(this,D,F,z,C,E,A);this.ref_seq=B};u.prototype.default_prefs=v({},p.prototype.default_prefs,{show_insertions:false});v(u.prototype,p.prototype,{get_row_height:function(){var z,A=this.mode;if(A==="Dense"){z=d}else{if(A==="Squish"){z=m}else{z=x;if(this.prefs.show_insertions){z*=2}}}return z},draw_read:function(V,Q,M,aa,B,U,J,G,F){V.textAlign="center";var T=this,A=[aa,B],P=0,W=0,S=0;ref_seq=this.ref_seq,char_width_px=V.canvas.manager.char_width_px;var af=[];if((Q==="Pack"||this.mode==="Auto")&&G!==undefined&&M>char_width_px){S=Math.round(M/2)}if(!J){J=[[0,G.length]]}for(var N=0,Y=J.length;N0){V.fillStyle=this.prefs.block_color;V.fillRect(ae-S,F+1,I-ae,9);V.fillStyle=h;for(var ac=0,z=R.length;ac=aa&&H+ac<=B){var ad=Math.floor(Math.max(0,(H+ac-aa)*M));V.fillText(R[ac],ad,F+9)}}}else{V.fillStyle=this.prefs.block_color;V.fillRect(ae,F+4,I-ae,e)}}W+=O;P+=O;break;case"N":V.fillStyle=h;V.fillRect(ae-S,F+5,I-ae,1);P+=O;break;case"D":V.fillStyle="red";V.fillRect(ae-S,F+4,I-ae,3);P+=O;break;case"P":break;case"I":var Z=ae-S;if(is_overlap([H,H+O],A)){var R=G.slice(W,W+O);if(this.prefs.show_insertions){var E=ae-(I-ae)/2;if((Q==="Pack"||this.mode==="Auto")&&G!==undefined&&M>char_width_px){V.fillStyle="yellow";V.fillRect(E-S,F-9,I-ae,9);af[af.length]={type:"triangle",data:[Z,F+4,5]};V.fillStyle=h;switch(seq_tile_overlap){case (OVERLAP_START):R=R.slice(aa-H);break;case (OVERLAP_END):R=R.slice(0,H-B);break;case (CONTAINED_BY):break;case (CONTAINS):R=R.slice(aa-H,H-B);break}for(var ac=0,z=R.length;acchar_width_px){af[af.length]={type:"text",data:[R.length,Z,F+9]}}else{}}}W+=O;break;case"X":W+=O;break}}V.fillStyle="yellow";var ab,D,ag;for(var X=0;XS.canvas.manager.char_width_px){var P=Math.round(J/2)}S.fillStyle=W;if(F[5] instanceof Array){var O=Math.floor(Math.max(0,(F[4][0]-V)*J)),M=Math.ceil(Math.min(Q,Math.max(0,(F[4][1]-V)*J))),L=Math.floor(Math.max(0,(F[5][0]-V)*J)),z=Math.ceil(Math.min(Q,Math.max(0,(F[5][1]-V)*J)));if(F[4][1]>=V&&F[4][0]<=A&&F[4][2]){this.draw_read(S,N,J,V,A,F[4][0],F[4][2],F[4][3],D)}if(F[5][1]>=V&&F[5][0]<=A&&F[5][2]){this.draw_read(S,N,J,V,A,F[5][0],F[5][2],F[5][3],D)}if(L>M){S.fillStyle=h;q(S,M-P,D+5,L-P,D+5)}}else{S.fillStyle=W;this.draw_read(S,N,J,V,A,R,F[4],F[5],D)}if(N==="Pack"&&R>V){S.fillStyle=this.prefs.label_color;var U=1;if(U===0&&E-S.measureText(K).width<0){S.textAlign="left";S.fillText(K,G+l-P,D+8)}else{S.textAlign="right";S.fillText(K,E-l-P,D+8)}S.fillStyle=W}return[0,0]}});y.AlphaGenerator=g;y.SummaryTreePainter=w;y.LinePainter=c;y.LinkedFeaturePainter=s;y.ReadPainter=u;y.VariantPainter=b};(function(d){var c={};var b=function(e){return c[e]};var a=function(f,g){var e={};g(b,e);c[f]=e};a("class",class_module);a("slotting",slotting_module);a("painters",painters_module);a("trackster",trackster_module);for(key in c.trackster){d[key]=c.trackster[key]}})(window); \ No newline at end of file diff --git a/static/scripts/packed/trackster_ui.js b/static/scripts/packed/trackster_ui.js new file mode 100644 index 00000000000..0dd3b73f91e --- /dev/null +++ b/static/scripts/packed/trackster_ui.js @@ -0,0 +1 @@ +var add_bookmark=function(b,a){var g=$("#bookmarks-container"),d=$("
    ").addClass("bookmark").appendTo(g),c=$("
    ").addClass("delete-icon-container").appendTo(d).click(function(){d.slideUp("fast");d.remove();view.has_changes=true;return false}),e=$("").addClass("icon-button delete").appendTo(c),f=$("
    ").addClass("position").appendTo(d),h=$("").text(b).appendTo(f).click(function(){view.go_to(b);return false});annotation_div=get_editable_text_elt(a,true).addClass("annotation").appendTo(d);view.has_changes=true;return d};var addable_objects={LineTrack:LineTrack,FeatureTrack:FeatureTrack,ReadTrack:ReadTrack,DrawableGroup:DrawableGroup};var track_from_dict=function(c,b){var a=new addable_objects[c.track_type](c.name,view,b,c.hda_ldda,c.dataset_id,c.prefs,c.filters,c.tool);if(c.mode){a.change_mode(c.mode)}return a};var drawable_collection_from_dict=function(f,a){var e=new addable_objects[f.obj_type](f.name,view,a,f.prefs,view.viewport_container,view);for(var d=0;d bottom) { + // Moving below container. + $(this).insertAfter(container); + var cur_container = html_elt_js_obj_dict[container.attr("id")]; + cur_container.remove_drawable(this_obj); + cur_container.container.add_drawable(this_obj); + return; + } + } + + // Handle dragging into container. Child is appended to container's content_div. + container = null; + for ( i = 0; i < children.length; i++ ) { + child = $(children.get(i)); + top = child.position().top; + bottom = top + child.outerHeight(); + // Dragging into container if child is a container and offset is inside container. + if ( child.is(container_selector) && this !== child.get(0) && + d.offsetY >= top && d.offsetY <= bottom ) { + // Append/prepend based on where offsetY is closest to and return. + if (d.offsetY - top < bottom - d.offsetY) { + child.find(".content-div").prepend(this); + } + else { + child.find(".content-div").append(this); + } + // Update containers. Object may not have container if it is being moved quickly. + if (this_obj.container) { + this_obj.container.remove_drawable(this_obj); + } + html_elt_js_obj_dict[child.attr("id")].add_drawable(this_obj); + return; + } + } + + // Handle sibling movement, aka sorting. + // Determine new position for ( i = 0; i < children.length; i++ ) { if ( d.offsetY < $(children.get(i)).position().top ) { @@ -148,31 +252,40 @@ var sortable = function( element, handle ) { if ( i === children.length ) { if ( this !== children.get(i - 1) ) { parent.append(this); + html_elt_js_obj_dict[parent.attr("id")].move_drawable(this_obj, i); } } else if ( this !== children.get(i) ) { $(this).insertBefore( children.get(i) ); + // Need to adjust insert position if moving down because move is changing + // indices of all list items. + html_elt_js_obj_dict[parent.attr("id")].move_drawable(this_obj, (d.deltaY > 0 ? i-1 : i) ); } }).bind("dragstart", function() { + css_border_props["border-top"] = element.css("border-top"); + css_border_props["border-bottom"] = element.css("border-bottom"); $(this).css({ "border-top": "1px solid blue", "border-bottom": "1px solid blue" }); }).bind("dragend", function() { - $(this).css("border", "0px"); + $(this).css(css_border_props); }); }; -exports.sortable = sortable; + +// TODO: do we need to export? +exports.moveable = moveable; /** * Init constants & functions used throughout trackster. */ var - // Drawing constants; track height is (constant) height of track, and feature height is the - // height of individual features within tracks. Feature height, then, should always be less - // than track height. - CHAR_HEIGHT_PX = 9, // FIXME: font size may not be static - ERROR_PADDING = 10, // Padding at the top of tracks for error messages + // Minimum height of a track's contents; this must correspond to the .track-content's minimum height. + MIN_TRACK_HEIGHT = 16, + // FIXME: font size may not be static + CHAR_HEIGHT_PX = 9, + // Padding at the top of tracks for error messages + ERROR_PADDING = 20, SUMMARY_TREE_TOP_PADDING = CHAR_HEIGHT_PX + 2, // Maximum number of rows un a slotted track MAX_FEATURE_DEPTH = 100, @@ -212,7 +325,14 @@ extend(Cache.prototype, { get: function(key) { var index = this.key_ary.indexOf(key); if (index !== -1) { - this.move_key_to_end(key, index); + if (this.obj_cache[key].stale) { + // Object is stale, so remove key and object. + this.key_ary.splice(index, 1); + delete this.obj_cache[key]; + } + else { + this.move_key_to_end(key, index); + } } return this.obj_cache[key]; }, @@ -256,9 +376,11 @@ extend(DataManager.prototype, Cache.prototype, { /** * Load data from server; returns AJAX object so that use of Deferred is possible. */ - load_data: function(chrom, low, high, mode, resolution, extra_params) { + load_data: function(low, high, mode, resolution, extra_params) { // Setup data request params. - var params = {"chrom": chrom, "low": low, "high": high, "mode": mode, + var + chrom = this.track.view.chrom, + params = {"chrom": chrom, "low": low, "high": high, "mode": mode, "resolution": resolution, "dataset_id" : this.track.dataset_id, "hda_ldda": this.track.hda_ldda}; $.extend(params, extra_params); @@ -280,9 +402,9 @@ extend(DataManager.prototype, Cache.prototype, { }); }, /** - * Get data and do callback. + * Get track data. */ - get_data: function(chrom, low, high, mode, resolution, extra_params) { + get_data: function(low, high, mode, resolution, extra_params) { // Debugging: //console.log("get_data", low, high, mode); /* @@ -293,10 +415,8 @@ extend(DataManager.prototype, Cache.prototype, { */ // Look for entry and return if found. - var entry = this.get(this.gen_key(low, high, mode)); - if (entry) { - return entry; - } + var entry = this.get_data_from_cache(low, high, mode); + if (entry) { return entry; } // // If data supports subsetting: @@ -308,6 +428,7 @@ extend(DataManager.prototype, Cache.prototype, { // /* Disabling for now, more detailed data is never loaded for line tracks + TODO: can using resolution in the key solve this problem? if (this.subset) { var key, split_key, entry_low, entry_high, mode, entry; for (var i = 0; i < this.key_ary.length; i++) { @@ -331,10 +452,79 @@ extend(DataManager.prototype, Cache.prototype, { // Load data from server. The deferred is immediately saved until the // data is ready, it then replaces itself with the actual data - entry = this.load_data(chrom, low, high, mode, resolution, extra_params); - this.set_data( low, high, mode, entry ); + entry = this.load_data(low, high, mode, resolution, extra_params); + this.set_data(low, high, mode, entry); return entry }, + /** "Deep" data request; used as a parameter for DataManager.get_more_data() */ + DEEP_DATA_REQ: "deep", + /** "Broad" data request; used as a parameter for DataManager.get_more_data() */ + BROAD_DATA_REQ: "breadth", + /** + * Gets more data for a region using either a depth-first or a breadth-first approach. + */ + get_more_data: function(low, high, mode, resolution, extra_params, req_type) { + // + // Get current data from cache and mark as stale. + // + var cur_data = this.get_data_from_cache(low, high, mode); + if (!cur_data) { + console.log("ERROR: no current data for: ", this.track, low, high, mode, resolution, extra_params); + return; + } + cur_data.stale = true; + + // + // Set parameters based on request type. + // + var query_low = low; + if (req_type === this.DEEP_DATA_REQ) { + // Use same interval but set start_val to skip data that's already in cur_data. + $.extend(extra_params, {start_val: cur_data.data.length + 1}); + } + else if (req_type === this.BROAD_DATA_REQ) { + // To get past an area of extreme feature depth, set query low to be after either + // (a) the maximum high or HACK/FIXME (b) the end of the last feature returned. + query_low = (cur_data.max_high ? cur_data.max_high : cur_data.data[cur_data.data.length - 1][2]) + 1; + } + + // + // Get additional data, append to current data, and set new data. Use a custom deferred object + // to signal when new data is available. + // + var + data_manager = this, + new_data_request = this.load_data(query_low, high, mode, resolution, extra_params) + new_data_available = $.Deferred(); + // load_data sets cache to new_data_request, but use custom deferred object so that signal and data + // is all data, not just new data. + this.set_data(low, high, mode, new_data_available); + $.when(new_data_request).then(function(result) { + // Update data and message. + if (result.data) { + result.data = cur_data.data.concat(result.data); + if (result.max_low) { + result.max_low = cur_data.max_low; + } + if (result.message) { + // HACK: replace number in message with current data length. Works but is ugly. + result.message = result.message.replace(/[0-9]+/, result.data.length); + } + } + data_manager.set_data(low, high, mode, result); + new_data_available.resolve(result); + }); + return new_data_available; + }, + /** + * Gets data from the cache. + */ + get_data_from_cache: function(low, high, mode) { + return this.get(this.gen_key(low, high, mode)); + }, + /** + * Sets data in the cache. + */ set_data: function(low, high, mode, result) { //console.log("set_data", low, high, mode, result); return this.set(this.gen_key(low, high, mode), result); @@ -342,6 +532,10 @@ extend(DataManager.prototype, Cache.prototype, { /** * Generate key for cache. */ + // TODO: use chrom in key so that (a) data is not thrown away when changing chroms and (b) + // manager does not need to be cleared when changing chroms. + // TODO: use resolution in key b/c summary tree data is dependent on resolution -- is this + // necessary, i.e. will resolution change but not low/high/mode? gen_key: function(low, high, mode) { var key = low + "_" + high + "_" + mode; return key; @@ -368,45 +562,281 @@ extend(ReferenceTrackDataManager.prototype, DataManager.prototype, Cache.prototy } }); +/** + * Drawables hierarchy: + * + * Drawable + * --> DrawableCollection + * --> DrawableGroup + * --> View + * --> Track + */ + +/** + * Base interface for all drawable objects. Drawable objects have a name and are + * associated with a view and container. They optionally have a drag handle class. + */ +var Drawable = function(name, view, container, prefs, drag_handle_class) { + this.name = name; + this.view = view; + this.container = container; + this.drag_handle_class = drag_handle_class; + this.config = new DrawableConfig({ + track: this, + params: [ + { key: 'name', label: 'Name', type: 'text', default_value: name } + ], + saved_values: prefs, + onchange: function() { + this.track.set_name(this.track.config.values.name); + } + }); + this.prefs = this.config.values; +}; + +extend(Drawable.prototype, { + init: function() {}, + request_draw: function() {}, + _draw: function() {}, + to_json: function() {}, + make_name_popup_menu: function() {}, + /** + * Set drawable name. + */ + set_name: function(new_name) { + this.old_name = this.name; + this.name = new_name; + this.name_div.text(this.name); + }, + /** + * Revert track name; currently name can be reverted only once. + */ + revert_name: function() { + this.name = this.old_name; + this.name_div.text(this.name); + }, + /** + * Remove drawable (a) from its container and (b) from the HTML. + */ + remove: function() { + this.container.remove_drawable(this); + + this.container_div.fadeOut('slow', function() { + $(this).remove(); + // HACK: is there a better way to update the view? + view.update_intro_div(); + view.has_changes = true; + }); + } +}); + +/** + * A collection of drawable objects. + */ +var DrawableCollection = function(obj_type, name, view, container, prefs, drag_handle_class) { + Drawable.call(this, name, view, container, prefs, drag_handle_class); + + // Attribute init. + this.obj_type = obj_type; + this.drawables = []; +}; +extend(DrawableCollection.prototype, Drawable.prototype, { + /** + * Init each drawable in the collection. + */ + init: function() { + for (var i = 0; i < this.drawables.length; i++) { + this.drawables[i].init(); + } + }, + /** + * Draw each drawable in the collection. + */ + _draw: function() { + for (var i = 0; i < this.drawables.length; i++) { + this.drawables[i]._draw(); + } + }, + /** + * Returns jsonified representation of collection. + */ + to_json: function() { + var jsonified_drawables = []; + for (var i = 0; i < this.drawables.length; i++) { + jsonified_drawables.push(this.drawables[i].to_json()); + } + return { + name: this.name, + prefs: this.prefs, + obj_type: this.obj_type, + drawables: jsonified_drawables + }; + }, + /** + * Add a drawable to the end of the collection. + */ + add_drawable: function(drawable) { + this.drawables.push(drawable); + drawable.container = this; + }, + /** + * Add a drawable before another drawable. + */ + add_drawable_before: function(drawable, other) { + var index = this.drawables.indexOf(other); + if (index != -1) { + this.drawables.splice(index, 0, drawable); + return true; + } + return false; + }, + /** + * Remove drawable from this collection. + */ + remove_drawable: function(drawable) { + var index = this.drawables.indexOf(drawable); + if (index != -1) { + // Found drawable to remove. + this.drawables.splice(index, 1); + drawable.container = null; + return true; + } + return false; + }, + /** + * Move drawable to another location in collection. + */ + move_drawable: function(drawable, new_position) { + var index = this.drawables.indexOf(drawable); + if (index != -1) { + // Remove from current position: + this.drawables.splice(index, 1); + // insert into new position: + this.drawables.splice(new_position, 0, drawable); + return true; + } + return false; + } +}); + +/** + * A group of drawables that are moveable, visible. + */ +var DrawableGroup = function(name, view, container, prefs) { + DrawableCollection.call(this, "DrawableGroup", name, view, container, prefs, "group-handle"); + + // HTML elements. + if (!DrawableGroup.id_counter) { DrawableGroup.id_counter = 0; } + var group_id = DrawableGroup.id_counter++ + this.container_div = $("
    ").addClass("group").attr("id", "group_" + group_id).appendTo(this.container.content_div); + this.header_div = $("
    ").addClass("track-header").appendTo(this.container_div); + this.header_div.append($("
    ").addClass(this.drag_handle_class)); + this.name_div = $("
    ").addClass("group-name menubutton popup").text(this.name).appendTo(this.header_div); + this.content_div = $("
    ").addClass("content-div").attr("id", "group_" + group_id + "_content_div").appendTo(this.container_div); + + // Set up containers/moving for group: register both container_div and content div as container + // because both are used as containers (container div to recognize container, content_div to + // store elements). Group can be moved. + is_container(this.container_div, this); + is_container(this.content_div, this); + moveable(this.container_div, this.drag_handle_class, ".group", this); + + this.make_name_popup_menu(); +}; + +extend(DrawableGroup.prototype, Drawable.prototype, DrawableCollection.prototype, { + /** + * Make popup menu for group. + */ + make_name_popup_menu: function() { + var group = this; + + var group_dropdown = {}; + + // + // Edit config option. + // + group_dropdown["Edit configuration"] = function() { + var cancel_fn = function() { hide_modal(); $(window).unbind("keypress.check_enter_esc"); }, + ok_fn = function() { + group.config.update_from_form( $(".dialog-box") ); + hide_modal(); + $(window).unbind("keypress.check_enter_esc"); + }, + check_enter_esc = function(e) { + if ((e.keyCode || e.which) === 27) { // Escape key + cancel_fn(); + } else if ((e.keyCode || e.which) === 13) { // Enter key + ok_fn(); + } + }; + + $(window).bind("keypress.check_enter_esc", check_enter_esc); + show_modal("Configure Group", group.config.build_form(), { + "Cancel": cancel_fn, + "OK": ok_fn + }); + }; + + // + // Remove option. + // + group_dropdown.Remove = function() { + group.remove(); + }; + + make_popupmenu(group.name_div, group_dropdown); + } +}); + /** * View object manages complete viz view, including tracks and user interactions. */ -var View = function( container, title, vis_id, dbkey, callback ) { +var View = function(container, title, vis_id, dbkey) { + DrawableCollection.call(this, "View"); this.container = container; this.chrom = null; this.vis_id = vis_id; this.dbkey = dbkey; this.title = title; - this.tracks = []; + // Alias tracks to point at drawables. TODO: changes tracks to 'drawables' or something similar. + this.tracks = this.drawables; this.label_tracks = []; + this.tracks_to_be_redrawn = []; this.max_low = 0; this.max_high = 0; - this.num_tracks = 0; - this.track_id_counter = 0; this.zoom_factor = 3; this.min_separation = 30; this.has_changes = false; - this.init( callback ); + // Deferred object that indicates when view's chrom data has been loaded. + this.load_chroms_deferred = null; + this.init(); this.canvas_manager = new CanvasManager( container.get(0).ownerDocument ); this.reset(); }; -extend( View.prototype, { - init: function( callback ) { +extend( View.prototype, DrawableCollection.prototype, { + init: function() { // Create DOM elements var parent_element = this.container, view = this; // Top container for things that are fixed at the top this.top_container = $("
    ").addClass("top-container").appendTo(parent_element); - // Content container, primary tracks are contained in here - this.content_div = $("
    ").addClass("content").css("position", "relative").appendTo(parent_element); + // Browser content, primary tracks are contained in here + this.browser_content_div = $("
    ").addClass("content").css("position", "relative").appendTo(parent_element); // Bottom container for things that are fixed at the bottom this.bottom_container = $("
    ").addClass("bottom-container").appendTo(parent_element); // Label track fixed at top this.top_labeltrack = $("
    ").addClass("top-labeltrack").appendTo(this.top_container); // Viewport for dragging tracks in center - this.viewport_container = $("
    ").addClass("viewport-container").addClass("viewport-container").appendTo(this.content_div); - // Future overlay? - this.intro_div = $("
    ").addClass("intro").text("Select a chrom from the dropdown below").hide(); + this.viewport_container = $("
    ").addClass("viewport-container").attr("id", "viewport-container").appendTo(this.browser_content_div); + // Alias viewport_container as content_div so that it matches function of DrawableCollection/Group content_div. + this.content_div = this.viewport_container; + is_container(this.viewport_container, view); + // Introduction div shown when there are no tracks. + this.intro_div = $("
    ").addClass("intro"); + var add_tracks_button = $("
    ").text("Add Datasets to Visualization").addClass("action-button").appendTo(this.intro_div).click(function () { + add_tracks(); + }); // Another label track at bottom this.nav_labeltrack = $("
    ").addClass("nav-labeltrack").appendTo(this.bottom_container); // Navigation at top @@ -436,7 +866,7 @@ extend( View.prototype, { }; this.nav_input = $("").addClass("nav-input").hide().bind("keyup focusout", submit_nav).appendTo(this.nav_controls); this.location_span = $("").addClass("location").appendTo(this.nav_controls); - this.location_span.bind("click", function() { + this.location_span.click(function() { view.location_span.hide(); view.chrom_select.hide(); view.nav_input.val(view.chrom + ":" + view.low + "-" + view.high); @@ -447,18 +877,17 @@ extend( View.prototype, { if (this.vis_id !== undefined) { this.hidden_input = $("").attr("type", "hidden").val(this.vis_id).appendTo(this.nav_controls); } - this.zo_link = $("").click(function() { view.zoom_out(); view.redraw(); }).appendTo(this.nav_controls); - this.zi_link = $("").click(function() { view.zoom_in(); view.redraw(); }).appendTo(this.nav_controls); + this.zo_link = $("").click(function() { view.zoom_out(); view.request_redraw(); }).appendTo(this.nav_controls); + this.zi_link = $("").click(function() { view.zoom_in(); view.request_redraw(); }).appendTo(this.nav_controls); // Get initial set of chroms. - this.load_chroms({low: 0}, callback); + this.load_chroms_deferred = this.load_chroms({low: 0}); this.chrom_select.bind("change", function() { view.change_chrom(view.chrom_select.val()); }); - this.intro_div.show(); /* - this.content_div.bind("mousewheel", function( e, delta ) { + this.browser_content_div.bind("mousewheel", function( e, delta ) { if (Math.abs(delta) < 0.5) { return; } @@ -472,12 +901,12 @@ extend( View.prototype, { */ // Blur tool/filter inputs when user clicks on content div. - this.content_div.bind("click", function( e ) { + this.browser_content_div.click(function( e ) { $(this).find("input").trigger("blur"); }); // Double clicking zooms in - this.content_div.bind("dblclick", function( e ) { + this.browser_content_div.bind("dblclick", function( e ) { view.zoom_in(e.pageX, this.viewport_container); }); @@ -491,14 +920,8 @@ extend( View.prototype, { view.move_delta(-delta_chrom); }); - this.overview_close.bind("click", function() { - for (var track_id = 0, len = view.tracks.length; track_id < len; track_id++) { - view.tracks[track_id].is_overview = false; - } - $(this).siblings().filter("canvas").remove(); - $(this).parent().css("height", view.overview_box.height()); - view.overview_highlight.hide(); - $(this).hide(); + this.overview_close.click(function() { + view.reset_overview(); }); // Dragging in the viewport scrolls @@ -534,7 +957,7 @@ extend( View.prototype, { // to zoom in this.top_labeltrack.bind( "dragstart", function( e, d ) { return $("
    ").css( { - "height": view.content_div.height() + view.top_labeltrack.height() + "height": view.browser_content_div.height() + view.top_labeltrack.height() + view.nav_labeltrack.height() + 1, "top": "0px", "position": "absolute", @@ -559,26 +982,41 @@ extend( View.prototype, { view.low = Math.round(min / width * span) + old_low; view.high = Math.round(max / width * span) + old_low; $(d.proxy).remove(); - view.redraw(); + view.request_redraw(); }); - this.add_label_track( new LabelTrack( this, this.top_labeltrack ) ); - this.add_label_track( new LabelTrack( this, this.nav_labeltrack ) ); + this.add_label_track( new LabelTrack( this, { content_div: this.top_labeltrack } ) ); + this.add_label_track( new LabelTrack( this, { content_div: this.nav_labeltrack } ) ); $(window).bind("resize", function() { view.resize_window(); }); $(document).bind("redraw", function() { view.redraw(); }); this.reset(); $(window).trigger("resize"); + this.update_intro_div(); + }, + /** Add or remove intro div depending on view state. */ + update_intro_div: function() { + if (this.num_tracks === 0) { + this.intro_div.appendTo(this.viewport_container); + } + else { + this.intro_div.remove(); + } }, update_location: function(low, high) { this.location_span.text( commatize(low) + ' - ' + commatize(high) ); this.nav_input.val( this.chrom + ':' + commatize(low) + '-' + commatize(high) ); }, - load_chroms: function(url_parms, callback) { + /** + * Load chrom data for the view. Returns a jQuery Deferred. + */ + load_chroms: function(url_parms) { url_parms['num'] = MAX_CHROMS_SELECTABLE; $.extend( url_parms, (this.vis_id !== undefined ? { vis_id: this.vis_id } : { dbkey: this.dbkey } ) ); - var view = this; + var + view = this, + chrom_data = $.Deferred(); $.ajax({ url: chrom_url, data: url_parms, @@ -607,16 +1045,16 @@ extend( View.prototype, { chrom_options += ''; } view.chrom_select.html(chrom_options); - if ( callback ) { - callback(); - } view.chrom_start_index = result.start_index; + + chrom_data.resolve(result); }, error: function() { alert("Could not load chroms for this dbkey:", view.dbkey); } }); + return chrom_data; }, change_chrom: function(chrom, low, high) { // Don't do anything if chrom is "None" (hackish but some browsers already have this set), or null/blank @@ -654,16 +1092,10 @@ extend( View.prototype, { // Switching to local chrom. if (chrom !== view.chrom) { view.chrom = chrom; - if (!view.chrom) { - // No chrom selected - view.intro_div.show(); - } else { - view.intro_div.hide(); - } view.chrom_select.val(view.chrom); view.max_high = found.len-1; // -1 because we're using 0-based indexing. view.reset(); - view.redraw(true); + view.request_redraw(true); for (var track_id = 0, len = view.tracks.length; track_id < len; track_id++) { var track = view.tracks[track_id]; @@ -677,7 +1109,7 @@ extend( View.prototype, { view.high = Math.min(high, view.max_high); } view.reset_overview(); - view.redraw(); + view.request_redraw(); } }, go_to: function(str) { @@ -718,36 +1150,88 @@ extend( View.prototype, { view.high -= delta_chrom; view.low -= delta_chrom; } - view.redraw(); + view.request_redraw(); }, - add_track: function(track) { - track.view = this; - track.track_id = this.track_id_counter; - this.tracks.push(track); - if (track.init) { track.init(); } - track.container_div.attr('id', 'track_' + track.track_id); - sortable( track.container_div, '.draghandle' ); - this.track_id_counter += 1; - this.num_tracks += 1; + /** + * Add a drawable to the view. + */ + add_drawable: function(drawable) { + DrawableCollection.prototype.add_drawable.call(this, drawable); + drawable.init(); + this.has_changes = true; + this.update_intro_div(); }, add_label_track: function (label_track) { label_track.view = this; + label_track.init(); this.label_tracks.push(label_track); }, - remove_track: function(track) { - this.has_changes = true; - track.container_div.fadeOut('slow', function() { $(this).remove(); }); - delete this.tracks[this.tracks.indexOf(track)]; - this.num_tracks -= 1; + /** + * Remove drawable from the view. + */ + remove_drawable: function(drawable, hide) { + DrawableCollection.prototype.remove_drawable.call(this, drawable); + if (hide) { + var view = this; + drawable.container_div.fadeOut('slow', function() { + $(this).remove(); + view.update_intro_div(); + }); + this.has_changes = true; + } }, reset: function() { this.low = this.max_low; this.high = this.max_high; this.viewport_container.find(".yaxislabel").remove(); - }, - redraw: function(nodraw) { - var span = this.high - this.low, - low = this.low, + }, + /** + * Request that view redraw some or all tracks. If a track is not specificied, redraw all tracks. + */ + request_redraw: function(nodraw, force, clear_after, track) { + var + view = this, + // Either redrawing a single track or all view's tracks. + track_list = (track ? [track] : view.tracks), + track_index; + + // Add/update tracks in track list to redraw list. + var track; + for (var i = 0; i < track_list.length; i++) { + track = track_list[i]; + + // Because list elements are arrays, need to look for track index manually. + track_index = -1; + for (var j = 0; j < view.tracks_to_be_redrawn.length; j++) { + if (view.tracks_to_be_redrawn[j][0] === track) { + track_index = j; + break; + } + } + + // Add track to list or update draw parameters. + if (track_index < 0) { + // Track not in list yet. + view.tracks_to_be_redrawn.push([track, force, clear_after]); + } + else { + // Track already in list; update force and clear_after. + view.tracks_to_be_redrawn[i][1] = force; + view.tracks_to_be_redrawn[i][2] = clear_after; + } + } + + // Set up redraw. + requestAnimationFrame(function() { view._redraw(nodraw) }); + }, + /** + * Redraws view and tracks. + * NOTE: this method should never be called directly; request_redraw() should be used so + * that requestAnimationFrame can manage redrawing. + */ + _redraw: function(nodraw) { + + var low = this.low, high = this.high; if (low < this.max_low) { @@ -756,6 +1240,7 @@ extend( View.prototype, { if (high > this.max_high) { high = this.max_high; } + var span = this.high - this.low; if (this.high !== 0 && span < this.min_separation) { high = low + this.min_separation; } @@ -781,13 +1266,18 @@ extend( View.prototype, { this.update_location(this.low, this.high); if (!nodraw) { - for (var i = 0, len = this.tracks.length; i < len; i++) { - if (this.tracks[i] && this.tracks[i].enabled) { - this.tracks[i].draw(); + var track, force, clear_after; + for (var i = 0, len = this.tracks_to_be_redrawn.length; i < len; i++) { + track = this.tracks_to_be_redrawn[i][0]; + force = this.tracks_to_be_redrawn[i][1]; + clear_after = this.tracks_to_be_redrawn[i][2]; + if (track) { + track._draw(force, clear_after); } } + this.tracks_to_be_redrawn = []; for (i = 0, len = this.label_tracks.length; i < len; i++) { - this.label_tracks[i].draw(); + this.label_tracks[i]._draw(); } } }, @@ -803,7 +1293,7 @@ extend( View.prototype, { } this.low = Math.round(cur_center - new_half); this.high = Math.round(cur_center + new_half); - this.redraw(); + this.request_redraw(); }, zoom_out: function () { if (this.max_high === 0) { @@ -814,19 +1304,49 @@ extend( View.prototype, { new_half = (span * this.zoom_factor) / 2; this.low = Math.round(cur_center - new_half); this.high = Math.round(cur_center + new_half); - this.redraw(); + this.request_redraw(); }, resize_window: function() { this.viewport_container.height( this.container.height() - this.top_container.height() - this.bottom_container.height() ); this.nav_container.width( this.container.width() ); - this.redraw(); + this.request_redraw(); }, + /** Show a track in the overview. */ + set_overview: function(track) { + // Get data and draw tile. + $.when(track.get_overview_tile()).then(function(tile) { + // Update UI. + view.overview_viewport.find(".track-tile").remove(); + view.overview_close.show(); + view.overview_viewport.append(tile.canvas); + view.overview_highlight.show().height(tile.canvas.height()); + view.overview_viewport.height(tile.canvas.height() + view.overview_box.outerHeight()); + view.resize_window(); + + // Update view, track states. + if (view.overview_track) { + view.overview_track.set_is_overview(false); + } + view.overview_track = track; + track.set_is_overview(true); + }); + view.has_changes = true; + }, + /** Close and reset overview. */ reset_overview: function() { - this.overview_viewport.find("canvas").remove(); + // Update UI. + this.overview_viewport.find(".track-tile").remove(); this.overview_viewport.height(this.default_overview_height); this.overview_box.height(this.default_overview_height); this.overview_close.hide(); this.overview_highlight.hide(); + view.resize_window(); + + // Update view, track states. + if (view.overview_track) { + view.overview_track.set_is_overview(false); + } + view.overview_track = null; } }); @@ -867,7 +1387,7 @@ var Tool = function(track, tool_dict) { // Disable dragging, clicking, double clicking on div so that actions on slider do not impact viz. this.parent_div.bind("drag", function(e) { e.stopPropagation(); - }).bind("click", function(e) { + }).click(function(e) { e.stopPropagation(); }).bind("dblclick", function(e) { e.stopPropagation(); @@ -892,12 +1412,13 @@ var Tool = function(track, tool_dict) { // Highlight value for inputs for easy replacement. this.parent_div.find("input").click(function() { $(this).select() }); - // Add 'Go' button. + // Add buttons for running on dataset, region. var run_tool_row = $("
    ").addClass("param-row").appendTo(this.parent_div); var run_on_dataset_button = $("").attr("value", "Run on complete dataset").appendTo(run_tool_row); var run_on_region_button = $("").attr("value", "Run on visible region").css("margin-left", "3em").appendTo(run_tool_row); var tool = this; run_on_region_button.click( function() { + // Run tool to create new track. tool.run_on_region(); }); run_on_dataset_button.click( function() { @@ -942,6 +1463,7 @@ extend(Tool.prototype, { dataset_id: this.track.original_dataset_id, tool_id: tool.name }, + null, // Success callback. function(track_data) { show_modal(tool.name + " is Running", @@ -972,19 +1494,33 @@ extend(Tool.prototype, { // Set name of track to include tool name, parameters, and region used. track_name = url_params.tool_id + current_track.tool_region_and_parameters_str(url_params.chrom, url_params.low, url_params.high), + container, new_track; - // Create and add track. - // TODO: add support for other kinds of tool data tracks. - if (current_track.track_type === 'FeatureTrack') { - new_track = new ToolDataFeatureTrack(track_name, view, current_track.hda_ldda, undefined, {}, {}, current_track); + // If track not in a group, create a group for it and add new track to group. If track + // already in group, add track to group. + if (current_track.container === view) { + var group = new DrawableGroup(this.name, this.track.view, this.track.container); + current_track.container.add_drawable(group); + // TODO: this is ugly way to move track from one container to another -- make this easier via + // a Drawable or DrawableCollection function. + current_track.container.remove_drawable(current_track); + group.add_drawable(current_track); + current_track.container_div.appendTo(group.content_div); + container = group; + } + else { + container = current_track.container; + } + if (current_track instanceof FeatureTrack) { + new_track = new ToolDataFeatureTrack(track_name, view, container, "hda"); new_track.change_mode(current_track.mode); + container.add_drawable(new_track); } - this.track.add_track(new_track); new_track.content_div.text("Starting job."); // Run tool. - this.run(url_params, + this.run(url_params, new_track, // Success callback. function(track_data) { new_track.dataset_id = track_data.dataset_id; @@ -996,7 +1532,7 @@ extend(Tool.prototype, { /** * Run tool using a set of URL params and a success callback. */ - run: function(url_params, success_callback) { + run: function(url_params, new_track, success_callback) { // Add tool params to URL params. $.extend(url_params, this.get_param_values_dict()); @@ -1017,7 +1553,7 @@ extend(Tool.prototype, { else if (response === "pending") { // Converting/indexing input datasets; show message and try again. new_track.container_div.addClass("pending"); - new_track.content_div.text("Converting input data so that it can be easily reused."); + new_track.content_div.text("Converting input data so that it can be used quickly with tool."); setTimeout(json_run_tool, 2000); } else { @@ -1057,6 +1593,7 @@ var Filter = function(name, index, tool_id, tool_exp_name) { // Name to use for filter when building expression for tool. this.tool_exp_name = tool_exp_name; }; + /** * Number filters have a min, max as well as a low, high; low and high are used */ @@ -1068,7 +1605,8 @@ var NumberFilter = function(name, index, tool_id, tool_exp_name) { // Slide min/max. These values are used to set/update slider. this.min = Number.MAX_VALUE; this.max = -Number.MAX_VALUE; - // UI Slider element and label that is associated with filter. + // UI elements associated with filter. + this.container = null; this.slider = null; this.slider_label = null; }; @@ -1091,7 +1629,7 @@ extend(NumberFilter.prototype, { // No element to filter on. return true; } - var val = parseInt(element[this.index]); + var val = parseFloat(element[this.index]); return (isNaN(val) || (val >= this.low && val <= this.high)); }, /** @@ -1118,6 +1656,14 @@ extend(NumberFilter.prototype, { * Update filter's slider. */ update_ui_elt: function () { + // Only show filter if min != max because filter is not useful otherwise. + if (this.min != this.max) { + this.container.show(); + } + else { + this.container.hide(); + } + var get_slider_step = function(min, max) { var range = max - min; return (range <= 2 ? 0.01 : 1); @@ -1230,30 +1776,37 @@ var FiltersManager = function(track, filters_list) { }); }; - // Create filtering div. + // + // Create parent div. + // this.parent_div = $("
    ").addClass("filters").hide(); // Disable dragging, double clicking, keys on div so that actions on slider do not impact viz. this.parent_div.bind("drag", function(e) { e.stopPropagation(); - }).bind("click", function(e) { + }).click(function(e) { e.stopPropagation(); }).bind("dblclick", function(e) { e.stopPropagation(); }).bind("keydown", function(e) { e.stopPropagation(); }); + + // + // Create sliders div. + // + var sliders_div = $("
    ").addClass("sliders").appendTo(this.parent_div); var manager = this; $.each(this.filters, function(index, filter) { - var filter_div = $("
    ").addClass("slider-row").appendTo(manager.parent_div); + filter.container = $("
    ").addClass("slider-row").appendTo(sliders_div); // Set up filter label (name, values). - var filter_label = $("
    ").addClass("slider-label").appendTo(filter_div) + var filter_label = $("
    ").addClass("elt-label").appendTo(filter.container) var name_span = $("").addClass("slider-name").text(filter.name + " ").appendTo(filter_label); var values_span = $(""); var values_span_container = $("").addClass("slider-value").appendTo(filter_label).append("[").append(values_span).append("]"); // Set up slider for filter. - var slider_div = $("
    ").addClass("slider").appendTo(filter_div); + var slider_div = $("
    ").addClass("slider").appendTo(filter.container); filter.control_element = $("
    ").attr("id", filter.name + "-filter-control").appendTo(slider_div); var prev_values = [0,0]; filter.control_element.slider({ @@ -1262,24 +1815,14 @@ var FiltersManager = function(track, filters_list) { max: -Number.MIN_VALUE, values: [0, 0], slide: function(event, ui) { - // - // Always update UI values, but set timeout for doing more--especially drawing-- - // so that viz is more responsive. - // - prev_values = ui.values; - values_span.text(ui.values[0] + "-" + ui.values[1]); - setTimeout(function() { - if (ui.values[0] == prev_values[0] && ui.values[1] == prev_values[1]) { - var values = ui.values; - // Set new values in UI. - values_span.text(values[0] + "-" + values[1]); - // Set new values in filter. - filter.low = values[0]; - filter.high = values[1]; - // Redraw track. - manager.track.draw(true, true); - } - }, 50); + var values = ui.values; + // Set new values in UI. + values_span.text(values[0] + "-" + values[1]); + // Set new values in filter. + filter.low = values[0]; + filter.high = values[1]; + // Redraw track. + manager.track.request_draw(true, true); }, change: function(event, ui) { filter.control_element.slider("option", "slide").call(filter.control_element, event, ui); @@ -1292,29 +1835,56 @@ var FiltersManager = function(track, filters_list) { edit_slider_values(values_span_container, values_span, filter.control_element); // Add to clear floating layout. - $("
    ").appendTo(filter_div); + $("
    ").appendTo(filter.container); }); // Add button to filter complete dataset. - if (this.filters.length != 0) { - var run_buttons_row = $("
    ").addClass("param-row").appendTo(this.parent_div); - var run_on_dataset_button = $("").attr("value", "Run on complete dataset").appendTo(run_buttons_row); + if (this.filters.length !== 0) { + var run_buttons_row = $("
    ").addClass("param-row").appendTo(sliders_div); + var run_on_dataset_button = $("").attr("value", "Run on complete dataset").appendTo(run_buttons_row); var filter_manager = this; run_on_dataset_button.click( function() { filter_manager.run_on_dataset(); }); } + + // + // Create filtering display controls. + // + var + display_controls_div = $("
    ").addClass("display-controls").appendTo(this.parent_div), + header_text = $("").addClass("elt-label").text("Transparency:").appendTo(display_controls_div), + alpha_select = $("').attr("id", id ).attr("name", id ).attr( 'checked', track_config.values[ param.key ] ) ); + row.append( $('').attr("id", id ).attr("name", id ).attr( 'checked', value ) ); + } else if ( param.type === 'text' ) { + row.append( $('').attr("id", id ).val(value).click( function() { $(this).select() })); } else if ( param.type === 'color' ) { - var value = track_config.values[ param.key ]; var input = $('').attr("id", id ).attr("name", id ).val( value ); // Color picker in tool tip style float var tip = $( "
    " ).hide(); @@ -1457,8 +2046,9 @@ extend( TrackConfig.prototype, { }); e.stopPropagation(); }); - } else { - row.append( $('').attr("id", id ).attr("name", id ).val( track_config.values[ param.key ] ) ); + } + else { + row.append( $('').attr("id", id ).attr("name", id ).val( value ) ); } } }); @@ -1478,7 +2068,7 @@ extend( TrackConfig.prototype, { value = parseInt( value ); } else if ( param.type === 'bool' ) { value = container.find( '#' + id ).is( ':checked' ); - } + } // Save value only if changed if ( value !== track_config.values[ param.key ] ) { track_config.values[ param.key ] = value; @@ -1495,20 +2085,37 @@ extend( TrackConfig.prototype, { /** * Tiles drawn by tracks. */ -var Tile = function(index, resolution, canvas) { +var Tile = function(index, resolution, canvas, data) { this.index = index; + this.low = index * DENSITY * resolution; + this.high = (index + 1) * DENSITY * resolution; this.resolution = resolution; // Wrap element in div for background. this.canvas = $("
    ").append(canvas); + this.data = data; + this.stale = false; }; -var SummaryTreeTile = function(index, resolution, canvas, max_val) { - Tile.call(this, index, resolution, canvas); +var SummaryTreeTile = function(index, resolution, canvas, data, max_val) { + Tile.call(this, index, resolution, canvas, data); this.max_val = max_val; }; -var FeatureTrackTile = function(index, resolution, canvas) { - Tile.call(this, index, resolution, canvas); +var FeatureTrackTile = function(index, resolution, canvas, data, message, feature_mapper) { + Tile.call(this, index, resolution, canvas, data); + this.message = message; + this.feature_mapper = feature_mapper; + + // + // Set up display of feature data on mouseover. + // + var tile = this; + $(this.canvas).mousemove(function (e) { + var feature_data = tile.feature_mapper.get_feature_data(e.offsetX, e.offsetY); + + // TODO: show popup with feature's information. + }); + }; /** @@ -1523,14 +2130,15 @@ var FeatureTrackTile = function(index, resolution, canvas) { * ----> FeatureTrack * -------> ReadTrack * -------> ToolDataFeatureTrack + * -------> VcfTrack */ -var Track = function(name, view, parent_element, data_url, data_query_wait) { +var Track = function(name, view, container, prefs, data_url, data_query_wait) { + // For now, track's container is always view. + Drawable.call(this, name, view, container, {}, "draghandle"); + // // Attribute init. // - this.name = name; - this.view = view; - this.parent_element = parent_element; this.data_url = (data_url ? data_url : default_data_url); this.data_url_extra_params = {} this.data_query_wait = (data_query_wait ? data_query_wait : DEFAULT_DATA_QUERY_WAIT); @@ -1539,10 +2147,11 @@ var Track = function(name, view, parent_element, data_url, data_query_wait) { // // Create HTML element structure for track. // - this.container_div = $("
    ").addClass('track').css("position", "relative"); + if (!Track.id_counter) { Track.id_counter = 0; } + this.container_div = $("
    ").addClass('track').attr("id", "track_" + Track.id_counter++).css("position", "relative"); if (!this.hidden) { this.header_div = $("
    ").appendTo(this.container_div); - if (this.view.editor) { this.drag_div = $("
    ").appendTo(this.header_div); } + if (this.view.editor) { this.drag_div = $("
    ").addClass(this.drag_handle_class).appendTo(this.header_div); } this.name_div = $("
    ").appendTo(this.header_div); this.name_div.text(this.name); this.name_div.attr( "id", this.name.replace(/\s+/g,'-').replace(/[^a-zA-Z0-9\-]/g,'').toLowerCase() ); @@ -1552,9 +2161,35 @@ var Track = function(name, view, parent_element, data_url, data_query_wait) { // Create content div, which is where track is displayed. // this.content_div = $("
    ").appendTo(this.container_div); - this.parent_element.append(this.container_div); + this.container.content_div.append(this.container_div); }; -extend(Track.prototype, { +extend(Track.prototype, Drawable.prototype, { + /** Returns track type. */ + get_type: function() { + // Order is important: start with most-specific classes and go up the track hierarchy. + if (this instanceof LabelTrack) { + return "LabelTrack"; + } + else if (this instanceof ReferenceTrack) { + return "ReferenceTrack"; + } + else if (this instanceof LineTrack) { + return "LineTrack"; + } + else if (this instanceof ReadTrack) { + return "ReadTrack"; + } + else if (this instanceof ToolDataFeatureTrack) { + return "ToolDataFeatureTrack"; + } + else if (this instanceof VcfTrack) { + return "VcfTrack"; + } + else if (this instanceof FeatureTrack) { + return "FeatureTrack"; + } + return ""; + }, /** * Initialize and draw the track. */ @@ -1562,7 +2197,7 @@ extend(Track.prototype, { var track = this; track.enabled = false; track.tile_cache.clear(); - track.data_cache.clear(); + track.data_manager.clear(); track.initial_canvas = undefined; track.content_div.css("height", "auto"); /* @@ -1574,6 +2209,7 @@ extend(Track.prototype, { // // Tracks with no dataset id are handled differently. + // FIXME: is this really necessary? // if (!track.dataset_id) { return; @@ -1588,7 +2224,7 @@ extend(Track.prototype, { track.content_div.text(DATA_ERROR); if (result.message) { var track_id = track.view.tracks.indexOf(track); - var error_link = $(" ").text("View error").bind("click", function() { + var error_link = $(" ").text("View error").click(function() { show_modal( "Trackster Error", "
    " + result.message + "
    ", { "Close" : hide_modal } ); }); track.content_div.append(error_link); @@ -1616,7 +2252,7 @@ extend(Track.prototype, { // predraw_init may be asynchronous, wait for it and then draw $.when(track.predraw_init()).done(function() { track.container_div.removeClass("nodata error pending"); - track.draw() + track.request_draw(); }); } } @@ -1626,58 +2262,25 @@ extend(Track.prototype, { * Additional initialization required before drawing track for the first time. */ predraw_init: function() {}, - // Provide support for updating and reverting track name. Currently, it's only possible to revert once. - update_name: function(new_name) { - this.old_name = this.name; - this.name = new_name; - this.name_div.text(this.name); - }, - revert_name: function() { - this.name = this.old_name; - this.name_div.text(this.name); - } }); -var TiledTrack = function(filters_list, tool_dict, parent_track) { +var TiledTrack = function(filters_list, tool_dict) { var track = this, view = track.view; + + // Make track moveable. + moveable(track.container_div, track.drag_handle_class, ".group", track); // Attribute init. - this.filters_manager = (filters_list !== undefined ? new FiltersManager(this, filters_list) : undefined); + this.filters_manager = new FiltersManager( this, (filters_list !== undefined ? filters_list : {}) ); // filters_available is determined by data, filters_visible is set by user. this.filters_available = false; this.filters_visible = false; this.tool = (tool_dict !== undefined && obj_length(tool_dict) > 0 ? new Tool(this, tool_dict) : undefined); - - // - // TODO: Right now there is only the notion of a parent track and multiple child tracks. However, - // a more general notion of a 'track group' is probably needed and can be easily created using - // the code already in place for parent/child tracks. The view would then manage track groups, and - // each track group could be managed on its own. - // - this.parent_track = parent_track; - this.child_tracks = []; + this.is_overview = false; if (track.hidden) { return; } - - // - // If track has parent: - // -replace drag handle with child-track icon button; (TODO: eventually, we'll want to be able - // to make a set of child tracks dragable.) - // -remove tool b/c child tracks cannot have tools. - // - if (this.parent_track) { - this.header_div.find(".draghandle").removeClass('draghandle').addClass('child-track-icon').addClass('icon-button'); - this.parent_element.addClass("child-track"); - this.tool = undefined; - } - - // - // Child tracks container setup. - // - track.child_tracks_container = $("
    ").addClass("child-tracks-container").hide(); - track.container_div.append(track.child_tracks_container); - + // // Create filters div. // @@ -1700,8 +2303,8 @@ var TiledTrack = function(filters_list, tool_dict, parent_track) { if (track.display_modes !== undefined) { if (track.mode_div === undefined) { track.mode_div = $("
    ").appendTo(track.header_div); - var init_mode = (track.track_config && track.track_config.values['mode'] ? - track.track_config.values['mode'] : track.display_modes[0]); + var init_mode = (track.config && track.config.values['mode'] ? + track.config.values['mode'] : track.display_modes[0]); track.mode = init_mode; track.mode_div.text(init_mode); @@ -1719,26 +2322,21 @@ var TiledTrack = function(filters_list, tool_dict, parent_track) { } this.make_name_popup_menu(); - - /* - if (track.overview_check_div === undefined) { - track.overview_check_div = $("
    ").css("margin-top", "-3px").appendTo(track.header_div); - track.overview_check = $("").appendTo(track.overview_check_div); - track.overview_check.bind("click", function() { - var curr = this; - view.overview_viewport.find("canvas").remove(); - track.set_overview(); - $(".overview_check").each(function() { - if (this !== curr) { - $(this).attr("checked", false); - } - }); - }); - track.overview_check_div.append( $("