From 952fdac419e9b4123c158fd088f90a9a09bc7fd3 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 6 Sep 2018 12:10:34 +0100 Subject: [PATCH 001/260] Remove unused and broken tool --- tools/data_source/bed_convert.xml | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 tools/data_source/bed_convert.xml diff --git a/tools/data_source/bed_convert.xml b/tools/data_source/bed_convert.xml deleted file mode 100644 index eb414066926..00000000000 --- a/tools/data_source/bed_convert.xml +++ /dev/null @@ -1,14 +0,0 @@ - - creates a bed or xbed file containing from text query - noop - - creates a bed or xbed file containing user assigned input of $input - - - - - - - User specifies delimiter, header information, and column assignments and the file will be converted to BED or xBED. - - \ No newline at end of file From f865e8359fa6c8d599011edcea61a2a728afb9ea Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 10 Sep 2018 11:09:57 +0100 Subject: [PATCH 002/260] Deprecate `size` attribute of `` and remove it from tools Also: - dos2unix test/functional/tools/for_workflows/head.xml - Single-quote text and data params in `` - Remove deprecated `interpreter` attribute of `` --- config/job_resource_params_conf.xml.sample | 6 +- .../workflow_resource_params_conf.xml.sample | 6 +- lib/galaxy/tools/parameters/validation.py | 8 +- lib/galaxy/tools/xsd/galaxy.xsd | 34 ++++---- test/functional/tools/for_workflows/head.xml | 26 +++---- test/unit/tools/test_wrappers.py | 2 +- tools/data_source/genbank.xml | 4 +- tools/data_source/upload.xml | 37 ++++----- tools/extract/liftOver_wrapper.xml | 8 +- tools/filters/CreateInterval.xml | 18 ++--- tools/filters/bed_to_bigbed.xml | 4 +- tools/filters/changeCase.xml | 12 +-- tools/filters/fileGrep.xml | 17 ++-- tools/filters/fixedValueColumn.xml | 16 ++-- tools/filters/gff/gff_filter_by_attribute.xml | 14 ++-- .../gff/gff_filter_by_feature_count.xml | 15 ++-- tools/filters/grep.xml | 2 +- tools/filters/headWrapper.xml | 12 +-- tools/filters/randomlines.xml | 10 +-- tools/filters/remove_beginning.xml | 14 ++-- tools/filters/tailWrapper.xml | 8 +- tools/filters/wig_to_bigwig.xml | 4 +- tools/maf/vcf_to_maf_customtrack.xml | 2 +- tools/metag_tools/blat_wrapper.xml | 6 +- tools/metag_tools/shrimp_color_wrapper.xml | 34 ++++---- tools/metag_tools/shrimp_wrapper.xml | 32 ++++---- tools/plotting/bar_chart.xml | 26 ++++--- tools/plotting/boxplot.xml | 6 +- tools/solid_tools/maq_cs_wrapper.xml | 77 +++++++++---------- tools/sr_mapping/PerM.xml | 12 +-- tools/sr_mapping/bfast_wrapper.xml | 2 +- tools/stats/filtering.xml | 2 +- tools/stats/gsummary.xml | 16 ++-- tools/stats/gsummary.xml.groups | 20 +++-- 34 files changed, 244 insertions(+), 268 deletions(-) diff --git a/config/job_resource_params_conf.xml.sample b/config/job_resource_params_conf.xml.sample index 9a201c8f4bb..c9adce5cb97 100644 --- a/config/job_resource_params_conf.xml.sample +++ b/config/job_resource_params_conf.xml.sample @@ -1,6 +1,6 @@ - - - + + + diff --git a/config/workflow_resource_params_conf.xml.sample b/config/workflow_resource_params_conf.xml.sample index c04d2909aa5..0495c2443ed 100644 --- a/config/workflow_resource_params_conf.xml.sample +++ b/config/workflow_resource_params_conf.xml.sample @@ -1,7 +1,7 @@ - - - + + + - - + + @@ -17,13 +17,13 @@ python '$__tool_directory__/random_lines_two_pass.py' '${input}' '${out_file1}' - + - + @@ -41,7 +41,6 @@ python '$__tool_directory__/random_lines_two_pass.py' '${input}' '${out_file1}' - **What it does** This tool selects N random lines from a file, with no repeats, and preserving ordering. @@ -62,6 +61,5 @@ Selecting 2 random lines might return this:: chr7 56736 56756 D17003_CTCF_R7 354 + chr7 56775 56795 D17003_CTCF_R4 207 + - diff --git a/tools/filters/remove_beginning.xml b/tools/filters/remove_beginning.xml index a929e483d83..2a1023f710b 100644 --- a/tools/filters/remove_beginning.xml +++ b/tools/filters/remove_beginning.xml @@ -1,12 +1,14 @@ of a file - remove_beginning.pl $input $num_lines $out_file1 + +perl '$__tool_directory__/remove_beginning.pl' '$input' $num_lines '$out_file1' + - - + + - + @@ -16,7 +18,6 @@ - **What it does** This tool removes a specified number of lines from the beginning of a dataset. @@ -37,6 +38,5 @@ After removing the first 3 lines the dataset will look like this:: chr7 56772 56792 D17003_CTCF_R7 372 + chr7 56775 56795 D17003_CTCF_R4 207 + - - + diff --git a/tools/filters/tailWrapper.xml b/tools/filters/tailWrapper.xml index 0865145dc88..166018ec1d2 100644 --- a/tools/filters/tailWrapper.xml +++ b/tools/filters/tailWrapper.xml @@ -3,11 +3,11 @@ perl '$__tool_directory__/tailWrapper.pl' '$input' $lineNum '$out_file1' - - + + - + @@ -17,7 +17,6 @@ perl '$__tool_directory__/tailWrapper.pl' '$input' $lineNum '$out_file1' - **What it does** This tool outputs specified number of lines from the **end** of a dataset @@ -38,6 +37,5 @@ This tool outputs specified number of lines from the **end** of a dataset chr7 57341 57361 D17003_CTCF_R7 375 + chr7 57457 57477 D17003_CTCF_R3 188 + - diff --git a/tools/filters/wig_to_bigwig.xml b/tools/filters/wig_to_bigwig.xml index 7db6785af6a..5e99e922722 100644 --- a/tools/filters/wig_to_bigwig.xml +++ b/tools/filters/wig_to_bigwig.xml @@ -31,8 +31,8 @@ - - + + diff --git a/tools/maf/vcf_to_maf_customtrack.xml b/tools/maf/vcf_to_maf_customtrack.xml index 3fbe06513cf..c10618ac41e 100644 --- a/tools/maf/vcf_to_maf_customtrack.xml +++ b/tools/maf/vcf_to_maf_customtrack.xml @@ -20,7 +20,7 @@ ${vcf_source_type.vcf_source} -n '$track_name' -g - + diff --git a/tools/metag_tools/blat_wrapper.xml b/tools/metag_tools/blat_wrapper.xml index f018f1a7b39..1171c524a61 100644 --- a/tools/metag_tools/blat_wrapper.xml +++ b/tools/metag_tools/blat_wrapper.xml @@ -19,9 +19,9 @@ - - - + + + diff --git a/tools/metag_tools/shrimp_color_wrapper.xml b/tools/metag_tools/shrimp_color_wrapper.xml index 55e72415659..c2b4379bf9d 100644 --- a/tools/metag_tools/shrimp_color_wrapper.xml +++ b/tools/metag_tools/shrimp_color_wrapper.xml @@ -19,23 +19,23 @@ python '$__tool_directory__/shrimp_color_wrapper.py' '$input_target' '$input_que - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + diff --git a/tools/metag_tools/shrimp_wrapper.xml b/tools/metag_tools/shrimp_wrapper.xml index f88180c5dc7..d4c974cf41a 100644 --- a/tools/metag_tools/shrimp_wrapper.xml +++ b/tools/metag_tools/shrimp_wrapper.xml @@ -25,7 +25,7 @@ python '$__tool_directory__/shrimp_wrapper.py' '$input_target' '$output1' '$outp - + @@ -38,21 +38,21 @@ python '$__tool_directory__/shrimp_wrapper.py' '$input_target' '$output1' '$outp - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + diff --git a/tools/plotting/bar_chart.xml b/tools/plotting/bar_chart.xml index 7d2855d0445..b9447a49a35 100644 --- a/tools/plotting/bar_chart.xml +++ b/tools/plotting/bar_chart.xml @@ -1,9 +1,13 @@ for multiple columns - - #if $xtic.userSpecified == "Yes" #bar_chart.py $input $xtic.xticColumn $colList "$title" "$ylabel" $ymin $ymax $out_file1 "$pdf_size" - #else #bar_chart.py $input 0 $colList "$title" "$ylabel" $ymin $ymax $out_file1 "$pdf_size" - #end if + +python '$__tool_directory__/bar_chart.py' '$input' +#if $xtic.userSpecified == "Yes" + $xtic.xticColumn +#else + 0 +#end if +$colList '$title' '$ylabel' $ymin $ymax '$out_file1' '$pdf_size' @@ -18,11 +22,11 @@ - - - - - + + + + + @@ -33,7 +37,7 @@ - + gnuplot-py @@ -53,5 +57,5 @@ Graphing columns 2 and 3 while using column 1 for X Tick Labels will produce the .. image:: ${static_path}/images/bar_chart.png :height: 324 :width: 540 - + diff --git a/tools/plotting/boxplot.xml b/tools/plotting/boxplot.xml index 415e02e2f8c..9c59980ec23 100644 --- a/tools/plotting/boxplot.xml +++ b/tools/plotting/boxplot.xml @@ -6,10 +6,10 @@ - + - - + + diff --git a/tools/solid_tools/maq_cs_wrapper.xml b/tools/solid_tools/maq_cs_wrapper.xml index 8fe3fb7e68e..a3d3ec04cbf 100644 --- a/tools/solid_tools/maq_cs_wrapper.xml +++ b/tools/solid_tools/maq_cs_wrapper.xml @@ -1,56 +1,53 @@ - - maq_cs_wrapper.py - $output1 - $output2 - $ref - $library_type.f3_reads - $library_type.f3_qual - $library_type.is_paired - #if $library_type.is_paired == "yes": - $library_type.r3_reads - $library_type.r3_qual - #else: - "None" - "None" - #end if - $min_mapqual - $max_mismatch - $output3 - + +python '$__tool_directory__/maq_cs_wrapper.py' +'$output1' +'$output2' +'$ref' +'$library_type.f3_reads' +'$library_type.f3_qual' +$library_type.is_paired +#if $library_type.is_paired == "yes": + '$library_type.r3_reads' + '$library_type.r3_qual' +#else: + "None" + "None" +#end if +$min_mapqual +$max_mismatch +'$output3' - + - - + + - - - - + + + + - - + + - - - + + + - - .. class:: infomark **What it does** -This tool maps SOLiD color-space reads against the target genome using MAQ. It produces three output datasets: +This tool maps SOLiD color-space reads against the target genome using MAQ. It produces three output datasets: -**ALIGNMENT INFO** : contains the read alignment information, +**ALIGNMENT INFO** : contains the read alignment information, **PILEUP** : contains the coverage and SNP statistics for every nucleotide of the target genome, -**CUSTOM TRACK** : contains the coverage and SNP statistics as custom tracks displayable in the UCSC browser. +**CUSTOM TRACK** : contains the coverage and SNP statistics as custom tracks displayable in the UCSC browser. ----- @@ -113,8 +108,6 @@ This tool maps SOLiD color-space reads against the target genome using MAQ. It p * column 7 = number of Ts * column 8 = number of Gs * column 9 = number of Cs - - - - + + diff --git a/tools/sr_mapping/PerM.xml b/tools/sr_mapping/PerM.xml index d0b84ea11fe..1df2dcd570d 100644 --- a/tools/sr_mapping/PerM.xml +++ b/tools/sr_mapping/PerM.xml @@ -84,8 +84,8 @@ - - + + @@ -116,14 +116,14 @@ - - + + - + @@ -147,7 +147,7 @@ - + diff --git a/tools/sr_mapping/bfast_wrapper.xml b/tools/sr_mapping/bfast_wrapper.xml index 4c28721990d..c901969d7e8 100644 --- a/tools/sr_mapping/bfast_wrapper.xml +++ b/tools/sr_mapping/bfast_wrapper.xml @@ -72,7 +72,7 @@ - + diff --git a/tools/stats/filtering.xml b/tools/stats/filtering.xml index 460479e664f..1a20cf40c94 100644 --- a/tools/stats/filtering.xml +++ b/tools/stats/filtering.xml @@ -11,7 +11,7 @@ - + diff --git a/tools/stats/gsummary.xml b/tools/stats/gsummary.xml index f7168d2400b..b314c8671e2 100644 --- a/tools/stats/gsummary.xml +++ b/tools/stats/gsummary.xml @@ -9,25 +9,26 @@ - python $__tool_directory__/gsummary.py "$input" "$out_file1" "$cond" + +python '$__tool_directory__/gsummary.py' '$input' '$out_file1' '$cond' + - - + + - + - + - .. class:: warningmark This tool expects input datasets consisting of tab-delimited columns (blank or comment lines beginning with a # character are automatically skipped). @@ -77,6 +78,5 @@ This tool computes basic summary statistics on a given column, or on a valid exp #sum mean stdev 0% 25% 50% 75% 100% 29250.000 7312.500 7198.636 1700.000 1895.000 5280.000 10697.500 16990.000 - - + diff --git a/tools/stats/gsummary.xml.groups b/tools/stats/gsummary.xml.groups index 218ab31aa38..b043ed5682e 100644 --- a/tools/stats/gsummary.xml.groups +++ b/tools/stats/gsummary.xml.groups @@ -1,17 +1,17 @@ of a column in a tab delimited file according to an expression - gsummary.py $input $out_file1 "$cond" "$groups" + +python '$__tool_directory__/gsummary.py' '$input' '$out_file1' '$cond' '$groups' + - - - - + + + - + - .. class:: warningmark This tool expects input datasets to consist of tab-delimited columns (blank or comment lines beginning with a # character are automatically skipped). @@ -33,7 +33,6 @@ This tool computes basic summary statistics on a given column, or on an expressi + **c1** *group by the values in column 1* + **c1,c4** *group by the values in column 1, then by the values in column 4* - ----- **Expression examples** @@ -52,11 +51,10 @@ This tool computes basic summary statistics on a given column, or on an expressi .. class:: infomark -**TIP:** Most functions (like *abs*) take only a single expression. *log* can take one or two parameters, like *log(expression,base)* +**TIP:** Most functions (like *abs*) take only a single expression. *log* can take one or two parameters, like *log(expression,base)* Currently, these R functions are supported: *abs, sign, sqrt, floor, ceiling, trunc, round, signif, exp, log, cos, sin, tan, acos, asin, atan, cosh, sinh, tanh, acosh, asinh, atanh, lgamma, gamma, gammaCody, digamma, trigamma, cumsum, cumprod, cummax, cummin* .. |INFO| image:: ./static/images/icon_info_sml.gif - - + From 7b027d8a1fbed0b5471f774fe3d3a3cab123093f Mon Sep 17 00:00:00 2001 From: M Bernt Date: Wed, 8 Aug 2018 13:36:27 +0200 Subject: [PATCH 003/260] improved grouping - replaces non 'floatable' values to a user supplied default value - do not strip lines because this removes empty cells at the ends - added some comments to python code --- tools/stats/grouping.py | 37 +++++++++++++++++++++++++++++++++---- tools/stats/grouping.xml | 7 +++---- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/tools/stats/grouping.py b/tools/stats/grouping.py index a003219217e..ad9038de1b4 100644 --- a/tools/stats/grouping.py +++ b/tools/stats/grouping.py @@ -3,6 +3,13 @@ # Refactored 2011 to use numpy instead of rpy, Kanwei Li """ This tool provides the SQL "group by" functionality. +Arguments: + 1 output file name + 2 input file name + 3 grouping column + 4 ignore case (1/0) + 5 ascii to delete (comma separated list) + 6... op,col,do_round,default """ from __future__ import print_function @@ -15,6 +22,22 @@ from itertools import groupby import numpy +def float_wdefault(s, d, c): + """ + convert list of strings s into list of floats + non convertable entries are replaced by d if d is not None (otherwise error) + """ + for i in range(len(s)): + try: + s[i] = float(s[i]) + except ValueError: + if d is not None: + s[i] = d + else: + stop_err("non float value '%s' found in colum %d" % (s[i], c)) + return s + + def stop_err(msg): sys.stderr.write(msg) sys.exit() @@ -38,7 +61,9 @@ def main(): ops = [] cols = [] round_val = [] + default_val = [] + # remove comment lines if sys.argv[5] != "None": asciitodelete = sys.argv[5] if asciitodelete: @@ -55,16 +80,20 @@ def main(): newfile.close() inputfile = newinputfile + # get operations and options in separate arrays for var in sys.argv[6:]: - op, col, do_round = var.split() + op, col, do_round, default = var.split(',') ops.append(op) cols.append(col) round_val.append(do_round) + default_val.append(float(default) if default != '' else None) + """ At this point, ops, cols and rounds will look something like this: ops: ['mean', 'min', 'c'] cols: ['1', '3', '4'] round_val: ['no', 'yes' 'no'] + default_val: [0, 1, None] """ try: @@ -72,8 +101,8 @@ def main(): except Exception: stop_err("Group column not specified.") + # sort file into a temporary file tmpfile = tempfile.NamedTemporaryFile(mode='r') - try: """ The -k option for the Posix sort command is as follows: @@ -112,7 +141,7 @@ def main(): out_str = key for line in line_list: - fields = line.strip().split("\t") + fields = line.split("\t") for i, col in enumerate(cols): col = int(col) - 1 # cXX from galaxy is 1-based try: @@ -141,7 +170,7 @@ def main(): else: # some kind of numpy fn try: - data = [float(_) for _ in data] + data = float_wdefault(data, default_val[i], col + 1) except ValueError: sys.stderr.write("Operation %s expected number values but got %s instead.\n" % (op, data)) sys.exit(1) diff --git a/tools/stats/grouping.xml b/tools/stats/grouping.xml index a4e646d648c..c0947769e57 100644 --- a/tools/stats/grouping.xml +++ b/tools/stats/grouping.xml @@ -1,4 +1,4 @@ - + data by a column and perform aggregate operation on other columns. grouping.py @@ -8,9 +8,7 @@ "${ignorecase}" "${ignorelines}" #for $op in $operations - '${op.optype} - ${op.opcol} - ${op.opround}' + '${op.optype},${op.opcol},${op.opround},${op.opdefault}' #end for @@ -57,6 +55,7 @@ + From 0c145b3326f2476b05ed8ca7ee66d754099a452f Mon Sep 17 00:00:00 2001 From: thobalose Date: Mon, 16 Jul 2018 14:01:34 +0200 Subject: [PATCH 004/260] Updated Neo4j IE to 3.1.9 --- .../neo/config/allowed_images.yml.sample | 2 +- .../interactive_environments/neo/config/neo.ini.sample | 4 ++-- .../plugins/interactive_environments/neo/templates/neo.mako | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/plugins/interactive_environments/neo/config/allowed_images.yml.sample b/config/plugins/interactive_environments/neo/config/allowed_images.yml.sample index b63b71624b8..86e84dd3ca5 100644 --- a/config/plugins/interactive_environments/neo/config/allowed_images.yml.sample +++ b/config/plugins/interactive_environments/neo/config/allowed_images.yml.sample @@ -6,6 +6,6 @@ # appropriate `apt-get/pip install` statements. --- - - image: quay.io/sanbi-sa/neo_ie:3.1 + image: quay.io/sanbi-sa/neo_ie:3.1.9 description: | Neo4j is a highly scalable, robust native graph database. diff --git a/config/plugins/interactive_environments/neo/config/neo.ini.sample b/config/plugins/interactive_environments/neo/config/neo.ini.sample index 019b5456ad8..0e84731e033 100644 --- a/config/plugins/interactive_environments/neo/config/neo.ini.sample +++ b/config/plugins/interactive_environments/neo/config/neo.ini.sample @@ -48,8 +48,8 @@ command_inject = --sig-proxy=true -e DEBUG=false -e DEFAULT_CONTAINER_RUNTIME=12 # To run containers in Docker Swarm mode on (an existing swarm), set the # following option to True *and*: -# - set docker_connect_port above. For thoba/neo4j_galaxy_ie the port -# should most likely be 7474. +# - set docker_connect_port above. For quay.io/sanbi-sa/neo_ie:3.1.9 the port +# should most likely be 80. # - If command_inject is uncommented and includes `--sig-proxy`, that option should # be removed. #swarm_mode = False diff --git a/config/plugins/interactive_environments/neo/templates/neo.mako b/config/plugins/interactive_environments/neo/templates/neo.mako index 0ed4be71b35..c1df7807375 100644 --- a/config/plugins/interactive_environments/neo/templates/neo.mako +++ b/config/plugins/interactive_environments/neo/templates/neo.mako @@ -7,7 +7,7 @@ # if the user knows ahead of time that they will need it. import os mount_path = str(os.path.dirname(hda.file_name)) + '/dataset_{}_files'.format( hda.dataset.id ) - data_vol = ie_request.volume(mount_path, '/data', how='rw') + data_vol = ie_request.volume(mount_path, '/data', mode='rw') # data_vol = ie_request.volume('${HOME}/neo4j/data', '/data/', how='rw') # Add all environment variables collected from Galaxy's IE infrastructure # Launch the IE. From 2da38803a3bc12a19e1fa53a1e8cc9e9448eb1b6 Mon Sep 17 00:00:00 2001 From: thobalose Date: Mon, 16 Jul 2018 14:03:06 +0200 Subject: [PATCH 005/260] Catering for shlex.split(cmd_inject) behaviour --- .../plugins/interactive_environments.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/visualization/plugins/interactive_environments.py b/lib/galaxy/visualization/plugins/interactive_environments.py index 099e710ab50..d3131cba7ba 100644 --- a/lib/galaxy/visualization/plugins/interactive_environments.py +++ b/lib/galaxy/visualization/plugins/interactive_environments.py @@ -294,7 +294,22 @@ class InteractiveEnvironmentRequest(object): def _flag_opts(flag, opts): return [arg for pair in product((flag,), opts) for arg in pair] + def _check_uid_and_gid(cmd_inject): + """ + Check and replace shell uid and gid using os + :param cmd_inject: + """ + # --user="$(id -u):$(id -g)" + # https://docs.docker.com/engine/reference/run/#user + # -e USER_UID=$(id -u) -e USER_GID=$(id -g) + uid_gid_subs = {"$(id -u)": "{}".format(os.geteuid()), "$(id -g)": "{}".format(os.getgid())} + subs = sorted(uid_gid_subs) + regex = re.compile('|'.join(map(re.escape, subs))) + return regex.sub(lambda match: uid_gid_subs[match.group(0)], cmd_inject) + command_inject = self.attr.viz_config.get("docker", "command_inject") + command_inject = _check_uid_and_gid(command_inject) + # --name should really not be set, but we'll try to honor it anyway name = ['--name=%s' % self._get_name_for_run()] if '--name' not in command_inject else [] env = self._get_env_for_run(env_override) @@ -429,7 +444,7 @@ class InteractiveEnvironmentRequest(object): host_port = self._find_port_mapping(port_mappings)[-1] log.debug("Container host/port: %s:%s", self.attr.docker_hostname, host_port) - # Now we configure our proxy_requst object and we manually specify + # Now we configure our proxy_request object and we manually specify # the port to map to and ensure the proxy is available. self.attr.proxy_request = self.trans.app.proxy_manager.setup_proxy( self.trans, From 3e57fb172fbf580da04131355326681e118cc82b Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 31 Oct 2018 13:42:16 -0400 Subject: [PATCH 006/260] Quick 'make everything private' implementation. Takes a single history or operates on all of a user's histories. Intentionally doesn't use an API, rather a simple webcontroller endpoint, because I think we'll want to handle this with group operations and the like down the road, and will enhance existing APIs --- .../scripts/mvc/history/options-menu.js | 24 ++++++++++++- .../scripts/mvc/user/user-preferences.js | 33 +++++++++++++++++ .../webapps/galaxy/controllers/history.py | 35 +++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) diff --git a/client/galaxy/scripts/mvc/history/options-menu.js b/client/galaxy/scripts/mvc/history/options-menu.js index 58a14849310..b44266532cc 100644 --- a/client/galaxy/scripts/mvc/history/options-menu.js +++ b/client/galaxy/scripts/mvc/history/options-menu.js @@ -92,7 +92,29 @@ var menu = [ } } }, - + { + html: _l("Make Data Private"), + anon: true, + func: function() { + if ( + Galaxy && + Galaxy.currHistoryPanel && + confirm( + _l( + "This will make all the data in this history private, and will set permissions such that all new data is created as private. Any datasets within that are currently shared will need to be re-shared or published. Are you sure you want to do this?" + ) + ) + ) { + $.post( + `${Galaxy.root}history/make_private`, + { history_id: "${Galaxy.currHistoryPanel.model.id}" }, + () => { + Galaxy.currHistoryPanel.loadCurrentHistory(); + } + ); + } + } + }, { html: _l("Dataset Actions"), header: true, diff --git a/client/galaxy/scripts/mvc/user/user-preferences.js b/client/galaxy/scripts/mvc/user/user-preferences.js index 0c31d8a53ca..6de0130097e 100644 --- a/client/galaxy/scripts/mvc/user/user-preferences.js +++ b/client/galaxy/scripts/mvc/user/user-preferences.js @@ -41,6 +41,38 @@ var Model = Backbone.Model.extend({ submit_title: "Save permissions", redirect: "user" }, + make_data_private: { + title: _l("Make all data private"), + description: _l("Click here to make all data private."), + icon: "fa-lock", + onclick: function() { + if ( + confirm( + _l( + "WARNING: This will make *all* datasets for which you have " + + "'management' permissions, in all of your histories " + + "private, and will set permissions such that all " + + "of your new data in these histories is created as private. Any " + + "datasets within that are currently shared will need " + + "to be re-shared or published. Are you sure you " + + "want to do this?" + ) + ) + ) { + $.post(`${Galaxy.root}history/make_private`, { all_histories: true }, () => { + Galaxy.modal.show({ + title: _l("Datasets are now private"), + body: "All of your histories and datsets have been made private.", + buttons: { + Close: function() { + Galaxy.modal.hide(); + } + } + }); + }); + } + } + }, api_key: { title: _l("Manage API key"), description: _l("Access your current API key or create a new one."), @@ -130,6 +162,7 @@ var View = Backbone.View.extend({ } self._addLink("custom_builds"); self._addLink("permissions"); + self._addLink("make_data_private"); self._addLink("api_key"); if (config.has_user_tool_filters) { self._addLink("toolbox_filters"); diff --git a/lib/galaxy/webapps/galaxy/controllers/history.py b/lib/galaxy/webapps/galaxy/controllers/history.py index f376c31b471..c5d62c0f776 100644 --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -694,6 +694,41 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt trans.app.security_agent.history_set_default_permissions(history, permissions) return {'message': 'Default history \'%s\' dataset permissions have been changed.' % history.name} + @web.expose_api + @web.require_login("make datasets private") + def make_private(self, trans, history_id=None, all_histories=False, **kwd): + """ + Sets the datasets within a history to private. Also sets the default + permissions for the history to private, for future datasets. + """ + histories = [] + if all_histories: + histories = trans.user.histories + elif history_id: + history = [self.history_manager.get_owned(self.decode_id(history_id), trans.user, current_history=trans.history)] + if history: + histories.append(history) + if not histories: + return self.message_exception(trans, 'Invalid history or histories specified.') + private_role = trans.app.security_agent.get_private_user_role(trans.user) + user_roles = trans.user.all_roles() + private_permissions = { + trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS: [private_role], + trans.app.security_agent.permitted_actions.DATASET_ACCESS: [private_role], + } + for history in histories: + # Set default role for history to private + trans.app.security_agent.history_set_default_permissions(history, private_permissions) + # Set private role for all datasets + for hda in history.datasets: + if trans.app.security_agent.dataset_is_public(hda.dataset): + if trans.app.security_agent.can_manage_dataset(user_roles, hda.dataset): + if not trans.app.security_agent.dataset_is_private_to_user(trans, hda.dataset): + trans.app.security_agent.set_all_dataset_permissions(hda.dataset, private_permissions) + if not trans.app.security_agent.dataset_is_private_to_user(trans, hda.dataset): + raise exceptions.InternalServerError('An error occured and the dataset is NOT private.') + return {'message': 'History \'%s\' dataset permissions have been changed.' % history.name} + @web.expose @web.require_login("share histories with other users") def share(self, trans, id=None, email="", **kwd): From a07a1c3a566571b9b83dc7a99fae1ed0d275f020 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 31 Oct 2018 13:46:02 -0400 Subject: [PATCH 007/260] Make logic more concise, fix typo, add commentary. --- lib/galaxy/webapps/galaxy/controllers/history.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/history.py b/lib/galaxy/webapps/galaxy/controllers/history.py index c5d62c0f776..0cea62b21b6 100644 --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -721,12 +721,11 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt trans.app.security_agent.history_set_default_permissions(history, private_permissions) # Set private role for all datasets for hda in history.datasets: - if trans.app.security_agent.dataset_is_public(hda.dataset): - if trans.app.security_agent.can_manage_dataset(user_roles, hda.dataset): - if not trans.app.security_agent.dataset_is_private_to_user(trans, hda.dataset): - trans.app.security_agent.set_all_dataset_permissions(hda.dataset, private_permissions) - if not trans.app.security_agent.dataset_is_private_to_user(trans, hda.dataset): - raise exceptions.InternalServerError('An error occured and the dataset is NOT private.') + if (not trans.app.security_agent.dataset_is_private_to_user(hda.dataset) and trans.app.security_agent.can_manage_dataset(user_roles, hda.dataset)): + # If it's not private to me, and I can manage it, set fixed private permissions. + trans.app.security_agent.set_all_dataset_permissions(hda.dataset, private_permissions) + if not trans.app.security_agent.dataset_is_private_to_user(trans, hda.dataset): + raise exceptions.InternalServerError('An error occurred and the dataset is NOT private.') return {'message': 'History \'%s\' dataset permissions have been changed.' % history.name} @web.expose From 43689c1f3fc8e2fc49c3d6dfc033f70fe06e4c32 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 31 Oct 2018 13:50:48 -0400 Subject: [PATCH 008/260] Update message to reflect all histories functionality. --- lib/galaxy/webapps/galaxy/controllers/history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/history.py b/lib/galaxy/webapps/galaxy/controllers/history.py index 0cea62b21b6..f14ef19ef4b 100644 --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -726,7 +726,7 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt trans.app.security_agent.set_all_dataset_permissions(hda.dataset, private_permissions) if not trans.app.security_agent.dataset_is_private_to_user(trans, hda.dataset): raise exceptions.InternalServerError('An error occurred and the dataset is NOT private.') - return {'message': 'History \'%s\' dataset permissions have been changed.' % history.name} + return {'message': 'Success, requested permissions have been changed in %s.' % ("all histories" if all_histories else history.name)} @web.expose @web.require_login("share histories with other users") From a3fd7084e2f02a9e9bc5383cd7f76532450774e9 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Fri, 9 Nov 2018 13:25:18 -0500 Subject: [PATCH 009/260] Exclude library associated datasets from history make-all-private functionality. --- lib/galaxy/webapps/galaxy/controllers/history.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/history.py b/lib/galaxy/webapps/galaxy/controllers/history.py index f14ef19ef4b..edab65bb149 100644 --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -721,7 +721,9 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt trans.app.security_agent.history_set_default_permissions(history, private_permissions) # Set private role for all datasets for hda in history.datasets: - if (not trans.app.security_agent.dataset_is_private_to_user(hda.dataset) and trans.app.security_agent.can_manage_dataset(user_roles, hda.dataset)): + if (not hda.dataset.library_associations + and not trans.app.security_agent.dataset_is_private_to_user(hda.dataset) + and trans.app.security_agent.can_manage_dataset(user_roles, hda.dataset)): # If it's not private to me, and I can manage it, set fixed private permissions. trans.app.security_agent.set_all_dataset_permissions(hda.dataset, private_permissions) if not trans.app.security_agent.dataset_is_private_to_user(trans, hda.dataset): From cc53b07253538e192b8196dee300aefaf8e2fde5 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Fri, 9 Nov 2018 13:31:58 -0500 Subject: [PATCH 010/260] Update messaging to indicate library datasets are not included in the privacy pass. --- client/galaxy/scripts/mvc/history/options-menu.js | 2 +- client/galaxy/scripts/mvc/user/user-preferences.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/galaxy/scripts/mvc/history/options-menu.js b/client/galaxy/scripts/mvc/history/options-menu.js index b44266532cc..8d7483b8d2d 100644 --- a/client/galaxy/scripts/mvc/history/options-menu.js +++ b/client/galaxy/scripts/mvc/history/options-menu.js @@ -101,7 +101,7 @@ var menu = [ Galaxy.currHistoryPanel && confirm( _l( - "This will make all the data in this history private, and will set permissions such that all new data is created as private. Any datasets within that are currently shared will need to be re-shared or published. Are you sure you want to do this?" + "This will make all the data in this history private (excluding library datasets), and will set permissions such that all new data is created as private. Any datasets within that are currently shared will need to be re-shared or published. Are you sure you want to do this?" ) ) ) { diff --git a/client/galaxy/scripts/mvc/user/user-preferences.js b/client/galaxy/scripts/mvc/user/user-preferences.js index 6de0130097e..4fd9b0cad85 100644 --- a/client/galaxy/scripts/mvc/user/user-preferences.js +++ b/client/galaxy/scripts/mvc/user/user-preferences.js @@ -49,7 +49,7 @@ var Model = Backbone.Model.extend({ if ( confirm( _l( - "WARNING: This will make *all* datasets for which you have " + + "WARNING: This will make all datasets (excluding library datasets) for which you have " + "'management' permissions, in all of your histories " + "private, and will set permissions such that all " + "of your new data in these histories is created as private. Any " + From 5d38e558f27b2ec14088376ec1a07c3bf4b25d08 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Fri, 9 Nov 2018 13:50:12 -0500 Subject: [PATCH 011/260] Resolve two issues from refactoring. --- client/galaxy/scripts/mvc/history/options-menu.js | 10 +++------- lib/galaxy/webapps/galaxy/controllers/history.py | 4 ++-- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/client/galaxy/scripts/mvc/history/options-menu.js b/client/galaxy/scripts/mvc/history/options-menu.js index 8d7483b8d2d..fffd81ca950 100644 --- a/client/galaxy/scripts/mvc/history/options-menu.js +++ b/client/galaxy/scripts/mvc/history/options-menu.js @@ -105,13 +105,9 @@ var menu = [ ) ) ) { - $.post( - `${Galaxy.root}history/make_private`, - { history_id: "${Galaxy.currHistoryPanel.model.id}" }, - () => { - Galaxy.currHistoryPanel.loadCurrentHistory(); - } - ); + $.post(`${Galaxy.root}history/make_private`, { history_id: Galaxy.currHistoryPanel.model.id }, () => { + Galaxy.currHistoryPanel.loadCurrentHistory(); + }); } } }, diff --git a/lib/galaxy/webapps/galaxy/controllers/history.py b/lib/galaxy/webapps/galaxy/controllers/history.py index edab65bb149..3a3716d05a0 100644 --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -705,7 +705,7 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt if all_histories: histories = trans.user.histories elif history_id: - history = [self.history_manager.get_owned(self.decode_id(history_id), trans.user, current_history=trans.history)] + history = self.history_manager.get_owned(self.decode_id(history_id), trans.user, current_history=trans.history) if history: histories.append(history) if not histories: @@ -722,7 +722,7 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt # Set private role for all datasets for hda in history.datasets: if (not hda.dataset.library_associations - and not trans.app.security_agent.dataset_is_private_to_user(hda.dataset) + and not trans.app.security_agent.dataset_is_private_to_user(trans, hda.dataset) and trans.app.security_agent.can_manage_dataset(user_roles, hda.dataset)): # If it's not private to me, and I can manage it, set fixed private permissions. trans.app.security_agent.set_all_dataset_permissions(hda.dataset, private_permissions) From 9b481ba9d7d34e101d54ae1ed0821f6a05b27be0 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Fri, 9 Nov 2018 13:50:50 -0500 Subject: [PATCH 012/260] Client format --- client/galaxy/scripts/apps/analysis.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/client/galaxy/scripts/apps/analysis.js b/client/galaxy/scripts/apps/analysis.js index 6767e658774..bbeea14a332 100644 --- a/client/galaxy/scripts/apps/analysis.js +++ b/client/galaxy/scripts/apps/analysis.js @@ -150,10 +150,9 @@ window.app = function app(options, bootstrapped) { new GridView({ url_base: `${Galaxy.root}workflow/list_published`, active_tab: "shared", - url_data: - { - 'f-username': ( userFilter == null ) ? "" : userFilter - } + url_data: { + "f-username": userFilter == null ? "" : userFilter + } }) ); }, From 53c3785986f68b5d65ec037381568a1092adbccb Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 9 Nov 2018 18:32:42 +0100 Subject: [PATCH 013/260] drmaa: soft catch of external runner fails --- lib/galaxy/jobs/runners/drmaa.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/jobs/runners/drmaa.py b/lib/galaxy/jobs/runners/drmaa.py index dddbef1b17a..a30893cb647 100644 --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -208,6 +208,9 @@ class DRMAAJobRunner(AsynchronousJobRunner): filename = self.store_jobtemplate(job_wrapper, jt) self.userid = pwent[2] external_job_id = self.external_runjob(external_runjob_script, filename, pwent[2]).strip() + if external_job_id is None: + job_wrapper.fail("(%s) could not queue job" % galaxy_id_tag) + return log.info("(%s) queued as %s" % (galaxy_id_tag, external_job_id)) # store runner information for tracking if Galaxy restarts @@ -365,9 +368,9 @@ class DRMAAJobRunner(AsynchronousJobRunner): return filename def external_runjob(self, external_runjob_script, jobtemplate_filename, username): - """ runs an external script the will QSUB a new job. + """ runs an external script that will QSUB a new job. The external script needs to be run with sudo, and will setuid() to the specified user. - Effectively, will QSUB as a different user (then the one used by Galaxy). + Effectively, will QSUB as a different user (than the one used by Galaxy). """ command = shlex.split(external_runjob_script) command.extend([str(username), jobtemplate_filename]) @@ -379,14 +382,14 @@ class DRMAAJobRunner(AsynchronousJobRunner): # os.unlink(jobtemplate_filename) if exitcode != 0: # There was an error in the child process - raise RuntimeError("External_runjob failed (exit code %s)\nChild process reported error:\n%s" % (str(exitcode), stderrdata)) - if not stdoutdata.strip(): - raise RuntimeError("External_runjob did return the job id: %s" % (stdoutdata)) - + log.exception("External_runjob failed (exit code %s). Child process reported error: %s" % (str(exitcode), stderrdata)) + return None # The expected output is a single line containing a single numeric value: # the DRMAA job-ID. If not the case, will throw an error. - jobId = stdoutdata - return jobId + if not stdoutdata.strip(): + log.exception("External_runjob did not returned nothing instead of the job id") + return None + return stdoutdata def _job_name(self, job_wrapper): external_runjob_script = job_wrapper.get_destination_configuration("drmaa_external_runjob_script", None) From 876b460178bfb52936dc9f1d6c22149fea1f2513 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 9 Nov 2018 17:31:13 +0100 Subject: [PATCH 014/260] reclaim ownership for failed jobs --- lib/galaxy/jobs/runners/__init__.py | 1 + test/unit/jobs/test_runner_local.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py index 3c36b7ae18f..dca47d5456c 100644 --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -423,6 +423,7 @@ class BaseJobRunner(object): def fail_job(self, job_state, exception=False): if getattr(job_state, 'stop_job', True): self.stop_job(self.sa_session.query(self.app.model.Job).get(job_state.job_wrapper.job_id)) + job_state.job_wrapper.reclaim_ownership() self._handle_runner_state('failure', job_state) # Not convinced this is the best way to indicate this state, but # something necessary diff --git a/test/unit/jobs/test_runner_local.py b/test/unit/jobs/test_runner_local.py index 2a5955d2389..36d93322fe1 100644 --- a/test/unit/jobs/test_runner_local.py +++ b/test/unit/jobs/test_runner_local.py @@ -217,3 +217,6 @@ class MockJobWrapper(object): def home_directory(self): return None + + def reclaim_ownership(self): + pass From 9a9b815d8fc9810600668ca358f72e0130e37d8f Mon Sep 17 00:00:00 2001 From: Qiang Gu Date: Thu, 15 Nov 2018 11:48:46 -0800 Subject: [PATCH 015/260] add datatype sniff button in edit dataset attributes page --- .../mvc/dataset/dataset-edit-attributes.js | 8 ++++++++ .../webapps/galaxy/controllers/dataset.py | 20 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/client/galaxy/scripts/mvc/dataset/dataset-edit-attributes.js b/client/galaxy/scripts/mvc/dataset/dataset-edit-attributes.js index f81fa6b8a1c..ba95a756415 100644 --- a/client/galaxy/scripts/mvc/dataset/dataset-edit-attributes.js +++ b/client/galaxy/scripts/mvc/dataset/dataset-edit-attributes.js @@ -172,6 +172,14 @@ var View = Backbone.View.extend({ onclick: function() { self._submit("datatype", form); } + }), + submit_datatype_sniff: new Ui.ButtonIcon({ + tooltip: _l("Sniff the datatype and change it."), + title: _l("Sniff datatype"), + icon: "fa-undo", + onclick: function() { + self._submit("datatype_sniff", form); + } }) } }); diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py index 46d5cf204c5..32a4369e8b9 100644 --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -18,6 +18,7 @@ from galaxy import ( web ) from galaxy.datatypes.display_applications.util import decode_dataset_user, encode_dataset_user +from galaxy.datatypes import sniff from galaxy.exceptions import RequestParameterInvalidException from galaxy.model.item_attrs import UsesAnnotations, UsesItemRatings from galaxy.util import ( @@ -25,6 +26,7 @@ from galaxy.util import ( sanitize_text, smart_str ) +from galaxy.util.checkers import check_binary from galaxy.util.sanitize_html import sanitize_html from galaxy.web import form_builder from galaxy.web.base.controller import BaseUIController, ERROR, SUCCESS, url_for, UsesExtendedMetadataMixin @@ -459,6 +461,24 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE message = 'Changed the type to %s.' % datatype else: return self.message_exception(trans, 'You are unable to change datatypes in this manner. Changing %s to %s is not allowed.' % (data.extension, datatype)) + elif operation == 'datatype_sniff': + # The user clicked the Sniff button on the 'Change data type' form + if data.datatype.allow_datatype_change: + # prevent modifying datatype when dataset is queued or running as input/output + if not __ok_to_edit_metadata(data.id): + return self.message_exception(trans, 'This dataset is currently being used as input or output. You cannot change datatype until the jobs have completed or you have canceled them.') + else: + path = data.dataset.file_name + is_binary = check_binary(path) + datatype = sniff.guess_ext(path, trans.app.datatypes_registry.sniff_order, is_binary=is_binary) + trans.app.datatypes_registry.change_datatype(data, datatype) + trans.sa_session.flush() + trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( + trans.app.datatypes_registry.set_external_metadata_tool, trans, incoming={'input1': data}, + overwrite=False) # overwrite is False as per existing behavior + message = 'Sniffed and changed the type to %s.' % datatype + else: + return self.message_exception(trans, 'Changing datatype %s is not allowed.' % (data.extension)) elif operation == 'autodetect': # The user clicked the Auto-detect button on the 'Edit Attributes' form # prevent modifying metadata when dataset is queued or running as input/output From 170ed2a8c2a4d324ceac6fabf2a115d6eb801c16 Mon Sep 17 00:00:00 2001 From: Qiang Gu Date: Thu, 15 Nov 2018 12:33:21 -0800 Subject: [PATCH 016/260] fix style and wording --- .../scripts/mvc/dataset/dataset-edit-attributes.js | 8 ++++---- lib/galaxy/webapps/galaxy/controllers/dataset.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/client/galaxy/scripts/mvc/dataset/dataset-edit-attributes.js b/client/galaxy/scripts/mvc/dataset/dataset-edit-attributes.js index ba95a756415..8b97b11fb46 100644 --- a/client/galaxy/scripts/mvc/dataset/dataset-edit-attributes.js +++ b/client/galaxy/scripts/mvc/dataset/dataset-edit-attributes.js @@ -173,12 +173,12 @@ var View = Backbone.View.extend({ self._submit("datatype", form); } }), - submit_datatype_sniff: new Ui.ButtonIcon({ - tooltip: _l("Sniff the datatype and change it."), - title: _l("Sniff datatype"), + submit_datatype_detect: new Ui.ButtonIcon({ + tooltip: _l("Detect the datatype and change it."), + title: _l("Detect datatype"), icon: "fa-undo", onclick: function() { - self._submit("datatype_sniff", form); + self._submit("datatype_detect", form); } }) } diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py index 32a4369e8b9..1fe7c3facfe 100644 --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -17,8 +17,8 @@ from galaxy import ( util, web ) -from galaxy.datatypes.display_applications.util import decode_dataset_user, encode_dataset_user from galaxy.datatypes import sniff +from galaxy.datatypes.display_applications.util import decode_dataset_user, encode_dataset_user from galaxy.exceptions import RequestParameterInvalidException from galaxy.model.item_attrs import UsesAnnotations, UsesItemRatings from galaxy.util import ( @@ -461,8 +461,8 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE message = 'Changed the type to %s.' % datatype else: return self.message_exception(trans, 'You are unable to change datatypes in this manner. Changing %s to %s is not allowed.' % (data.extension, datatype)) - elif operation == 'datatype_sniff': - # The user clicked the Sniff button on the 'Change data type' form + elif operation == 'datatype_detect': + # The user clicked the 'Detect datatype' button on the 'Change data type' form if data.datatype.allow_datatype_change: # prevent modifying datatype when dataset is queued or running as input/output if not __ok_to_edit_metadata(data.id): @@ -476,9 +476,9 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, trans, incoming={'input1': data}, overwrite=False) # overwrite is False as per existing behavior - message = 'Sniffed and changed the type to %s.' % datatype + message = 'Detection was finished and changed the datatype to %s.' % datatype else: - return self.message_exception(trans, 'Changing datatype %s is not allowed.' % (data.extension)) + return self.message_exception(trans, 'Changing datatype "%s" is not allowed.' % (data.extension)) elif operation == 'autodetect': # The user clicked the Auto-detect button on the 'Edit Attributes' form # prevent modifying metadata when dataset is queued or running as input/output From b0bc14a90524a01a162f803b5604cbb361dac386 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 16 Nov 2018 15:29:44 +0100 Subject: [PATCH 017/260] drmaa: make external_runjob return stripped stdio --- lib/galaxy/jobs/runners/drmaa.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/jobs/runners/drmaa.py b/lib/galaxy/jobs/runners/drmaa.py index a30893cb647..b5caf4cb4db 100644 --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -207,7 +207,7 @@ class DRMAAJobRunner(AsynchronousJobRunner): log.debug('(%s) submitting with credentials: %s [uid: %s]' % (galaxy_id_tag, pwent[0], pwent[2])) filename = self.store_jobtemplate(job_wrapper, jt) self.userid = pwent[2] - external_job_id = self.external_runjob(external_runjob_script, filename, pwent[2]).strip() + external_job_id = self.external_runjob(external_runjob_script, filename, pwent[2]) if external_job_id is None: job_wrapper.fail("(%s) could not queue job" % galaxy_id_tag) return @@ -386,7 +386,8 @@ class DRMAAJobRunner(AsynchronousJobRunner): return None # The expected output is a single line containing a single numeric value: # the DRMAA job-ID. If not the case, will throw an error. - if not stdoutdata.strip(): + stdoutdata = stdoutdata.strip() + if not stdoutdata: log.exception("External_runjob did not returned nothing instead of the job id") return None return stdoutdata From 4ff3c95e38d1c62a5d1b85994e2dde909a2f7e38 Mon Sep 17 00:00:00 2001 From: Qiang Gu Date: Mon, 19 Nov 2018 22:20:32 -0800 Subject: [PATCH 018/260] fix a typo error --- client/galaxy/scripts/mvc/dataset/dataset-edit-attributes.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/galaxy/scripts/mvc/dataset/dataset-edit-attributes.js b/client/galaxy/scripts/mvc/dataset/dataset-edit-attributes.js index 8b97b11fb46..e3b8e0558cf 100644 --- a/client/galaxy/scripts/mvc/dataset/dataset-edit-attributes.js +++ b/client/galaxy/scripts/mvc/dataset/dataset-edit-attributes.js @@ -173,7 +173,7 @@ var View = Backbone.View.extend({ self._submit("datatype", form); } }), - submit_datatype_detect: new Ui.ButtonIcon({ + submit_datatype_detect: new Ui.Button({ tooltip: _l("Detect the datatype and change it."), title: _l("Detect datatype"), icon: "fa-undo", From 9bac3c44faf51e0e4c396d7f1d3470f7aa4a588e Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 20 Nov 2018 15:48:34 +0100 Subject: [PATCH 019/260] Path-aware workflows. - Allow workflows to be uploaded via path using the GUI and and API for admins. - Track the path and resync workflows on save (both .ga and format 2 workflows) - this will allow Planemo to leverage the Galaxy workflow editor to interactively save workflows. --- lib/galaxy/managers/workflows.py | 75 +++++++++++++++---- lib/galaxy/model/mapping.py | 1 + .../migrate/versions/0146_workflow_paths.py | 62 +++++++++++++++ lib/galaxy/web/base/controller.py | 5 +- lib/galaxy/webapps/galaxy/api/workflows.py | 41 ++++++---- test/api/test_workflows.py | 37 +++------ test/base/populators.py | 22 ++++++ test/base/workflow_fixtures.py | 13 ++++ test/integration/test_dockerized_jobs.py | 3 +- test/integration/test_workflow_sync.py | 53 +++++++++++++ 10 files changed, 255 insertions(+), 57 deletions(-) create mode 100644 lib/galaxy/model/migrate/versions/0146_workflow_paths.py create mode 100644 test/integration/test_workflow_sync.py diff --git a/lib/galaxy/managers/workflows.py b/lib/galaxy/managers/workflows.py index c81da03c9f7..6a3f15e833a 100644 --- a/lib/galaxy/managers/workflows.py +++ b/lib/galaxy/managers/workflows.py @@ -2,6 +2,7 @@ from __future__ import absolute_import import json import logging +import os import uuid from collections import namedtuple @@ -11,6 +12,7 @@ from gxformat2 import ( ImportOptions, python_to_workflow, ) +from gxformat2.converter import ordered_load from six import string_types from sqlalchemy import and_ from sqlalchemy.orm import joinedload, subqueryload @@ -241,7 +243,12 @@ class WorkflowContentsManager(UsesAnnotations): self.app = app self._resource_mapper_function = get_resource_mapper_function(app) - def normalize_workflow_format(self, as_dict): + def ensure_raw_description(self, dict_or_raw_description): + if not isinstance(dict_or_raw_description, RawWorkflowDescription): + dict_or_raw_description = RawWorkflowDescription(dict_or_raw_description) + return dict_or_raw_description + + def normalize_workflow_format(self, trans, as_dict): """Process incoming workflow descriptions for consumption by other methods. Currently this mostly means converting format 2 workflows into standard Galaxy @@ -250,6 +257,18 @@ class WorkflowContentsManager(UsesAnnotations): side the data model and apply updates in a way that largely preserves YAML structure so workflows can be extracted. """ + workflow_directory = None + workflow_path = None + + if as_dict.get("src", None) == "from_path": + if not trans.user_is_admin: + raise exceptions.AdminRequiredException() + + workflow_path = as_dict.get("path") + with open(workflow_path, "r") as f: + as_dict = ordered_load(f) + workflow_directory = os.path.normpath(os.path.dirname(workflow_path)) + workflow_class = as_dict.get("class", None) if workflow_class == "GalaxyWorkflow" or "$graph" in as_dict or "yaml_content" in as_dict: if not self.app.config.enable_beta_workflow_format: @@ -259,13 +278,14 @@ class WorkflowContentsManager(UsesAnnotations): galaxy_interface = Format2ConverterGalaxyInterface() import_options = ImportOptions() import_options.deduplicate_subworkflows = True - as_dict = python_to_workflow(as_dict, galaxy_interface, workflow_directory=None, import_options=import_options) - return as_dict + as_dict = python_to_workflow(as_dict, galaxy_interface, workflow_directory=workflow_directory, import_options=import_options) - def build_workflow_from_dict( + return RawWorkflowDescription(as_dict, workflow_path) + + def build_workflow_from_raw_description( self, trans, - data, + raw_workflow_description, source=None, add_to_menu=False, publish=False, @@ -273,6 +293,7 @@ class WorkflowContentsManager(UsesAnnotations): exact_tools=True, fill_defaults=False, ): + data = raw_workflow_description.as_dict # Put parameters in workflow mode trans.workflow_building_mode = workflow_building_modes.ENABLED # If there's a source, put it in the workflow name. @@ -280,9 +301,9 @@ class WorkflowContentsManager(UsesAnnotations): name = "%s (imported from %s)" % (data['name'], source) else: name = data['name'] - workflow, missing_tool_tups = self._workflow_from_dict( + workflow, missing_tool_tups = self._workflow_from_raw_description( trans, - data, + raw_workflow_description, name=name, exact_tools=exact_tools, fill_defaults=fill_defaults, @@ -293,6 +314,7 @@ class WorkflowContentsManager(UsesAnnotations): if create_stored_workflow: # Connect up stored = model.StoredWorkflow() + stored.from_path = raw_workflow_description.workflow_path stored.name = workflow.name workflow.stored_workflow = stored stored.latest_workflow = workflow @@ -327,13 +349,15 @@ class WorkflowContentsManager(UsesAnnotations): missing_tools=missing_tool_tups ) - def update_workflow_from_dict(self, trans, stored_workflow, workflow_data, **kwds): + def update_workflow_from_raw_description(self, trans, stored_workflow, raw_workflow_description, **kwds): + raw_workflow_description = self.ensure_raw_description(raw_workflow_description) + # Put parameters in workflow mode trans.workflow_building_mode = workflow_building_modes.ENABLED - workflow, missing_tool_tups = self._workflow_from_dict( + workflow, missing_tool_tups = self._workflow_from_raw_description( trans, - workflow_data, + raw_workflow_description, name=stored_workflow.name, **kwds ) @@ -349,6 +373,8 @@ class WorkflowContentsManager(UsesAnnotations): stored_workflow.latest_workflow = workflow # Persist trans.sa_session.flush() + if stored_workflow.from_path: + self._sync_stored_workflow(trans, stored_workflow) # Return something informative errors = [] if workflow.has_errors: @@ -357,7 +383,8 @@ class WorkflowContentsManager(UsesAnnotations): errors.append("This workflow contains cycles") return workflow, errors - def _workflow_from_dict(self, trans, data, name, **kwds): + def _workflow_from_raw_description(self, trans, raw_workflow_description, name, **kwds): + data = raw_workflow_description.as_dict if isinstance(data, string_types): data = json.loads(data) @@ -449,6 +476,18 @@ class WorkflowContentsManager(UsesAnnotations): wf_dict['version'] = len(stored.workflows) - 1 return wf_dict + def _sync_stored_workflow(self, trans, stored_workflow): + workflow_path = stored_workflow.from_path + workflow = stored_workflow.latest_workflow + with open(workflow_path, "w") as f: + if workflow_path.endswith(".ga"): + wf_dict = self._workflow_to_dict_export(trans, stored_workflow, workflow=workflow) + json.dump(wf_dict, f, indent=4) + else: + wf_dict = self._workflow_to_dict_export(trans, stored_workflow, workflow=workflow) + wf_dict = from_galaxy_native(wf_dict, None, json_wrapper=True) + f.write(wf_dict["yaml_content"]) + def _workflow_to_dict_run(self, trans, stored, workflow): """ Builds workflow dictionary used by run workflow form @@ -821,7 +860,7 @@ class WorkflowContentsManager(UsesAnnotations): # newer Galaxy instances can be used with older Galaxy # instances if they do no include multiple input # tools. This should be removed at some point. Mirrored - # hack in _workflow_from_dict should never be removed so + # hack in _workflow_from_raw_description should never be removed so # existing workflow exports continue to function. for input_name, input_conn in dict(input_conn_dict).items(): if len(input_conn) == 1: @@ -1041,8 +1080,9 @@ class WorkflowContentsManager(UsesAnnotations): return subworkflow def __build_embedded_subworkflow(self, trans, data, **kwds): - subworkflow = self.build_workflow_from_dict( - trans, data, create_stored_workflow=False, fill_defaults=kwds.get("fill_defaults", False) + raw_workflow_description = self.ensure_raw_description(data) + subworkflow = self.build_workflow_from_raw_description( + trans, raw_workflow_description, create_stored_workflow=False, fill_defaults=kwds.get("fill_defaults", False) ).workflow return subworkflow @@ -1095,6 +1135,13 @@ class MissingToolsException(exceptions.MessageException): self.errors = errors +class RawWorkflowDescription(object): + + def __init__(self, as_dict, workflow_path=None): + self.as_dict = as_dict + self.workflow_path = workflow_path + + class Format2ConverterGalaxyInterface(ImporterGalaxyInterface): def import_workflow(self, workflow, **kwds): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 14eb274e1af..e7fdfb6b0d1 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -874,6 +874,7 @@ model.StoredWorkflow.table = Table( Column("deleted", Boolean, default=False), Column("importable", Boolean, default=False), Column("slug", TEXT, index=True), + Column("from_path", TEXT, index=True), Column("published", Boolean, index=True, default=False)) model.Workflow.table = Table( diff --git a/lib/galaxy/model/migrate/versions/0146_workflow_paths.py b/lib/galaxy/model/migrate/versions/0146_workflow_paths.py new file mode 100644 index 00000000000..163be6d72f3 --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0146_workflow_paths.py @@ -0,0 +1,62 @@ +""" +Migration script for workflow paths. +""" +from __future__ import print_function + +import logging + +from sqlalchemy import ( + Column, + MetaData, + Table, + TEXT, +) + +log = logging.getLogger(__name__) +metadata = MetaData() + +from_path_column = Column("from_path", TEXT, nullable=True) + + +def upgrade(migrate_engine): + metadata.bind = migrate_engine + print(__doc__) + metadata.reflect() + + __add_column(from_path_column, "stored_workflow", metadata) + + +def downgrade(migrate_engine): + metadata.bind = migrate_engine + + __drop_column(from_path_column, "stored_workflow", metadata) + + +def __add_column(column, table_name, metadata, **kwds): + try: + table = Table(table_name, metadata, autoload=True) + column.create(table, **kwds) + except Exception: + log.exception("Adding column %s failed.", column) + + +def __drop_column(column_name, table_name, metadata): + try: + table = Table(table_name, metadata, autoload=True) + getattr(table.c, column_name).drop() + except Exception: + log.exception("Dropping column %s failed.", column_name) + + +def _create(table): + try: + table.create() + except Exception: + log.exception("Creating %s table failed.", table.name) + + +def _drop(table): + try: + table.drop() + except Exception: + log.exception("Dropping %s table failed.", table.name) diff --git a/lib/galaxy/web/base/controller.py b/lib/galaxy/web/base/controller.py index c222ff24741..07d2a9770b8 100644 --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -1243,9 +1243,10 @@ class UsesStoredWorkflowMixin(SharableItemSecurityMixin, UsesAnnotations): """ # TODO: replace this method with direct access to manager. workflow_contents_manager = workflows.WorkflowContentsManager(self.app) - created_workflow = workflow_contents_manager.build_workflow_from_dict( + raw_workflow_description = workflow_contents_manager.ensure_raw_description(data) + created_workflow = workflow_contents_manager.build_workflow_from_raw_description( trans, - data, + raw_workflow_description, source=source, add_to_menu=add_to_menu, publish=publish, diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index 481b553f50b..d5b6a3af3d9 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -284,6 +284,7 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn 'workflow_id', 'installed_repository_file', 'from_history_id', + 'from_path', 'shared_workflow_id', 'workflow', ]) @@ -313,10 +314,17 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn archive_file = payload.get('archive_file') archive_data = None if archive_source: - try: - archive_data = requests.get(archive_source).text - except Exception: - raise exceptions.MessageException("Failed to open URL '%s'." % escape(archive_source)) + if archive_source.startswith("file://"): + if not trans.user_is_admin: + raise exceptions.AdminRequiredException() + workflow_src = {"src": "from_path", "path": archive_source[len("file://"):]} + payload["workflow"] = workflow_src + return self.__api_import_new_workflow(trans, payload, **kwd) + else: + try: + archive_data = requests.get(archive_source).text + except Exception: + raise exceptions.MessageException("Failed to open URL '%s'." % escape(archive_source)) elif hasattr(archive_file, 'file'): uploaded_file = archive_file.file uploaded_file_name = uploaded_file.name @@ -350,6 +358,11 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn item['url'] = url_for('workflow', id=item['id']) return item + if 'from_path' in payload: + from_path = payload.get('from_path') + payload["workflow"] = {"src": "from_path", "path": from_path} + return self.__api_import_new_workflow(trans, payload, **kwd) + if 'shared_workflow_id' in payload: workflow_id = payload['shared_workflow_id'] return self.__api_import_shared_workflow(trans, workflow_id, payload) @@ -492,7 +505,8 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn stored_workflow = self.__get_stored_workflow(trans, id) workflow_dict = payload.get('workflow') or payload if workflow_dict: - workflow_dict = self.__normalize_workflow(workflow_dict) + raw_workflow_description = self.__normalize_workflow(trans, workflow_dict) + workflow_dict = raw_workflow_description.as_dict new_workflow_name = workflow_dict.get('name') or workflow_dict.get('name') if new_workflow_name and new_workflow_name != stored_workflow.name: sanitized_name = sanitize_html(new_workflow_name) @@ -524,10 +538,10 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn if 'steps' in workflow_dict: try: from_dict_kwds = self.__import_or_update_kwds(payload) - workflow, errors = self.workflow_contents_manager.update_workflow_from_dict( + workflow, errors = self.workflow_contents_manager.update_workflow_from_raw_description( trans, stored_workflow, - workflow_dict, + raw_workflow_description, **from_dict_kwds ) except workflows.MissingToolsException: @@ -573,8 +587,8 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn raise exceptions.MessageException("The data content does not appear to be a valid workflow.") if not data: raise exceptions.MessageException("The data content is missing.") - data = self.__normalize_workflow(data) - workflow, missing_tool_tups = self._workflow_from_dict(trans, data, source=source) + raw_workflow_description = self.__normalize_workflow(trans, data) + workflow, missing_tool_tups = self._workflow_from_dict(trans, raw_workflow_description, source=source) workflow = workflow.latest_workflow if workflow.has_errors: return {"message": "Imported, but some steps in this workflow have validation errors.", "status": "error"} @@ -586,7 +600,8 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn def __api_import_new_workflow(self, trans, payload, **kwd): data = payload['workflow'] - data = self.__normalize_workflow(data) + raw_workflow_description = self.__normalize_workflow(trans, data) + data = raw_workflow_description.as_dict import_tools = util.string_as_bool(payload.get("import_tools", False)) if import_tools and not trans.user_is_admin: raise exceptions.AdminRequiredException() @@ -601,7 +616,7 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn raise exceptions.RequestParameterInvalidException("Published workflow must be importable.") from_dict_kwds["publish"] = publish - workflow, missing_tool_tups = self._workflow_from_dict(trans, data, **from_dict_kwds) + workflow, missing_tool_tups = self._workflow_from_dict(trans, raw_workflow_description, **from_dict_kwds) if importable: self._make_item_accessible(trans.sa_session, workflow) trans.sa_session.flush() @@ -654,8 +669,8 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn 'fill_defaults': fill_defaults, } - def __normalize_workflow(self, as_dict): - return self.workflow_contents_manager.normalize_workflow_format(as_dict) + def __normalize_workflow(self, trans, as_dict): + return self.workflow_contents_manager.normalize_workflow_format(trans, as_dict) @expose_api def import_shared_workflow_deprecated(self, trans, payload, **kwd): diff --git a/test/api/test_workflows.py b/test/api/test_workflows.py index c29ff11cd55..5fb2fedce23 100644 --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -20,6 +20,7 @@ from base.workflow_fixtures import ( # noqa: I100 WORKFLOW_NESTED_REPLACEMENT_PARAMETER, WORKFLOW_NESTED_RUNTIME_PARAMETER, WORKFLOW_NESTED_SIMPLE, + WORKFLOW_ONE_STEP_DEFAULT, WORKFLOW_RENAME_ON_INPUT, WORKFLOW_RUNTIME_PARAMETER_AFTER_PAUSE, WORKFLOW_WITH_DYNAMIC_OUTPUT_COLLECTION, @@ -175,13 +176,7 @@ class BaseWorkflowsApiTestCase(api.ApiTestCase): self.assertEqual(len(jobs), n) def _download_workflow(self, workflow_id, style=None): - params = {} - if style: - params = {"style": style} - download_response = self._get("workflows/%s/download" % workflow_id, params) - self._assert_status_code_is(download_response, 200) - downloaded_workflow = download_response.json() - return downloaded_workflow + return self.workflow_populator.download_workflow(workflow_id, style=style) def wait_for_invocation_and_jobs(self, history_id, workflow_id, invocation_id, assert_ok=True): state = self.workflow_populator.wait_for_invocation(workflow_id, invocation_id) @@ -2872,18 +2867,7 @@ steps: @skip_without_tool("random_lines1") def test_run_replace_params_over_default(self): with self.dataset_populator.test_history() as history_id: - self._run_jobs(""" -class: GalaxyWorkflow -inputs: - input: data -steps: - randomlines: - tool_id: random_lines1 - in: - input: input - num_lines: - default: 6 -""", test_data=""" + self._run_jobs(WORKFLOW_ONE_STEP_DEFAULT, test_data=""" step_parameters: '1': num_lines: 4 @@ -2894,6 +2878,13 @@ input: result = self.dataset_populator.get_history_dataset_content(history_id) assert result.count("\n") == 4 + @skip_without_tool("random_lines1") + def test_defaults_editor(self): + workflow_id = self._upload_yaml_workflow(WORKFLOW_ONE_STEP_DEFAULT, publish=True) + workflow_object = self._download_workflow(workflow_id, style="editor") + put_response = self._update_workflow(workflow_id, workflow_object) + assert put_response.status_code == 200 + @skip_without_tool("random_lines1") def test_run_replace_params_over_default_delayed(self): with self.dataset_populator.test_history() as history_id: @@ -3167,13 +3158,7 @@ input_c: return target_state_reached def _update_workflow(self, workflow_id, workflow_object): - data = dict( - workflow=workflow_object - ) - raw_url = 'workflows/%s' % workflow_id - url = self._api_url(raw_url, use_key=True) - put_response = put(url, data=dumps(data)) - return put_response + return self.workflow_populator.update_workflow(workflow_id, workflow_object) def _invocation_step_details(self, workflow_id, invocation_id, step_id): invocation_step_response = self._get("workflows/%s/usage/%s/steps/%s" % (workflow_id, invocation_id, step_id)) diff --git a/test/base/populators.py b/test/base/populators.py index 4f2e7ef9068..6d2b51f180b 100644 --- a/test/base/populators.py +++ b/test/base/populators.py @@ -556,6 +556,14 @@ class BaseWorkflowPopulator(object): workflow = self.load_workflow(name) return self.create_workflow(workflow, **create_kwds) + def import_workflow_from_path(self, from_path): + data = dict( + from_path=from_path + ) + import_response = self._post("workflows", data=data) + api_asserts.assert_status_code_is(import_response, 200) + return import_response.json()["id"] + def create_workflow(self, workflow, **create_kwds): upload_response = self.create_workflow_response(workflow, **create_kwds) uploaded_workflow_id = upload_response.json()["id"] @@ -625,6 +633,20 @@ class BaseWorkflowPopulator(object): api_asserts.assert_status_code_is(response, 200) return response.json() + def update_workflow(self, workflow_id, workflow_object): + data = dict( + workflow=workflow_object + ) + raw_url = 'workflows/%s' % workflow_id + put_response = self.galaxy_interactor._put(raw_url, data=json.dumps(data)) + return put_response + + @contextlib.contextmanager + def export_for_update(self, workflow_id): + workflow_object = self.download_workflow(workflow_id) + yield workflow_object + self.update_workflow(workflow_id, workflow_object) + def run_workflow(self, has_workflow, test_data=None, history_id=None, wait=True, source_type=None, jobs_descriptions=None, expected_response=200, assert_ok=True, client_convert=None, round_trip_format_conversion=False, raw_yaml=False): """High-level wrapper around workflow API, etc. to invoke format 2 workflows.""" workflow_populator = self diff --git a/test/base/workflow_fixtures.py b/test/base/workflow_fixtures.py index 519dd750131..7f96b3c41db 100644 --- a/test/base/workflow_fixtures.py +++ b/test/base/workflow_fixtures.py @@ -417,6 +417,19 @@ steps: inner_input: outer_input """ +WORKFLOW_ONE_STEP_DEFAULT = """ +class: GalaxyWorkflow +inputs: + input: data +steps: + randomlines: + tool_id: random_lines1 + in: + input: input + num_lines: + default: 6 +""" + WORKFLOW_WITH_OUTPUTS = """ class: GalaxyWorkflow inputs: diff --git a/test/integration/test_dockerized_jobs.py b/test/integration/test_dockerized_jobs.py index a9bb909b09e..43e56492c00 100644 --- a/test/integration/test_dockerized_jobs.py +++ b/test/integration/test_dockerized_jobs.py @@ -1,7 +1,6 @@ """Integration tests for running tools in Docker containers.""" import os -import tempfile from base import integration_util from base.populators import ( @@ -21,7 +20,7 @@ class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, Ru @classmethod def handle_galaxy_config_kwds(cls, config): - cls.jobs_directory = tempfile.mkdtemp() + cls.jobs_directory = cls._test_driver.mkdtemp() config["jobs_directory"] = cls.jobs_directory config["job_config_file"] = DOCKERIZED_JOB_CONFIG_FILE # Disable tool dependency resolution. diff --git a/test/integration/test_workflow_sync.py b/test/integration/test_workflow_sync.py new file mode 100644 index 00000000000..9fdbdc8061d --- /dev/null +++ b/test/integration/test_workflow_sync.py @@ -0,0 +1,53 @@ +"""Integration tests for workflow syncing.""" + +import json +import os + +import yaml + +from base import integration_util # noqa: I100,I202 +from base.populators import ( # noqa: I100 + DatasetPopulator, + WorkflowPopulator, +) +from base.workflow_fixtures import WORKFLOW_SIMPLE_CAT_TWICE # noqa: I100 + + +class WorkflowSyncTestCase(integration_util.IntegrationTestCase): + + framework_tool_and_types = True + require_admin_user = True + + def setUp(self): + super(WorkflowSyncTestCase, self).setUp() + self.dataset_populator = DatasetPopulator(self.galaxy_interactor) + self.workflow_populator = WorkflowPopulator(self.galaxy_interactor) + + @classmethod + def handle_galaxy_config_kwds(cls, config): + cls.workflow_directory = cls._test_driver.mkdtemp() + + def test_sync_format2(self): + workflow_path = self._write_workflow_content("workflow.yml", WORKFLOW_SIMPLE_CAT_TWICE) + workflow_id = self.workflow_populator.import_workflow_from_path(workflow_path) + with self.workflow_populator.export_for_update(workflow_id) as workflow_object: + workflow_object["annotation"] = "new annotation" + with open(workflow_path, "r") as f: + data = yaml.load(f) + assert data["doc"] == "new annotation" + + def test_sync_ga(self): + workflow_json = self.workflow_populator.load_workflow("synctest") + workflow_path = self._write_workflow_content("workflow.ga", json.dumps(workflow_json)) + workflow_id = self.workflow_populator.import_workflow_from_path(workflow_path) + with self.workflow_populator.export_for_update(workflow_id) as workflow_object: + workflow_object["annotation"] = "new annotation" + with open(workflow_path, "r") as f: + data = json.load(f) + assert data["annotation"] == "new annotation" + + def _write_workflow_content(self, filename, content): + workflow_path = os.path.join(self.workflow_directory, filename) + with open(workflow_path, "w") as f: + f.write(content) + return workflow_path From 855ccb12791a649d72598ba2433491bd5d705899 Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Fri, 23 Nov 2018 12:10:14 +0100 Subject: [PATCH 020/260] GRT: Rip out all parameters, round file sizes --- scripts/grt/export.py | 109 ++----------------------------------- scripts/grt/grt.yml.sample | 14 ----- 2 files changed, 4 insertions(+), 119 deletions(-) diff --git a/scripts/grt/export.py b/scripts/grt/export.py index 070eeef220b..24e5f447ce0 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -48,74 +48,8 @@ def kw_metrics(job): } -class Sanitization: - - def __init__(self, sanitization_config, model, sa_session): - self.sanitization_config = sanitization_config - # SA Stuff - self.model = model - self.sa_session = sa_session - - if 'tool_params' not in self.sanitization_config: - self.sanitization_config['tool_params'] = {} - - def blacklisted_tree(self, path): - if self.tool_id in self.sanitization_config['tool_params'] and path.lstrip('.') in self.sanitization_config['tool_params'][self.tool_id]: - return True - return False - - def sanitize_data(self, tool_id, key, value): - # If the tool is blacklisted, skip it. - if tool_id in self.sanitization_config['tools']: - return 'null' - # Thus, all tools below here are not blacklisted at the top level. - - # If the key is listed precisely (not a sub-tree), we can also return slightly more quickly. - if tool_id in self.sanitization_config['tool_params'] and key in self.sanitization_config['tool_params'][tool_id]: - return 'null' - - # If the key isn't a prefix for any of the keys being sanitized, then this is safe. - if tool_id in self.sanitization_config['tool_params'] and not any(san_key.startswith(key) for san_key in self.sanitization_config['tool_params'][tool_id]): - return value - - # Slow path. - if isinstance(value, str): - try: - unsanitized = {key: json.loads(value)} - except ValueError: - unsanitized = {key: value} - else: - unsanitized = {key: value} - - self.tool_id = tool_id - return json.dumps(self._sanitize_value(unsanitized)) - - def _sanitize_dict(self, unsanitized_dict, path=""): - return { - k: self._sanitize_value(v, path=path + '.' + k) - for (k, v) - in unsanitized_dict.items() - } - - def _sanitize_list(self, unsanitized_list, path=""): - return [ - self._sanitize_value(v, path=path + '.*') - for v in unsanitized_list - ] - - def _sanitize_value(self, unsanitized_value, path=""): - logging.debug("%sSAN %s" % (' ' * path.count('.'), unsanitized_value)) - if self.blacklisted_tree(path): - logging.debug("%sSAN ***REDACTED***" % (' ' * path.count('.'))) - return None - - if type(unsanitized_value) is dict: - return self._sanitize_dict(unsanitized_value, path=path) - elif type(unsanitized_value) is list: - return self._sanitize_list(unsanitized_value, path=path) - else: - logging.debug("%s> Sanitizing %s = %s" % (' ' * path.count('.'), path, unsanitized_value)) - return unsanitized_value +def round_to_2sd(number): + return str(int(float('%.2g' % number))) def main(argv): @@ -178,10 +112,6 @@ def main(argv): active_users = defaultdict(int) job_state_data = defaultdict(int) - annotate('san_init', 'Building Sanitizer') - san = Sanitization(config['sanitization'], model, sa_session) - annotate('san_end') - if not os.path.exists(REPORT_DIR): os.makedirs(REPORT_DIR) @@ -314,7 +244,7 @@ def main(argv): handle_datasets.write('\t') handle_datasets.write(str(hdas[hda_id][1])) handle_datasets.write('\t') - handle_datasets.write(str(datasets[dataset_id][0])) + handle_datasets.write(round_to_2sd(datasets[dataset_id][0])) handle_datasets.write('\t') handle_datasets.write(str(job[2])) handle_datasets.write('\t') @@ -357,37 +287,6 @@ def main(argv): handle_metric_num.close() annotate('export_metric_num_end') - annotate('export_params_start', 'Export Job Parameters') - handle_params = open(REPORT_BASE + '.params.tsv', 'w') - handle_params.write('\t'.join(('job_id', 'name', 'value')) + '\n') - for offset_start in range(last_job_sent, end_job_id, args.batch_size): - logging.debug("Processing %s:%s", offset_start, min(end_job_id, offset_start + args.batch_size)) - for param in sa_session.query(model.JobParameter.job_id, model.JobParameter.name, model.JobParameter.value) \ - .filter(model.JobParameter.job_id > offset_start) \ - .filter(model.JobParameter.job_id <= min(end_job_id, offset_start + args.batch_size)) \ - .all(): - # No associated job - if param[0] not in job_tool_map: - continue - # If the tool is blacklisted, exclude everywhere - if job_tool_map[param[0]] in blacklisted_tools: - continue - - try: - sanitized = san.sanitize_data(job_tool_map[param[0]], param[1], param[2]) - - handle_params.write(str(param[0])) - handle_params.write('\t') - handle_params.write(param[1]) - handle_params.write('\t') - handle_params.write(json.dumps(sanitized)) - handle_params.write('\n') - except Exception: - logging.warning("Unable to write out a 'handle_params' row. Ignoring the row.", exc_info=True) - continue - handle_params.close() - annotate('export_params_end') - # Now on to outputs. with tarfile.open(REPORT_BASE + '.tar.gz', 'w:gz') as handle: for name in ('jobs', 'metric_num', 'params', 'datasets'): @@ -403,7 +302,7 @@ def main(argv): # Now serialize the individual report data. with open(REPORT_BASE + '.json', 'w') as handle: json.dump({ - "version": 2, + "version": 3, "galaxy_version": gxconfig.version_major, "generated": REPORT_IDENTIFIER, "report_hash": "sha256:" + sha, diff --git a/scripts/grt/grt.yml.sample b/scripts/grt/grt.yml.sample index e775d66f0cf..3e201aa0fc1 100644 --- a/scripts/grt/grt.yml.sample +++ b/scripts/grt/grt.yml.sample @@ -13,17 +13,3 @@ sanitization: tools: - __SET_METADATA__ - upload1 - # Or you can blacklist individual parameters from being submitted, e.g. if - # you have API keys as a tool parameter. - tool_params: - # Or to blacklist under a specific tool, just specify the ID - some_tool_id: - - dbkey - # If you need to specify a parameter multiple levels deep, you can - # do that as well. Currently we only support blacklisting via the - # full path, rather than just a path component. So everything under - # `path.to.parameter` will be blacklisted. - - path.to.parameter - # However you could not do "parameter" and have everything under - # `path.to.parameter` be removed. - # Repeats are rendered as an *, e.g.: repeat_name.*.values From c4c8af036fda560d5340e77d9f38782754729eb1 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 27 Nov 2018 18:47:57 +0100 Subject: [PATCH 021/260] doc: updated error parsing docs see https://github.com/galaxyproject/galaxy/pull/697://github.com/galaxyproject/galaxy/pull/6977 --- lib/galaxy/tools/xsd/galaxy.xsd | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/tools/xsd/galaxy.xsd b/lib/galaxy/tools/xsd/galaxy.xsd index 664faa8b376..bf934430d4f 100644 --- a/lib/galaxy/tools/xsd/galaxy.xsd +++ b/lib/galaxy/tools/xsd/galaxy.xsd @@ -2699,20 +2699,13 @@ on the topic of ``GALAXY_SLOTS`` for more information and examples. ### Error detection -The ``detect_errors`` attribute of ``command``, if present, can be one of: +The ``detect_errors`` attribute of ``command``, if present, loads a preset of error detection checks (for exit codes and content of stdio to indicate fatal tool errors or fatal out of memory errors). It can be one of: -* ``default``: no-op fallback to ``stdio`` tags and erroring on standard error output (for legacy tools). -* ``exit_code``: error if tool exit code is not 0. (The @jmchilton recommendation). -* ``aggressive``: error if tool exit code is not 0 or ``Exception:``, ``Error:``, or - various messages related to being out of memory appear in the standard error or output. - (The @bgruening recommendation). +* ``default``: for non-legacy tools with absent stdio block non-zero exit codes are added. For legacy tools or if a stdio block is present nothing is added. +* ``exit_code``: adds checks for non zero exit codes (The @jmchilton recommendation). The ``oom_exit_code`` parameter can be used to add an additional out of memory indicating exit code. +* ``aggressive``: adds checks for non zero exit codes, and checks for ``Exception:``, ``Error:`` in the standard error. Additionally checks for messages in the standard error that indicate an out of memory error (``MemoryError``, ``std::bad_alloc``, ``java.lang.OutOfMemoryError``, ``Out of memory``). (The @bgruening recommendation). -For newer tools with ``profile>=16.04``, the default behavior is ``exit_code``. -Legacy tools default to ``default`` behavior described above (erroring if the tool -produces any standard error output). - -See [pull request 117](https://github.com/galaxyproject/galaxy/pull/117) for more implementation -information and discussion on the ``detect_errors`` attribute. +Prior to Galaxy release 19.01 the stdio block has only been used for non-legacy tools using ``default``. From release 19.01 checks defined in the stdio tag are prepended to the checks defined by the presets loaded in the command block. ]]> From 83417cab622309f9b53d8881c6946bc11e6cb3ba Mon Sep 17 00:00:00 2001 From: Ivan Kuzmin Date: Wed, 28 Nov 2018 17:56:08 +0200 Subject: [PATCH 022/260] Add styled loader vue component --- .../scripts/components/GalaxyLoader.vue | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 client/galaxy/scripts/components/GalaxyLoader.vue diff --git a/client/galaxy/scripts/components/GalaxyLoader.vue b/client/galaxy/scripts/components/GalaxyLoader.vue new file mode 100644 index 00000000000..afe8ebb75d8 --- /dev/null +++ b/client/galaxy/scripts/components/GalaxyLoader.vue @@ -0,0 +1,155 @@ + + + + + From 356d949199359a974a96d531db0dc39d97007df6 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Wed, 28 Nov 2018 13:53:38 +0530 Subject: [PATCH 023/260] Allow pulsar jobs to be resubmitted on connection failures --- lib/galaxy/jobs/runners/pulsar.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 5fc388e126f..3c4779ffa14 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -380,6 +380,9 @@ class PulsarJobRunner(AsynchronousJobRunner): except UnsupportedPulsarException as e: job_wrapper.fail(str(e), exception=False) log.exception("failure running job %d", job_wrapper.job_id) + except PulsarClientTransportError: + job_state = self._job_state(job_wrapper.get_job(), job_wrapper) + self.work_queue.put((self.fail_job, job_state)) except Exception: job_wrapper.fail("failure preparing job", exception=True) log.exception("failure running job %d", job_wrapper.job_id) @@ -436,9 +439,11 @@ class PulsarJobRunner(AsynchronousJobRunner): if hasattr(job_wrapper, 'task_id'): job_id = "%s_%s" % (job_id, job_wrapper.task_id) params = job_wrapper.job_destination.params.copy() - for key, value in params.items(): - if value: - params[key] = model.User.expand_user_properties(job_wrapper.get_job().user, value) + user = job_wrapper.get_job().user + if user: + for key, value in params.items(): + if value: + params[key] = model.User.expand_user_properties(user, value) env = getattr(job_wrapper.job_destination, "env", []) return self.get_client(params, job_id, env) @@ -498,8 +503,7 @@ class PulsarJobRunner(AsynchronousJobRunner): if failed: job_wrapper.fail("Failed to find or download one or more job outputs from remote server.", exception=True) except Exception: - message = GENERIC_REMOTE_ERROR - job_wrapper.fail(message, exception=True) + self.fail_job(job_state, message=GENERIC_REMOTE_ERROR, exception=True) log.exception("failure finishing job %d", job_wrapper.job_id) return if not PulsarJobRunner.__remote_metadata(client): @@ -517,7 +521,7 @@ class PulsarJobRunner(AsynchronousJobRunner): log.exception("Job wrapper finish method failed") job_wrapper.fail("Unable to finish job", exception=True) - def fail_job(self, job_state, message=GENERIC_REMOTE_ERROR, full_status=None): + def fail_job(self, job_state, message=GENERIC_REMOTE_ERROR, full_status=None, exception=False): """Seperated out so we can use the worker threads for it.""" self.stop_job(job_state.job_wrapper) stdout = "" @@ -525,7 +529,10 @@ class PulsarJobRunner(AsynchronousJobRunner): if full_status: stdout = full_status.get("stdout", "") stderr = full_status.get("stderr", "") - job_state.job_wrapper.fail(getattr(job_state, "fail_message", message), stdout=stdout, stderr=stderr) + self._handle_runner_state('failure', job_state) + if not job_state.runner_state_handled: + job_state.job_wrapper.fail(getattr(job_state, "fail_message", message), + stdout=stdout, stderr=stderr, exception=exception) def check_pid(self, pid): try: @@ -540,6 +547,8 @@ class PulsarJobRunner(AsynchronousJobRunner): def stop_job(self, job_wrapper): job = job_wrapper.get_job() + if not job.job_runner_external_id: + return # if our local job has JobExternalOutputMetadata associated, then our primary job has to have already finished client = self.get_client(job.destination_params, job.job_runner_external_id) job_ext_output_metadata = job.get_external_output_metadata() From 263d10ca1c7577f3138df83f19468c1c20b85ea0 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Wed, 28 Nov 2018 13:55:18 +0530 Subject: [PATCH 024/260] Added test for pulsar resubmission --- .../resubmission_pulsar_job_conf.xml | 35 +++++++++++++++++++ test/integration/test_job_resubmission.py | 12 +++++++ 2 files changed, 47 insertions(+) create mode 100644 test/integration/resubmission_pulsar_job_conf.xml diff --git a/test/integration/resubmission_pulsar_job_conf.xml b/test/integration/resubmission_pulsar_job_conf.xml new file mode 100644 index 00000000000..fae63837d39 --- /dev/null +++ b/test/integration/resubmission_pulsar_job_conf.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + https://192.0.2.0:8913/ + some_token + + + + + + + + + + + + + + + diff --git a/test/integration/test_job_resubmission.py b/test/integration/test_job_resubmission.py index 1785749eec2..39b4bfe8c70 100644 --- a/test/integration/test_job_resubmission.py +++ b/test/integration/test_job_resubmission.py @@ -13,6 +13,7 @@ JOB_RESUBMISSION_SMALL_MEMORY_RESUBMISSION_TO_LARGE_JOB_CONFIG_FILE = os.path.jo JOB_RESUBMISSION_TOOL_DETECTED_ALWAYS_ERROR_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "resubmission_tool_detected_always_error_job_conf.xml") JOB_RESUBMISSION_TOOL_DETECTED_RESUBMIT_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "resubmission_tool_detected_resubmit_job_conf.xml") JOB_RESUBMISSION_JOB_RESOURCES_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "resubmission_job_resource_parameters_conf.xml") +JOB_RESUBMISSION_PULSAR_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "resubmission_pulsar_job_conf.xml") class _BaseResubmissionIntegerationTestCase(integration_util.IntegrationTestCase): @@ -178,3 +179,14 @@ class JobResubmissionToolDetectedErrorResubmitsIntegrationTestCase(_BaseResubmis def test_dynamic_resubmission(self): self._assert_job_passes(tool_id="exit_code_from_env") + + +# Verify that a failure to connect to pulsar can trigger a resubmit +class JobResubmissionPulsarIntegrationTestCase(_BaseResubmissionIntegerationTestCase): + + @classmethod + def handle_galaxy_config_kwds(cls, config): + config["job_config_file"] = JOB_RESUBMISSION_PULSAR_JOB_CONFIG_FILE + + def test_resubmit_on_invalid_pulsar_url(self): + self._assert_job_passes() From 2632c071cdf1f6f49c30d243eea4931ee6fcaab2 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Wed, 28 Nov 2018 21:38:28 +0530 Subject: [PATCH 025/260] Added referrer parameter for dynamic destinations + test --- lib/galaxy/jobs/mapper.py | 11 ++++++----- test/unit/jobs/test_mapper.py | 2 +- test/unit/jobs/test_rules/10_site.py | 3 +++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/jobs/mapper.py b/lib/galaxy/jobs/mapper.py index 563309cd483..d97bba62df6 100644 --- a/lib/galaxy/jobs/mapper.py +++ b/lib/galaxy/jobs/mapper.py @@ -55,7 +55,7 @@ class JobRunnerMapper(object): module_name = job_config.dynamic_params['rules_module'] self.rules_module = importlib.import_module(module_name) - def __invoke_expand_function(self, expand_function, destination_params): + def __invoke_expand_function(self, expand_function, destination): function_arg_names = inspect.getargspec(expand_function).args app = self.job_wrapper.app possible_args = { @@ -64,15 +64,16 @@ class JobRunnerMapper(object): "tool_id": self.job_wrapper.tool.id, "job_wrapper": self.job_wrapper, "rule_helper": RuleHelper(app), - "app": app + "app": app, + "referrer": destination } actual_args = {} # Send through any job_conf.xml defined args to function - for destination_param in destination_params.keys(): + for destination_param in destination.params.keys(): if destination_param in function_arg_names: - actual_args[destination_param] = destination_params[destination_param] + actual_args[destination_param] = destination.params[destination_param] # Populate needed args for possible_arg_name in possible_args: @@ -196,7 +197,7 @@ class JobRunnerMapper(object): return self.__handle_rule(expand_function, destination) def __handle_rule(self, rule_function, destination): - job_destination = self.__invoke_expand_function(rule_function, destination.params) + job_destination = self.__invoke_expand_function(rule_function, destination) if not isinstance(job_destination, galaxy.jobs.JobDestination): job_destination_rep = str(job_destination) # Should be either id or url if '://' in job_destination_rep: diff --git a/test/unit/jobs/test_mapper.py b/test/unit/jobs/test_mapper.py index 9f0e70d6bbc..947a9b63fa9 100644 --- a/test/unit/jobs/test_mapper.py +++ b/test/unit/jobs/test_mapper.py @@ -63,7 +63,7 @@ def test_dynamic_mapping_job_conf_params(): def test_dynamic_mapping_function_parameters(): - mapper = __mapper(__dynamic_destination(dict(function="check_rule_params"))) + mapper = __mapper(__dynamic_destination(dict(function="check_rule_params", param1="referrer_param"))) assert mapper.get_job_destination({}) is DYNAMICALLY_GENERATED_DESTINATION assert mapper.job_config.rule_response == "all_passed" diff --git a/test/unit/jobs/test_rules/10_site.py b/test/unit/jobs/test_rules/10_site.py index 0712df11d87..8b2ac476869 100644 --- a/test/unit/jobs/test_rules/10_site.py +++ b/test/unit/jobs/test_rules/10_site.py @@ -36,6 +36,7 @@ def check_rule_params( tool_id, job_wrapper, rule_helper, + referrer, app, job, user, @@ -47,6 +48,8 @@ def check_rule_params( assert job_wrapper.is_mock_job_wrapper() assert app == job_wrapper.app assert rule_helper is not None + assert isinstance(referrer, JobDestination) + assert referrer.params['param1'] == "referrer_param" assert job.user == user assert user.id == 6789 From 6fa0c8f932831fe91db492de09a2bb4b48621738 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Wed, 28 Nov 2018 12:49:33 -0500 Subject: [PATCH 026/260] add new item to the help menu --- client/galaxy/scripts/layout/menu.js | 5 +++++ config/galaxy.yml.sample | 6 ++++++ doc/source/admin/galaxy_options.rst | 21 +++++++++++++++++++++ lib/galaxy/config.py | 1 + lib/galaxy/managers/configuration.py | 1 + lib/galaxy/webapps/galaxy/config_schema.yml | 9 ++++++++- 6 files changed, 42 insertions(+), 1 deletion(-) diff --git a/client/galaxy/scripts/layout/menu.js b/client/galaxy/scripts/layout/menu.js index e6c6fdaea16..fbd41aaf72c 100644 --- a/client/galaxy/scripts/layout/menu.js +++ b/client/galaxy/scripts/layout/menu.js @@ -162,6 +162,11 @@ var Collection = Backbone.Collection.extend({ title: _l("Help"), tooltip: _l("Support, contact, and community"), menu: [ + { + title: _l("Galaxy Help"), + url: options.helpsite_url, + target: "_blank" + }, { title: _l("Support"), url: options.support_url, diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample index dbc490ba900..622f58700ed 100644 --- a/config/galaxy.yml.sample +++ b/config/galaxy.yml.sample @@ -697,6 +697,9 @@ galaxy: # The URL linked by the "Galaxy/brand" text. #logo_url: / + # The URL linked by the "Galaxy Help" link in the "Help" menu. + #helpsite_url: 'https://help.galaxyproject.org/' + # The URL linked by the "Wiki" link in the "Help" menu. #wiki_url: 'https://galaxyproject.org/' @@ -1355,6 +1358,9 @@ galaxy: # of Galaxy's stable API. #enable_beta_workflow_modules: false + # Enable import and export of workflows as Galaxy Format 2 workflows. + #enable_beta_workflow_format: false + # Following options only apply to workflows scheduled using the legacy # workflow run API (running workflows via a POST to /api/workflows). # Force usage of Galaxy's beta workflow scheduler under certain diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 91fceddd309..58970f9cea2 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -1309,6 +1309,16 @@ :Type: str +~~~~~~~~~~~~~~~~ +``helpsite_url`` +~~~~~~~~~~~~~~~~ + +:Description: + The URL linked by the "Galaxy Help" link in the "Help" menu. +:Default: ``https://help.galaxyproject.org/`` +:Type: str + + ~~~~~~~~~~~~ ``wiki_url`` ~~~~~~~~~~~~ @@ -2810,6 +2820,17 @@ :Type: bool +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``enable_beta_workflow_format`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Enable import and export of workflows as Galaxy Format 2 + workflows. +:Default: ``false`` +:Type: bool + + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``force_beta_workflow_scheduled_min_steps`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index a7187b5ebae..27f71b7ee47 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -445,6 +445,7 @@ class Configuration(object): self.message_box_class = kwargs.get('message_box_class', 'info') self.support_url = kwargs.get('support_url', 'https://galaxyproject.org/support') self.citation_url = kwargs.get('citation_url', 'https://galaxyproject.org/citing-galaxy') + self.helpsite_url = kwargs.get('helpsite_url', 'https://help.galaxyproject.org/') self.wiki_url = kwargs.get('wiki_url', 'https://galaxyproject.org/') self.blog_url = kwargs.get('blog_url', None) self.screencasts_url = kwargs.get('screencasts_url', None) diff --git a/lib/galaxy/managers/configuration.py b/lib/galaxy/managers/configuration.py index 60f33202c95..be14dff11b5 100644 --- a/lib/galaxy/managers/configuration.py +++ b/lib/galaxy/managers/configuration.py @@ -48,6 +48,7 @@ class ConfigSerializer(base.ModelSerializer): 'genomespace_ui_url' : _defaults_to(None), 'citation_url' : _defaults_to(self.app.config.citation_url), 'support_url' : _defaults_to(self.app.config.support_url), + 'helpsite_url' : _defaults_to(self.app.config.helpsite_url), 'lims_doc_url' : _defaults_to("https://usegalaxy.org/u/rkchak/p/sts"), 'biostar_url' : _defaults_to(''), 'biostar_url_redirect' : lambda *a, **c: self.url_for(controller='biostar', action='biostar_redirect', qualified=True), diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index ceb9284df8b..3ef85b537c3 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -731,7 +731,7 @@ mapping: smtp_username: type: str - default: '' + default: '' required: false desc: | If your SMTP server requires a username and password, you can provide them @@ -997,6 +997,13 @@ mapping: desc: | The URL linked by the "Galaxy/brand" text. + helpsite_url: + type: str + default: https://help.galaxyproject.org/ + required: false + desc: | + The URL linked by the "Galaxy Help" link in the "Help" menu. + wiki_url: type: str default: https://galaxyproject.org/ From be43693ef05dfae85872412f198d87a31424f486 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 29 Nov 2018 11:11:11 -0500 Subject: [PATCH 027/260] Avoid using make for autobuild; reducing necessary dependencies out of the box. --- scripts/common_startup.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index 522a522603e..63951929efe 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -241,10 +241,16 @@ if [ $SKIP_CLIENT_BUILD -eq 0 ]; then fi # Build client - if ! make client-production-maps; then - echo "ERROR: Galaxy client build failed. See ./client/README.md for more information, including how to get help." - exit 1 + pushd client + if yarn install --network-timeout 120000 --check-files; then + if ! yarn run build-production-maps; then + echo "ERROR: Galaxy client build failed. See ./client/README.md for more information, including how to get help." + exit 1 + fi + else + echo "ERROR: Galaxy client dependency installation failed. See ./client/README.md for more information, including how to get help." fi + popd else echo "Regenerating static plugin directories." python ./scripts/plugin_staging.py From 77aec5a79dc18e224ca6246659f68a49864a46cf Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 29 Nov 2018 11:11:37 -0500 Subject: [PATCH 028/260] Standardize on spaces in common_startup.sh (tabs were mixed in here) --- scripts/common_startup.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index 63951929efe..b7f053515cb 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -69,14 +69,14 @@ in_conda_env() { } if [ $COPY_SAMPLE_FILES -eq 1 ]; then - # Create any missing config/location files - for sample in $SAMPLES; do - file=${sample%.sample} - if [ ! -f "$file" -a -f "$sample" ]; then - echo "Initializing $file from $(basename "$sample")" - cp "$sample" "$file" - fi - done + # Create any missing config/location files + for sample in $SAMPLES; do + file=${sample%.sample} + if [ ! -f "$file" -a -f "$sample" ]; then + echo "Initializing $file from $(basename "$sample")" + cp "$sample" "$file" + fi + done fi # remove problematic cached files From fa408f485048ee8c7afeb96ac6c9da8dea4661f3 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 29 Nov 2018 11:19:45 -0500 Subject: [PATCH 029/260] exit with error code (and appropriate message) on either build failure --- scripts/common_startup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index b7f053515cb..a3dfde519fe 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -249,6 +249,7 @@ if [ $SKIP_CLIENT_BUILD -eq 0 ]; then fi else echo "ERROR: Galaxy client dependency installation failed. See ./client/README.md for more information, including how to get help." + exit 1 fi popd else From 247fd018105ae8d08e7eeb89b5f92dfbe6b8548c Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Thu, 29 Nov 2018 11:28:15 -0500 Subject: [PATCH 030/260] make the transition non-instrusive --- client/galaxy/scripts/layout/menu.js | 12 +- config/galaxy.yml.sample | 5 +- doc/source/admin/galaxy_options.rst | 13 +- lib/galaxy/config.py | 2 +- lib/galaxy/webapps/galaxy/config_schema.yml | 2 +- static/style/blue/base.css | 631 ++++++++------------ 6 files changed, 262 insertions(+), 403 deletions(-) diff --git a/client/galaxy/scripts/layout/menu.js b/client/galaxy/scripts/layout/menu.js index fbd41aaf72c..e8c22996f8c 100644 --- a/client/galaxy/scripts/layout/menu.js +++ b/client/galaxy/scripts/layout/menu.js @@ -162,11 +162,6 @@ var Collection = Backbone.Collection.extend({ title: _l("Help"), tooltip: _l("Support, contact, and community"), menu: [ - { - title: _l("Galaxy Help"), - url: options.helpsite_url, - target: "_blank" - }, { title: _l("Support"), url: options.support_url, @@ -224,6 +219,13 @@ var Collection = Backbone.Collection.extend({ target: "_blank" }); } + if (options.helpsite_url) { + helpTab.menu.unshift({ + title: _l("Galaxy Help"), + url: options.helpsite_url, + target: "_blank" + }); + } this.add(helpTab); // diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample index 622f58700ed..f2f30ce7bee 100644 --- a/config/galaxy.yml.sample +++ b/config/galaxy.yml.sample @@ -698,7 +698,7 @@ galaxy: #logo_url: / # The URL linked by the "Galaxy Help" link in the "Help" menu. - #helpsite_url: 'https://help.galaxyproject.org/' + #helpsite_url: '' # The URL linked by the "Wiki" link in the "Help" menu. #wiki_url: 'https://galaxyproject.org/' @@ -1358,9 +1358,6 @@ galaxy: # of Galaxy's stable API. #enable_beta_workflow_modules: false - # Enable import and export of workflows as Galaxy Format 2 workflows. - #enable_beta_workflow_format: false - # Following options only apply to workflows scheduled using the legacy # workflow run API (running workflows via a POST to /api/workflows). # Force usage of Galaxy's beta workflow scheduler under certain diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 58970f9cea2..c9a2b47dc69 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -1315,7 +1315,7 @@ :Description: The URL linked by the "Galaxy Help" link in the "Help" menu. -:Default: ``https://help.galaxyproject.org/`` +:Default: ```` :Type: str @@ -2820,17 +2820,6 @@ :Type: bool -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``enable_beta_workflow_format`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:Description: - Enable import and export of workflows as Galaxy Format 2 - workflows. -:Default: ``false`` -:Type: bool - - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``force_beta_workflow_scheduled_min_steps`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 27f71b7ee47..d243b59f352 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -445,7 +445,7 @@ class Configuration(object): self.message_box_class = kwargs.get('message_box_class', 'info') self.support_url = kwargs.get('support_url', 'https://galaxyproject.org/support') self.citation_url = kwargs.get('citation_url', 'https://galaxyproject.org/citing-galaxy') - self.helpsite_url = kwargs.get('helpsite_url', 'https://help.galaxyproject.org/') + self.helpsite_url = kwargs.get('helpsite_url', None) self.wiki_url = kwargs.get('wiki_url', 'https://galaxyproject.org/') self.blog_url = kwargs.get('blog_url', None) self.screencasts_url = kwargs.get('screencasts_url', None) diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 3ef85b537c3..ad44ea8d7c5 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -999,7 +999,7 @@ mapping: helpsite_url: type: str - default: https://help.galaxyproject.org/ + default: '' required: false desc: | The URL linked by the "Galaxy Help" link in the "Help" menu. diff --git a/static/style/blue/base.css b/static/style/blue/base.css index d93ba458ad1..3848129a62c 100644 --- a/static/style/blue/base.css +++ b/static/style/blue/base.css @@ -1039,7 +1039,8 @@ pre { margin-bottom: 1rem; background-color: transparent; } .table th, .ui-table-striped th, - .table td, .ui-table-striped td { + .table td, + .ui-table-striped td { padding: 0.75rem; vertical-align: top; border-top: 1px solid #bfbfbf; } @@ -1324,50 +1325,26 @@ select.form-control:focus::-ms-value { .form-control-plaintext.form-control-sm, .input-group-sm > .form-control-plaintext.form-control, .input-group-sm > .input-group-prepend > .form-control-plaintext.input-group-text, .input-group-sm > .input-group-append > .form-control-plaintext.input-group-text, - .input-group-sm > .input-group-prepend > .form-control-plaintext.btn, - .input-group-sm > .input-group-prepend > input.form-control-plaintext[type="submit"], - .input-group-sm > .input-group-prepend > - button.form-control-plaintext, - .input-group-sm > .input-group-prepend > .form-control-plaintext.action-button, - .input-group-sm > .input-group-prepend > .form-control-plaintext.menubutton, - .input-group-sm > .input-group-append > .form-control-plaintext.btn, - .input-group-sm > .input-group-append > input.form-control-plaintext[type="submit"], - .input-group-sm > .input-group-append > - button.form-control-plaintext, - .input-group-sm > .input-group-append > .form-control-plaintext.action-button, - .input-group-sm > .input-group-append > .form-control-plaintext.menubutton, .form-control-plaintext.form-control-lg, .input-group-lg > .form-control-plaintext.form-control, + .input-group-sm > .input-group-prepend > .form-control-plaintext.btn, .input-group-sm > .input-group-prepend > input.form-control-plaintext[type="submit"], + .input-group-sm > .input-group-prepend > button.form-control-plaintext, .input-group-sm > .input-group-prepend > .form-control-plaintext.action-button, .input-group-sm > .input-group-prepend > .form-control-plaintext.menubutton, + .input-group-sm > .input-group-append > .form-control-plaintext.btn, .input-group-sm > .input-group-append > input.form-control-plaintext[type="submit"], + .input-group-sm > .input-group-append > button.form-control-plaintext, .input-group-sm > .input-group-append > .form-control-plaintext.action-button, .input-group-sm > .input-group-append > .form-control-plaintext.menubutton, .form-control-plaintext.form-control-lg, .input-group-lg > .form-control-plaintext.form-control, .input-group-lg > .input-group-prepend > .form-control-plaintext.input-group-text, .input-group-lg > .input-group-append > .form-control-plaintext.input-group-text, - .input-group-lg > .input-group-prepend > .form-control-plaintext.btn, - .input-group-lg > .input-group-prepend > input.form-control-plaintext[type="submit"], - .input-group-lg > .input-group-prepend > - button.form-control-plaintext, - .input-group-lg > .input-group-prepend > .form-control-plaintext.action-button, - .input-group-lg > .input-group-prepend > .form-control-plaintext.menubutton, - .input-group-lg > .input-group-append > .form-control-plaintext.btn, - .input-group-lg > .input-group-append > input.form-control-plaintext[type="submit"], - .input-group-lg > .input-group-append > - button.form-control-plaintext, - .input-group-lg > .input-group-append > .form-control-plaintext.action-button, - .input-group-lg > .input-group-append > .form-control-plaintext.menubutton { + .input-group-lg > .input-group-prepend > .form-control-plaintext.btn, .input-group-lg > .input-group-prepend > input.form-control-plaintext[type="submit"], + .input-group-lg > .input-group-prepend > button.form-control-plaintext, .input-group-lg > .input-group-prepend > .form-control-plaintext.action-button, .input-group-lg > .input-group-prepend > .form-control-plaintext.menubutton, + .input-group-lg > .input-group-append > .form-control-plaintext.btn, .input-group-lg > .input-group-append > input.form-control-plaintext[type="submit"], + .input-group-lg > .input-group-append > button.form-control-plaintext, .input-group-lg > .input-group-append > .form-control-plaintext.action-button, .input-group-lg > .input-group-append > .form-control-plaintext.menubutton { padding-right: 0; padding-left: 0; } .form-control-sm, .input-group-sm > .form-control, .input-group-sm > .input-group-prepend > .input-group-text, .input-group-sm > .input-group-append > .input-group-text, -.input-group-sm > .input-group-prepend > .btn, -.input-group-sm > .input-group-prepend > input[type="submit"], -.input-group-sm > .input-group-prepend > -button, -.input-group-sm > .input-group-prepend > .action-button, -.input-group-sm > .input-group-prepend > .menubutton, -.input-group-sm > .input-group-append > .btn, -.input-group-sm > .input-group-append > input[type="submit"], -.input-group-sm > .input-group-append > -button, -.input-group-sm > .input-group-append > .action-button, -.input-group-sm > .input-group-append > .menubutton { +.input-group-sm > .input-group-prepend > .btn, .input-group-sm > .input-group-prepend > input[type="submit"], +.input-group-sm > .input-group-prepend > button, .input-group-sm > .input-group-prepend > .action-button, .input-group-sm > .input-group-prepend > .menubutton, +.input-group-sm > .input-group-append > .btn, .input-group-sm > .input-group-append > input[type="submit"], +.input-group-sm > .input-group-append > button, .input-group-sm > .input-group-append > .action-button, .input-group-sm > .input-group-append > .menubutton { padding: 0.25rem 0.5rem; font-size: 0.65625rem; line-height: 1.5; @@ -1376,29 +1353,17 @@ button, select.form-control-sm:not([size]):not([multiple]), .input-group-sm > select.form-control:not([size]):not([multiple]), .input-group-sm > .input-group-prepend > select.input-group-text:not([size]):not([multiple]), .input-group-sm > .input-group-append > select.input-group-text:not([size]):not([multiple]), -.input-group-sm > .input-group-prepend > select.btn:not([size]):not([multiple]), -.input-group-sm > .input-group-prepend > select.action-button:not([size]):not([multiple]), -.input-group-sm > .input-group-prepend > select.menubutton:not([size]):not([multiple]), -.input-group-sm > .input-group-append > select.btn:not([size]):not([multiple]), -.input-group-sm > .input-group-append > select.action-button:not([size]):not([multiple]), -.input-group-sm > .input-group-append > select.menubutton:not([size]):not([multiple]) { +.input-group-sm > .input-group-prepend > select.btn:not([size]):not([multiple]), .input-group-sm > .input-group-prepend > select.action-button:not([size]):not([multiple]), .input-group-sm > .input-group-prepend > select.menubutton:not([size]):not([multiple]), +.input-group-sm > .input-group-append > select.btn:not([size]):not([multiple]), .input-group-sm > .input-group-append > select.action-button:not([size]):not([multiple]), .input-group-sm > .input-group-append > select.menubutton:not([size]):not([multiple]) { height: calc(1.48438rem + 2px); } .form-control-lg, .input-group-lg > .form-control, .input-group-lg > .input-group-prepend > .input-group-text, .input-group-lg > .input-group-append > .input-group-text, -.input-group-lg > .input-group-prepend > .btn, -.input-group-lg > .input-group-prepend > input[type="submit"], -.input-group-lg > .input-group-prepend > -button, -.input-group-lg > .input-group-prepend > .action-button, -.input-group-lg > .input-group-prepend > .menubutton, -.input-group-lg > .input-group-append > .btn, -.input-group-lg > .input-group-append > input[type="submit"], -.input-group-lg > .input-group-append > -button, -.input-group-lg > .input-group-append > .action-button, -.input-group-lg > .input-group-append > .menubutton { +.input-group-lg > .input-group-prepend > .btn, .input-group-lg > .input-group-prepend > input[type="submit"], +.input-group-lg > .input-group-prepend > button, .input-group-lg > .input-group-prepend > .action-button, .input-group-lg > .input-group-prepend > .menubutton, +.input-group-lg > .input-group-append > .btn, .input-group-lg > .input-group-append > input[type="submit"], +.input-group-lg > .input-group-append > button, .input-group-lg > .input-group-append > .action-button, .input-group-lg > .input-group-append > .menubutton { padding: 0.5rem 1rem; font-size: 0.9375rem; line-height: 1.5; @@ -1407,12 +1372,8 @@ button, select.form-control-lg:not([size]):not([multiple]), .input-group-lg > select.form-control:not([size]):not([multiple]), .input-group-lg > .input-group-prepend > select.input-group-text:not([size]):not([multiple]), .input-group-lg > .input-group-append > select.input-group-text:not([size]):not([multiple]), -.input-group-lg > .input-group-prepend > select.btn:not([size]):not([multiple]), -.input-group-lg > .input-group-prepend > select.action-button:not([size]):not([multiple]), -.input-group-lg > .input-group-prepend > select.menubutton:not([size]):not([multiple]), -.input-group-lg > .input-group-append > select.btn:not([size]):not([multiple]), -.input-group-lg > .input-group-append > select.action-button:not([size]):not([multiple]), -.input-group-lg > .input-group-append > select.menubutton:not([size]):not([multiple]) { +.input-group-lg > .input-group-prepend > select.btn:not([size]):not([multiple]), .input-group-lg > .input-group-prepend > select.action-button:not([size]):not([multiple]), .input-group-lg > .input-group-prepend > select.menubutton:not([size]):not([multiple]), +.input-group-lg > .input-group-append > select.btn:not([size]):not([multiple]), .input-group-lg > .input-group-append > select.action-button:not([size]):not([multiple]), .input-group-lg > .input-group-append > select.menubutton:not([size]):not([multiple]) { height: calc(2.40625rem + 2px); } .form-group { @@ -1684,24 +1645,24 @@ button, .action-button, .menubutton { .btn, input[type="submit"], button, .action-button, .menubutton { transition: none; } } - .btn:hover, input[type="submit"]:hover, - button:hover, .action-button:hover, .menubutton:hover, .btn:focus, input[type="submit"]:focus, + .btn:hover, input:hover[type="submit"], + button:hover, .action-button:hover, .menubutton:hover, .btn:focus, input:focus[type="submit"], button:focus, .action-button:focus, .menubutton:focus { text-decoration: none; } - .btn:focus, input[type="submit"]:focus, + .btn:focus, input:focus[type="submit"], button:focus, .action-button:focus, .menubutton:focus, .btn.focus, input.focus[type="submit"], button.focus, .focus.action-button, .focus.menubutton { outline: 0; box-shadow: 0 0 0 0.2rem rgba(37, 83, 123, 0.25); } .btn.disabled, input.disabled[type="submit"], - button.disabled, .disabled.action-button, .disabled.menubutton, .btn:disabled, input[type="submit"]:disabled, + button.disabled, .disabled.action-button, .disabled.menubutton, .btn:disabled, input:disabled[type="submit"], button:disabled, .action-button:disabled, .menubutton:disabled { opacity: 0.65; } - .btn:not(:disabled):not(.disabled), input[type="submit"]:not(:disabled):not(.disabled), + .btn:not(:disabled):not(.disabled), input:not(:disabled):not(.disabled)[type="submit"], button:not(:disabled):not(.disabled), .action-button:not(:disabled):not(.disabled), .menubutton:not(:disabled):not(.disabled) { cursor: pointer; } - .btn:not(:disabled):not(.disabled):active, input[type="submit"]:not(:disabled):not(.disabled):active, - button:not(:disabled):not(.disabled):active, .action-button:not(:disabled):not(.disabled):active, .menubutton:not(:disabled):not(.disabled):active, .btn:not(:disabled):not(.disabled).active, input[type="submit"]:not(:disabled):not(.disabled).active, + .btn:not(:disabled):not(.disabled):active, input:not(:disabled):not(.disabled):active[type="submit"], + button:not(:disabled):not(.disabled):active, .action-button:not(:disabled):not(.disabled):active, .menubutton:not(:disabled):not(.disabled):active, .btn:not(:disabled):not(.disabled).active, input:not(:disabled):not(.disabled).active[type="submit"], button:not(:disabled):not(.disabled).active, .action-button:not(:disabled):not(.disabled).active, .action-button:not(:disabled):not(.disabled):active, .menubutton:not(:disabled):not(.disabled).active, .menubutton:not(:disabled):not(.disabled):active { background-image: none; } @@ -1711,18 +1672,18 @@ fieldset:disabled a.action-button, fieldset:disabled a.menubutton { pointer-events: none; } -.btn-primary, input[type="submit"].btn-primary, +.btn-primary, input.btn-primary[type="submit"], button.btn-primary { color: #fff; background-color: #25537b; border-color: #25537b; } - .btn-primary:hover, input[type="submit"].btn-primary:hover { + .btn-primary:hover, input.btn-primary:hover[type="submit"] { color: #fff; background-color: #1c3f5e; border-color: #193954; } - .btn-primary:focus, input[type="submit"].btn-primary:focus, .btn-primary.focus, input.focus[type="submit"].btn-primary { + .btn-primary:focus, input.btn-primary:focus[type="submit"], .btn-primary.focus, input.focus.btn-primary[type="submit"] { box-shadow: 0 0 0 0.2rem rgba(37, 83, 123, 0.5); } - .btn-primary.disabled, input.disabled[type="submit"].btn-primary, .btn-primary:disabled, input[type="submit"].btn-primary:disabled { + .btn-primary.disabled, input.disabled.btn-primary[type="submit"], .btn-primary:disabled, input.btn-primary:disabled[type="submit"] { color: #fff; background-color: #25537b; border-color: #25537b; } @@ -1740,41 +1701,39 @@ button, .action-button, .btn-default, .menubutton { color: #212529; background-color: #dee2e6; border-color: #dee2e6; } - .btn-secondary:hover, input[type="submit"]:hover, + .btn-secondary:hover, input:hover[type="submit"], button:hover, .action-button:hover, .btn-default:hover, .menubutton:hover { color: #212529; background-color: #c8cfd6; border-color: #c1c9d0; } - .btn-secondary:focus, input[type="submit"]:focus, + .btn-secondary:focus, input:focus[type="submit"], button:focus, .action-button:focus, .btn-default:focus, .menubutton:focus, .btn-secondary.focus, input.focus[type="submit"], button.focus, .focus.action-button, .focus.btn-default, .focus.menubutton { box-shadow: 0 0 0 0.2rem rgba(222, 226, 230, 0.5); } .btn-secondary.disabled, input.disabled[type="submit"], - button.disabled, .disabled.action-button, .disabled.btn-default, .disabled.menubutton, .btn-secondary:disabled, input[type="submit"]:disabled, + button.disabled, .disabled.action-button, .disabled.btn-default, .disabled.menubutton, .btn-secondary:disabled, input:disabled[type="submit"], button:disabled, .action-button:disabled, .btn-default:disabled, .menubutton:disabled { color: #212529; background-color: #dee2e6; border-color: #dee2e6; } - .btn-secondary:not(:disabled):not(.disabled):active, input[type="submit"]:not(:disabled):not(.disabled):active, - button:not(:disabled):not(.disabled):active, .action-button:not(:disabled):not(.disabled):active, .btn-default:not(:disabled):not(.disabled):active, .menubutton:not(:disabled):not(.disabled):active, .btn-secondary:not(:disabled):not(.disabled).active, input[type="submit"]:not(:disabled):not(.disabled).active, + .btn-secondary:not(:disabled):not(.disabled):active, input:not(:disabled):not(.disabled):active[type="submit"], + button:not(:disabled):not(.disabled):active, .action-button:not(:disabled):not(.disabled):active, .btn-default:not(:disabled):not(.disabled):active, .menubutton:not(:disabled):not(.disabled):active, .btn-secondary:not(:disabled):not(.disabled).active, input:not(:disabled):not(.disabled).active[type="submit"], button:not(:disabled):not(.disabled).active, .action-button:not(:disabled):not(.disabled).active, .btn-default:not(:disabled):not(.disabled).active, .menubutton:not(:disabled):not(.disabled).active, .show > .btn-secondary.dropdown-toggle, .show > input.dropdown-toggle[type="submit"], - .show > - button.dropdown-toggle, + .show > button.dropdown-toggle, .show > .dropdown-toggle.action-button, .show > .dropdown-toggle.btn-default, .show > .dropdown-toggle.menubutton { color: #212529; background-color: #c1c9d0; border-color: #bac2cb; } - .btn-secondary:not(:disabled):not(.disabled):active:focus, input[type="submit"]:not(:disabled):not(.disabled):active:focus, - button:not(:disabled):not(.disabled):active:focus, .action-button:not(:disabled):not(.disabled):active:focus, .btn-default:not(:disabled):not(.disabled):active:focus, .menubutton:not(:disabled):not(.disabled):active:focus, .btn-secondary:not(:disabled):not(.disabled).active:focus, input[type="submit"]:not(:disabled):not(.disabled).active:focus, + .btn-secondary:not(:disabled):not(.disabled):active:focus, input:not(:disabled):not(.disabled):active:focus[type="submit"], + button:not(:disabled):not(.disabled):active:focus, .action-button:not(:disabled):not(.disabled):active:focus, .btn-default:not(:disabled):not(.disabled):active:focus, .menubutton:not(:disabled):not(.disabled):active:focus, .btn-secondary:not(:disabled):not(.disabled).active:focus, input:not(:disabled):not(.disabled).active:focus[type="submit"], button:not(:disabled):not(.disabled).active:focus, .action-button:not(:disabled):not(.disabled).active:focus, .btn-default:not(:disabled):not(.disabled).active:focus, .menubutton:not(:disabled):not(.disabled).active:focus, .show > .btn-secondary.dropdown-toggle:focus, - .show > input.dropdown-toggle[type="submit"]:focus, - .show > - button.dropdown-toggle:focus, + .show > input.dropdown-toggle:focus[type="submit"], + .show > button.dropdown-toggle:focus, .show > .dropdown-toggle.action-button:focus, .show > .dropdown-toggle.btn-default:focus, .show > .dropdown-toggle.menubutton:focus { @@ -2119,17 +2078,17 @@ button, .action-button, .btn-default, .menubutton { color: #868e96; pointer-events: none; } -.btn-lg, .btn-group-lg > .btn, .btn-group-lg > input[type="submit"], .btn-group-lg > -button, .btn-group-lg > .action-button, .btn-group-lg > .menubutton { +.btn-lg, .btn-group-lg > .btn, .btn-group-lg > input[type="submit"], +.btn-group-lg > button, .btn-group-lg > .action-button, .btn-group-lg > .menubutton { padding: 0.5rem 1rem; font-size: 0.9375rem; line-height: 1.5; border-radius: 0.3rem; } -.btn-sm, .btn-group-sm > .btn, .btn-group-sm > input[type="submit"], .btn-group-sm > -button, .btn-group-sm > .action-button, .btn-group-sm > .menubutton, .btn-xs, .multi-panel-history .control-column .btn, .multi-panel-history .control-column input[type="submit"], .multi-panel-history .control-column -button, .multi-panel-history .control-column .action-button, .multi-panel-history .control-column .menubutton, .multi-panel-history .history-column .panel-controls .btn, .multi-panel-history .history-column .panel-controls input[type="submit"], .multi-panel-history .history-column .panel-controls -button, .multi-panel-history .history-column .panel-controls .action-button, .multi-panel-history .history-column .panel-controls .menubutton { +.btn-sm, .btn-group-sm > .btn, .btn-group-sm > input[type="submit"], +.btn-group-sm > button, .btn-group-sm > .action-button, .btn-group-sm > .menubutton, .btn-xs, .multi-panel-history .control-column .btn, .multi-panel-history .control-column input[type="submit"], +.multi-panel-history .control-column button, .multi-panel-history .control-column .action-button, .multi-panel-history .control-column .menubutton, .multi-panel-history .history-column .panel-controls .btn, .multi-panel-history .history-column .panel-controls input[type="submit"], +.multi-panel-history .history-column .panel-controls button, .multi-panel-history .history-column .panel-controls .action-button, .multi-panel-history .history-column .panel-controls .menubutton { padding: 0.25rem 0.5rem; font-size: 0.65625rem; line-height: 1.5; @@ -2345,119 +2304,95 @@ input[type="button"].btn-block { position: relative; display: inline-flex; vertical-align: middle; } - .btn-group > .btn, .btn-group > input[type="submit"], .btn-group > - button, .btn-group > .action-button, .btn-group > .menubutton, + .btn-group > .btn, .btn-group > input[type="submit"], + .btn-group > button, .btn-group > .action-button, .btn-group > .menubutton, .btn-group-vertical > .btn, .btn-group-vertical > input[type="submit"], - .btn-group-vertical > - button, + .btn-group-vertical > button, .btn-group-vertical > .action-button, .btn-group-vertical > .menubutton { position: relative; flex: 0 1 auto; } - .btn-group > .btn:hover, .btn-group > input[type="submit"]:hover, .btn-group > - button:hover, .btn-group > .action-button:hover, .btn-group > .menubutton:hover, + .btn-group > .btn:hover, .btn-group > input:hover[type="submit"], + .btn-group > button:hover, .btn-group > .action-button:hover, .btn-group > .menubutton:hover, .btn-group-vertical > .btn:hover, - .btn-group-vertical > input[type="submit"]:hover, - .btn-group-vertical > - button:hover, + .btn-group-vertical > input:hover[type="submit"], + .btn-group-vertical > button:hover, .btn-group-vertical > .action-button:hover, .btn-group-vertical > .menubutton:hover { z-index: 1; } - .btn-group > .btn:focus, .btn-group > input[type="submit"]:focus, .btn-group > - button:focus, .btn-group > .action-button:focus, .btn-group > .menubutton:focus, .btn-group > .btn:active, .btn-group > input[type="submit"]:active, .btn-group > - button:active, .btn-group > .action-button:active, .btn-group > .menubutton:active, .btn-group > .btn.active, .btn-group > input.active[type="submit"], .btn-group > - button.active, .btn-group > .active.action-button, .btn-group > .action-button:active, .btn-group > .active.menubutton, .btn-group > .menubutton:active, + .btn-group > .btn:focus, .btn-group > input:focus[type="submit"], + .btn-group > button:focus, .btn-group > .action-button:focus, .btn-group > .menubutton:focus, .btn-group > .btn:active, .btn-group > input:active[type="submit"], + .btn-group > button:active, .btn-group > .action-button:active, .btn-group > .menubutton:active, .btn-group > .btn.active, .btn-group > input.active[type="submit"], + .btn-group > button.active, .btn-group > .active.action-button, .btn-group > .action-button:active, .btn-group > .active.menubutton, .btn-group > .menubutton:active, .btn-group-vertical > .btn:focus, - .btn-group-vertical > input[type="submit"]:focus, - .btn-group-vertical > - button:focus, + .btn-group-vertical > input:focus[type="submit"], + .btn-group-vertical > button:focus, .btn-group-vertical > .action-button:focus, .btn-group-vertical > .menubutton:focus, .btn-group-vertical > .btn:active, - .btn-group-vertical > input[type="submit"]:active, - .btn-group-vertical > - button:active, + .btn-group-vertical > input:active[type="submit"], + .btn-group-vertical > button:active, .btn-group-vertical > .action-button:active, .btn-group-vertical > .menubutton:active, .btn-group-vertical > .btn.active, .btn-group-vertical > input.active[type="submit"], - .btn-group-vertical > - button.active, + .btn-group-vertical > button.active, .btn-group-vertical > .active.action-button, .btn-group-vertical > .action-button:active, .btn-group-vertical > .active.menubutton, .btn-group-vertical > .menubutton:active { z-index: 1; } - .btn-group .btn + .btn, .btn-group input[type="submit"] + .btn, .btn-group - button + .btn, .btn-group .action-button + .btn, .btn-group .menubutton + .btn, .btn-group .btn + input[type="submit"], .btn-group input[type="submit"] + input[type="submit"], .btn-group - button + input[type="submit"], .btn-group .action-button + input[type="submit"], .btn-group .menubutton + input[type="submit"], .btn-group .btn + - button, .btn-group input[type="submit"] + - button, .btn-group - button + - button, .btn-group .action-button + - button, .btn-group .menubutton + - button, .btn-group .btn + .action-button, .btn-group input[type="submit"] + .action-button, .btn-group - button + .action-button, .btn-group .action-button + .action-button, .btn-group .menubutton + .action-button, .btn-group .btn + .menubutton, .btn-group input[type="submit"] + .menubutton, .btn-group - button + .menubutton, .btn-group .action-button + .menubutton, .btn-group .menubutton + .menubutton, + .btn-group .btn + .btn, .btn-group input[type="submit"] + .btn, .btn-group button + .btn, .btn-group .action-button + .btn, .btn-group .menubutton + .btn, .btn-group .btn + input[type="submit"], .btn-group input[type="submit"] + input[type="submit"], .btn-group button + input[type="submit"], .btn-group .action-button + input[type="submit"], .btn-group .menubutton + input[type="submit"], + .btn-group .btn + button, + .btn-group input[type="submit"] + button, + .btn-group button + button, + .btn-group .action-button + button, + .btn-group .menubutton + button, .btn-group .btn + .action-button, .btn-group input[type="submit"] + .action-button, .btn-group button + .action-button, .btn-group .action-button + .action-button, .btn-group .menubutton + .action-button, .btn-group .btn + .menubutton, .btn-group input[type="submit"] + .menubutton, .btn-group button + .menubutton, .btn-group .action-button + .menubutton, .btn-group .menubutton + .menubutton, .btn-group .btn + .btn-group, .btn-group input[type="submit"] + .btn-group, - .btn-group - button + .btn-group, + .btn-group button + .btn-group, .btn-group .action-button + .btn-group, .btn-group .menubutton + .btn-group, .btn-group .btn-group + .btn, .btn-group .btn-group + input[type="submit"], - .btn-group .btn-group + - button, + .btn-group .btn-group + button, .btn-group .btn-group + .action-button, .btn-group .btn-group + .menubutton, .btn-group .btn-group + .btn-group, .btn-group-vertical .btn + .btn, .btn-group-vertical input[type="submit"] + .btn, - .btn-group-vertical - button + .btn, + .btn-group-vertical button + .btn, .btn-group-vertical .action-button + .btn, .btn-group-vertical .menubutton + .btn, .btn-group-vertical .btn + input[type="submit"], .btn-group-vertical input[type="submit"] + input[type="submit"], - .btn-group-vertical - button + input[type="submit"], + .btn-group-vertical button + input[type="submit"], .btn-group-vertical .action-button + input[type="submit"], .btn-group-vertical .menubutton + input[type="submit"], - .btn-group-vertical .btn + - button, - .btn-group-vertical input[type="submit"] + - button, - .btn-group-vertical - button + - button, - .btn-group-vertical .action-button + - button, - .btn-group-vertical .menubutton + - button, + .btn-group-vertical .btn + button, + .btn-group-vertical input[type="submit"] + button, + .btn-group-vertical button + button, + .btn-group-vertical .action-button + button, + .btn-group-vertical .menubutton + button, .btn-group-vertical .btn + .action-button, .btn-group-vertical input[type="submit"] + .action-button, - .btn-group-vertical - button + .action-button, + .btn-group-vertical button + .action-button, .btn-group-vertical .action-button + .action-button, .btn-group-vertical .menubutton + .action-button, .btn-group-vertical .btn + .menubutton, .btn-group-vertical input[type="submit"] + .menubutton, - .btn-group-vertical - button + .menubutton, + .btn-group-vertical button + .menubutton, .btn-group-vertical .action-button + .menubutton, .btn-group-vertical .menubutton + .menubutton, .btn-group-vertical .btn + .btn-group, .btn-group-vertical input[type="submit"] + .btn-group, - .btn-group-vertical - button + .btn-group, + .btn-group-vertical button + .btn-group, .btn-group-vertical .action-button + .btn-group, .btn-group-vertical .menubutton + .btn-group, .btn-group-vertical .btn-group + .btn, .btn-group-vertical .btn-group + input[type="submit"], - .btn-group-vertical .btn-group + - button, + .btn-group-vertical .btn-group + button, .btn-group-vertical .btn-group + .action-button, .btn-group-vertical .btn-group + .menubutton, .btn-group-vertical .btn-group + .btn-group { @@ -2470,27 +2405,25 @@ input[type="button"].btn-block { .btn-toolbar .input-group { width: auto; } -.btn-group > .btn:first-child, .btn-group > input[type="submit"]:first-child, .btn-group > -button:first-child, .btn-group > .action-button:first-child, .btn-group > .menubutton:first-child { +.btn-group > .btn:first-child, .btn-group > input:first-child[type="submit"], +.btn-group > button:first-child, .btn-group > .action-button:first-child, .btn-group > .menubutton:first-child { margin-left: 0; } -.btn-group > .btn:not(:last-child):not(.dropdown-toggle), .btn-group > input[type="submit"]:not(:last-child):not(.dropdown-toggle), .btn-group > -button:not(:last-child):not(.dropdown-toggle), .btn-group > .action-button:not(:last-child):not(.dropdown-toggle), .btn-group > .menubutton:not(:last-child):not(.dropdown-toggle), +.btn-group > .btn:not(:last-child):not(.dropdown-toggle), .btn-group > input:not(:last-child):not(.dropdown-toggle)[type="submit"], +.btn-group > button:not(:last-child):not(.dropdown-toggle), .btn-group > .action-button:not(:last-child):not(.dropdown-toggle), .btn-group > .menubutton:not(:last-child):not(.dropdown-toggle), .btn-group > .btn-group:not(:last-child) > .btn, .btn-group > .btn-group:not(:last-child) > input[type="submit"], -.btn-group > .btn-group:not(:last-child) > -button, +.btn-group > .btn-group:not(:last-child) > button, .btn-group > .btn-group:not(:last-child) > .action-button, .btn-group > .btn-group:not(:last-child) > .menubutton { border-top-right-radius: 0; border-bottom-right-radius: 0; } -.btn-group > .btn:not(:first-child), .btn-group > input[type="submit"]:not(:first-child), .btn-group > -button:not(:first-child), .btn-group > .action-button:not(:first-child), .btn-group > .menubutton:not(:first-child), +.btn-group > .btn:not(:first-child), .btn-group > input:not(:first-child)[type="submit"], +.btn-group > button:not(:first-child), .btn-group > .action-button:not(:first-child), .btn-group > .menubutton:not(:first-child), .btn-group > .btn-group:not(:first-child) > .btn, .btn-group > .btn-group:not(:first-child) > input[type="submit"], -.btn-group > .btn-group:not(:first-child) > -button, +.btn-group > .btn-group:not(:first-child) > button, .btn-group > .btn-group:not(:first-child) > .action-button, .btn-group > .btn-group:not(:first-child) > .menubutton { border-top-left-radius: 0; @@ -2506,15 +2439,11 @@ button, .dropleft .dropdown-toggle-split::before { margin-right: 0; } -.btn-sm + .dropdown-toggle-split, .btn-group-sm > .btn + .dropdown-toggle-split, .btn-group-sm > input[type="submit"] + .dropdown-toggle-split, .btn-group-sm > -button + .dropdown-toggle-split, .btn-group-sm > .action-button + .dropdown-toggle-split, .btn-group-sm > .menubutton + .dropdown-toggle-split, .btn-xs + .dropdown-toggle-split, .multi-panel-history .control-column .btn + .dropdown-toggle-split, .multi-panel-history .control-column input[type="submit"] + .dropdown-toggle-split, .multi-panel-history .control-column -button + .dropdown-toggle-split, .multi-panel-history .control-column .action-button + .dropdown-toggle-split, .multi-panel-history .control-column .menubutton + .dropdown-toggle-split, .multi-panel-history .history-column .panel-controls .btn + .dropdown-toggle-split, .multi-panel-history .history-column .panel-controls input[type="submit"] + .dropdown-toggle-split, .multi-panel-history .history-column .panel-controls -button + .dropdown-toggle-split, .multi-panel-history .history-column .panel-controls .action-button + .dropdown-toggle-split, .multi-panel-history .history-column .panel-controls .menubutton + .dropdown-toggle-split { +.btn-sm + .dropdown-toggle-split, .btn-group-sm > .btn + .dropdown-toggle-split, .btn-group-sm > input[type="submit"] + .dropdown-toggle-split, .btn-group-sm > button + .dropdown-toggle-split, .btn-group-sm > .action-button + .dropdown-toggle-split, .btn-group-sm > .menubutton + .dropdown-toggle-split, .btn-xs + .dropdown-toggle-split, .multi-panel-history .control-column .btn + .dropdown-toggle-split, .multi-panel-history .control-column input[type="submit"] + .dropdown-toggle-split, .multi-panel-history .control-column button + .dropdown-toggle-split, .multi-panel-history .control-column .action-button + .dropdown-toggle-split, .multi-panel-history .control-column .menubutton + .dropdown-toggle-split, .multi-panel-history .history-column .panel-controls .btn + .dropdown-toggle-split, .multi-panel-history .history-column .panel-controls input[type="submit"] + .dropdown-toggle-split, .multi-panel-history .history-column .panel-controls button + .dropdown-toggle-split, .multi-panel-history .history-column .panel-controls .action-button + .dropdown-toggle-split, .multi-panel-history .history-column .panel-controls .menubutton + .dropdown-toggle-split { padding-right: 0.375rem; padding-left: 0.375rem; } -.btn-lg + .dropdown-toggle-split, .btn-group-lg > .btn + .dropdown-toggle-split, .btn-group-lg > input[type="submit"] + .dropdown-toggle-split, .btn-group-lg > -button + .dropdown-toggle-split, .btn-group-lg > .action-button + .dropdown-toggle-split, .btn-group-lg > .menubutton + .dropdown-toggle-split { +.btn-lg + .dropdown-toggle-split, .btn-group-lg > .btn + .dropdown-toggle-split, .btn-group-lg > input[type="submit"] + .dropdown-toggle-split, .btn-group-lg > button + .dropdown-toggle-split, .btn-group-lg > .action-button + .dropdown-toggle-split, .btn-group-lg > .menubutton + .dropdown-toggle-split { padding-right: 0.75rem; padding-left: 0.75rem; } @@ -2522,84 +2451,70 @@ button + .dropdown-toggle-split, .btn-group-lg > .action-button + .dropdown-togg flex-direction: column; align-items: flex-start; justify-content: center; } - .btn-group-vertical .btn, .btn-group-vertical input[type="submit"], .btn-group-vertical - button, .btn-group-vertical .action-button, .btn-group-vertical .menubutton, + .btn-group-vertical .btn, .btn-group-vertical input[type="submit"], + .btn-group-vertical button, .btn-group-vertical .action-button, .btn-group-vertical .menubutton, .btn-group-vertical .btn-group { width: 100%; } - .btn-group-vertical > .btn + .btn, .btn-group-vertical > input[type="submit"] + .btn, .btn-group-vertical > - button + .btn, .btn-group-vertical > .action-button + .btn, .btn-group-vertical > .menubutton + .btn, .btn-group-vertical > .btn + input[type="submit"], .btn-group-vertical > input[type="submit"] + input[type="submit"], .btn-group-vertical > - button + input[type="submit"], .btn-group-vertical > .action-button + input[type="submit"], .btn-group-vertical > .menubutton + input[type="submit"], .btn-group-vertical > .btn + - button, .btn-group-vertical > input[type="submit"] + - button, .btn-group-vertical > - button + - button, .btn-group-vertical > .action-button + - button, .btn-group-vertical > .menubutton + - button, .btn-group-vertical > .btn + .action-button, .btn-group-vertical > input[type="submit"] + .action-button, .btn-group-vertical > - button + .action-button, .btn-group-vertical > .action-button + .action-button, .btn-group-vertical > .menubutton + .action-button, .btn-group-vertical > .btn + .menubutton, .btn-group-vertical > input[type="submit"] + .menubutton, .btn-group-vertical > - button + .menubutton, .btn-group-vertical > .action-button + .menubutton, .btn-group-vertical > .menubutton + .menubutton, + .btn-group-vertical > .btn + .btn, .btn-group-vertical > input[type="submit"] + .btn, .btn-group-vertical > button + .btn, .btn-group-vertical > .action-button + .btn, .btn-group-vertical > .menubutton + .btn, .btn-group-vertical > .btn + input[type="submit"], .btn-group-vertical > input[type="submit"] + input[type="submit"], .btn-group-vertical > button + input[type="submit"], .btn-group-vertical > .action-button + input[type="submit"], .btn-group-vertical > .menubutton + input[type="submit"], + .btn-group-vertical > .btn + button, + .btn-group-vertical > input[type="submit"] + button, + .btn-group-vertical > button + button, + .btn-group-vertical > .action-button + button, + .btn-group-vertical > .menubutton + button, .btn-group-vertical > .btn + .action-button, .btn-group-vertical > input[type="submit"] + .action-button, .btn-group-vertical > button + .action-button, .btn-group-vertical > .action-button + .action-button, .btn-group-vertical > .menubutton + .action-button, .btn-group-vertical > .btn + .menubutton, .btn-group-vertical > input[type="submit"] + .menubutton, .btn-group-vertical > button + .menubutton, .btn-group-vertical > .action-button + .menubutton, .btn-group-vertical > .menubutton + .menubutton, .btn-group-vertical > .btn + .btn-group, .btn-group-vertical > input[type="submit"] + .btn-group, - .btn-group-vertical > - button + .btn-group, + .btn-group-vertical > button + .btn-group, .btn-group-vertical > .action-button + .btn-group, .btn-group-vertical > .menubutton + .btn-group, .btn-group-vertical > .btn-group + .btn, .btn-group-vertical > .btn-group + input[type="submit"], - .btn-group-vertical > .btn-group + - button, + .btn-group-vertical > .btn-group + button, .btn-group-vertical > .btn-group + .action-button, .btn-group-vertical > .btn-group + .menubutton, .btn-group-vertical > .btn-group + .btn-group { margin-top: -1px; margin-left: 0; } - .btn-group-vertical > .btn:not(:last-child):not(.dropdown-toggle), .btn-group-vertical > input[type="submit"]:not(:last-child):not(.dropdown-toggle), .btn-group-vertical > - button:not(:last-child):not(.dropdown-toggle), .btn-group-vertical > .action-button:not(:last-child):not(.dropdown-toggle), .btn-group-vertical > .menubutton:not(:last-child):not(.dropdown-toggle), + .btn-group-vertical > .btn:not(:last-child):not(.dropdown-toggle), .btn-group-vertical > input:not(:last-child):not(.dropdown-toggle)[type="submit"], + .btn-group-vertical > button:not(:last-child):not(.dropdown-toggle), .btn-group-vertical > .action-button:not(:last-child):not(.dropdown-toggle), .btn-group-vertical > .menubutton:not(:last-child):not(.dropdown-toggle), .btn-group-vertical > .btn-group:not(:last-child) > .btn, .btn-group-vertical > .btn-group:not(:last-child) > input[type="submit"], - .btn-group-vertical > .btn-group:not(:last-child) > - button, + .btn-group-vertical > .btn-group:not(:last-child) > button, .btn-group-vertical > .btn-group:not(:last-child) > .action-button, .btn-group-vertical > .btn-group:not(:last-child) > .menubutton { border-bottom-right-radius: 0; border-bottom-left-radius: 0; } - .btn-group-vertical > .btn:not(:first-child), .btn-group-vertical > input[type="submit"]:not(:first-child), .btn-group-vertical > - button:not(:first-child), .btn-group-vertical > .action-button:not(:first-child), .btn-group-vertical > .menubutton:not(:first-child), + .btn-group-vertical > .btn:not(:first-child), .btn-group-vertical > input:not(:first-child)[type="submit"], + .btn-group-vertical > button:not(:first-child), .btn-group-vertical > .action-button:not(:first-child), .btn-group-vertical > .menubutton:not(:first-child), .btn-group-vertical > .btn-group:not(:first-child) > .btn, .btn-group-vertical > .btn-group:not(:first-child) > input[type="submit"], - .btn-group-vertical > .btn-group:not(:first-child) > - button, + .btn-group-vertical > .btn-group:not(:first-child) > button, .btn-group-vertical > .btn-group:not(:first-child) > .action-button, .btn-group-vertical > .btn-group:not(:first-child) > .menubutton { border-top-left-radius: 0; border-top-right-radius: 0; } -.btn-group-toggle > .btn, .btn-group-toggle > input[type="submit"], .btn-group-toggle > -button, .btn-group-toggle > .action-button, .btn-group-toggle > .menubutton, +.btn-group-toggle > .btn, .btn-group-toggle > input[type="submit"], +.btn-group-toggle > button, .btn-group-toggle > .action-button, .btn-group-toggle > .menubutton, .btn-group-toggle > .btn-group > .btn, .btn-group-toggle > .btn-group > input[type="submit"], -.btn-group-toggle > .btn-group > -button, +.btn-group-toggle > .btn-group > button, .btn-group-toggle > .btn-group > .action-button, .btn-group-toggle > .btn-group > .menubutton { margin-bottom: 0; } - .btn-group-toggle > .btn input[type="radio"], .btn-group-toggle > input[type="submit"] input[type="radio"], .btn-group-toggle > - button input[type="radio"], .btn-group-toggle > .action-button input[type="radio"], .btn-group-toggle > .menubutton input[type="radio"], + .btn-group-toggle > .btn input[type="radio"], .btn-group-toggle > input[type="submit"] input[type="radio"], .btn-group-toggle > button input[type="radio"], .btn-group-toggle > .action-button input[type="radio"], .btn-group-toggle > .menubutton input[type="radio"], .btn-group-toggle > .btn input[type="checkbox"], .btn-group-toggle > input[type="submit"] input[type="checkbox"], - .btn-group-toggle > - button input[type="checkbox"], + .btn-group-toggle > button input[type="checkbox"], .btn-group-toggle > .action-button input[type="checkbox"], .btn-group-toggle > .menubutton input[type="checkbox"], .btn-group-toggle > .btn-group > .btn input[type="radio"], .btn-group-toggle > .btn-group > input[type="submit"] input[type="radio"], - .btn-group-toggle > .btn-group > - button input[type="radio"], + .btn-group-toggle > .btn-group > button input[type="radio"], .btn-group-toggle > .btn-group > .action-button input[type="radio"], .btn-group-toggle > .btn-group > .menubutton input[type="radio"], .btn-group-toggle > .btn-group > .btn input[type="checkbox"], .btn-group-toggle > .btn-group > input[type="submit"] input[type="checkbox"], - .btn-group-toggle > .btn-group > - button input[type="checkbox"], + .btn-group-toggle > .btn-group > button input[type="checkbox"], .btn-group-toggle > .btn-group > .action-button input[type="checkbox"], .btn-group-toggle > .btn-group > .menubutton input[type="checkbox"] { position: absolute; @@ -2655,86 +2570,66 @@ button, .input-group-prepend, .input-group-append { display: flex; } - .input-group-prepend .btn, .input-group-prepend input[type="submit"], .input-group-prepend - button, .input-group-prepend .action-button, .input-group-prepend .menubutton, + .input-group-prepend .btn, .input-group-prepend input[type="submit"], + .input-group-prepend button, .input-group-prepend .action-button, .input-group-prepend .menubutton, .input-group-append .btn, .input-group-append input[type="submit"], - .input-group-append - button, + .input-group-append button, .input-group-append .action-button, .input-group-append .menubutton { position: relative; z-index: 2; } - .input-group-prepend .btn + .btn, .input-group-prepend input[type="submit"] + .btn, .input-group-prepend - button + .btn, .input-group-prepend .action-button + .btn, .input-group-prepend .menubutton + .btn, .input-group-prepend .btn + input[type="submit"], .input-group-prepend input[type="submit"] + input[type="submit"], .input-group-prepend - button + input[type="submit"], .input-group-prepend .action-button + input[type="submit"], .input-group-prepend .menubutton + input[type="submit"], .input-group-prepend .btn + - button, .input-group-prepend input[type="submit"] + - button, .input-group-prepend - button + - button, .input-group-prepend .action-button + - button, .input-group-prepend .menubutton + - button, .input-group-prepend .btn + .action-button, .input-group-prepend input[type="submit"] + .action-button, .input-group-prepend - button + .action-button, .input-group-prepend .action-button + .action-button, .input-group-prepend .menubutton + .action-button, .input-group-prepend .btn + .menubutton, .input-group-prepend input[type="submit"] + .menubutton, .input-group-prepend - button + .menubutton, .input-group-prepend .action-button + .menubutton, .input-group-prepend .menubutton + .menubutton, + .input-group-prepend .btn + .btn, .input-group-prepend input[type="submit"] + .btn, .input-group-prepend button + .btn, .input-group-prepend .action-button + .btn, .input-group-prepend .menubutton + .btn, .input-group-prepend .btn + input[type="submit"], .input-group-prepend input[type="submit"] + input[type="submit"], .input-group-prepend button + input[type="submit"], .input-group-prepend .action-button + input[type="submit"], .input-group-prepend .menubutton + input[type="submit"], + .input-group-prepend .btn + button, + .input-group-prepend input[type="submit"] + button, + .input-group-prepend button + button, + .input-group-prepend .action-button + button, + .input-group-prepend .menubutton + button, .input-group-prepend .btn + .action-button, .input-group-prepend input[type="submit"] + .action-button, .input-group-prepend button + .action-button, .input-group-prepend .action-button + .action-button, .input-group-prepend .menubutton + .action-button, .input-group-prepend .btn + .menubutton, .input-group-prepend input[type="submit"] + .menubutton, .input-group-prepend button + .menubutton, .input-group-prepend .action-button + .menubutton, .input-group-prepend .menubutton + .menubutton, .input-group-prepend .btn + .input-group-text, .input-group-prepend input[type="submit"] + .input-group-text, - .input-group-prepend - button + .input-group-text, + .input-group-prepend button + .input-group-text, .input-group-prepend .action-button + .input-group-text, .input-group-prepend .menubutton + .input-group-text, .input-group-prepend .input-group-text + .input-group-text, .input-group-prepend .input-group-text + .btn, .input-group-prepend .input-group-text + input[type="submit"], - .input-group-prepend .input-group-text + - button, + .input-group-prepend .input-group-text + button, .input-group-prepend .input-group-text + .action-button, .input-group-prepend .input-group-text + .menubutton, .input-group-append .btn + .btn, .input-group-append input[type="submit"] + .btn, - .input-group-append - button + .btn, + .input-group-append button + .btn, .input-group-append .action-button + .btn, .input-group-append .menubutton + .btn, .input-group-append .btn + input[type="submit"], .input-group-append input[type="submit"] + input[type="submit"], - .input-group-append - button + input[type="submit"], + .input-group-append button + input[type="submit"], .input-group-append .action-button + input[type="submit"], .input-group-append .menubutton + input[type="submit"], - .input-group-append .btn + - button, - .input-group-append input[type="submit"] + - button, - .input-group-append - button + - button, - .input-group-append .action-button + - button, - .input-group-append .menubutton + - button, + .input-group-append .btn + button, + .input-group-append input[type="submit"] + button, + .input-group-append button + button, + .input-group-append .action-button + button, + .input-group-append .menubutton + button, .input-group-append .btn + .action-button, .input-group-append input[type="submit"] + .action-button, - .input-group-append - button + .action-button, + .input-group-append button + .action-button, .input-group-append .action-button + .action-button, .input-group-append .menubutton + .action-button, .input-group-append .btn + .menubutton, .input-group-append input[type="submit"] + .menubutton, - .input-group-append - button + .menubutton, + .input-group-append button + .menubutton, .input-group-append .action-button + .menubutton, .input-group-append .menubutton + .menubutton, .input-group-append .btn + .input-group-text, .input-group-append input[type="submit"] + .input-group-text, - .input-group-append - button + .input-group-text, + .input-group-append button + .input-group-text, .input-group-append .action-button + .input-group-text, .input-group-append .menubutton + .input-group-text, .input-group-append .input-group-text + .input-group-text, .input-group-append .input-group-text + .btn, .input-group-append .input-group-text + input[type="submit"], - .input-group-append .input-group-text + - button, + .input-group-append .input-group-text + button, .input-group-append .input-group-text + .action-button, .input-group-append .input-group-text + .menubutton { margin-left: -1px; } @@ -2763,40 +2658,36 @@ button, .input-group-text input[type="checkbox"] { margin-top: 0; } -.input-group > .input-group-prepend > .btn, .input-group > .input-group-prepend > input[type="submit"], .input-group > .input-group-prepend > -button, .input-group > .input-group-prepend > .action-button, .input-group > .input-group-prepend > .menubutton, +.input-group > .input-group-prepend > .btn, .input-group > .input-group-prepend > input[type="submit"], +.input-group > .input-group-prepend > button, .input-group > .input-group-prepend > .action-button, .input-group > .input-group-prepend > .menubutton, .input-group > .input-group-prepend > .input-group-text, .input-group > .input-group-append:not(:last-child) > .btn, .input-group > .input-group-append:not(:last-child) > input[type="submit"], -.input-group > .input-group-append:not(:last-child) > -button, +.input-group > .input-group-append:not(:last-child) > button, .input-group > .input-group-append:not(:last-child) > .action-button, .input-group > .input-group-append:not(:last-child) > .menubutton, .input-group > .input-group-append:not(:last-child) > .input-group-text, .input-group > .input-group-append:last-child > .btn:not(:last-child):not(.dropdown-toggle), -.input-group > .input-group-append:last-child > input[type="submit"]:not(:last-child):not(.dropdown-toggle), -.input-group > .input-group-append:last-child > -button:not(:last-child):not(.dropdown-toggle), +.input-group > .input-group-append:last-child > input:not(:last-child):not(.dropdown-toggle)[type="submit"], +.input-group > .input-group-append:last-child > button:not(:last-child):not(.dropdown-toggle), .input-group > .input-group-append:last-child > .action-button:not(:last-child):not(.dropdown-toggle), .input-group > .input-group-append:last-child > .menubutton:not(:last-child):not(.dropdown-toggle), .input-group > .input-group-append:last-child > .input-group-text:not(:last-child) { border-top-right-radius: 0; border-bottom-right-radius: 0; } -.input-group > .input-group-append > .btn, .input-group > .input-group-append > input[type="submit"], .input-group > .input-group-append > -button, .input-group > .input-group-append > .action-button, .input-group > .input-group-append > .menubutton, +.input-group > .input-group-append > .btn, .input-group > .input-group-append > input[type="submit"], +.input-group > .input-group-append > button, .input-group > .input-group-append > .action-button, .input-group > .input-group-append > .menubutton, .input-group > .input-group-append > .input-group-text, .input-group > .input-group-prepend:not(:first-child) > .btn, .input-group > .input-group-prepend:not(:first-child) > input[type="submit"], -.input-group > .input-group-prepend:not(:first-child) > -button, +.input-group > .input-group-prepend:not(:first-child) > button, .input-group > .input-group-prepend:not(:first-child) > .action-button, .input-group > .input-group-prepend:not(:first-child) > .menubutton, .input-group > .input-group-prepend:not(:first-child) > .input-group-text, .input-group > .input-group-prepend:first-child > .btn:not(:first-child), -.input-group > .input-group-prepend:first-child > input[type="submit"]:not(:first-child), -.input-group > .input-group-prepend:first-child > -button:not(:first-child), +.input-group > .input-group-prepend:first-child > input:not(:first-child)[type="submit"], +.input-group > .input-group-prepend:first-child > button:not(:first-child), .input-group > .input-group-prepend:first-child > .action-button:not(:first-child), .input-group > .input-group-prepend:first-child > .menubutton:not(:first-child), .input-group > .input-group-prepend:first-child > .input-group-text:not(:first-child) { @@ -3683,8 +3574,7 @@ button:not(:first-child), .badge:empty { display: none; } -.btn .badge, input[type="submit"] .badge, -button .badge, .action-button .badge, .menubutton .badge { +.btn .badge, input[type="submit"] .badge, button .badge, .action-button .badge, .menubutton .badge { position: relative; top: -1px; } @@ -3720,7 +3610,7 @@ button .badge, .action-button .badge, .menubutton .badge { .badge-info, .badge-new { color: #fff; background-color: #1cbceb; } - .badge-info[href]:hover, [href].badge-new:hover, .badge-info[href]:focus, [href].badge-new:focus { + .badge-info[href]:hover, .badge-new[href]:hover, .badge-info[href]:focus, .badge-new[href]:focus { color: #fff; text-decoration: none; background-color: #119ac3; } @@ -3728,7 +3618,7 @@ button .badge, .action-button .badge, .menubutton .badge { .badge-warning, .badge-beta { color: #212529; background-color: #f69713; } - .badge-warning[href]:hover, [href].badge-beta:hover, .badge-warning[href]:focus, [href].badge-beta:focus { + .badge-warning[href]:hover, .badge-beta[href]:hover, .badge-warning[href]:focus, .badge-beta[href]:focus { color: #212529; text-decoration: none; background-color: #ce7b08; } @@ -3824,46 +3714,42 @@ button .badge, .action-button .badge, .menubutton .badge { .alert-secondary .alert-link { color: #6c6e70; } -.alert-success, .donemessagelarge, .donemessage, +.alert-success, +.donemessagelarge, +.donemessage, .donemessagesmall { color: #146f14; background-color: #b1e7b1; border-color: #64ce64; } - .alert-success hr, .donemessagelarge hr, .donemessage hr, - .donemessagesmall hr { + .alert-success hr, .donemessagelarge hr, .donemessage hr, .donemessagesmall hr { border-top-color: #51c851; } - .alert-success .alert-link, .donemessagelarge .alert-link, .donemessage .alert-link, - .donemessagesmall .alert-link { + .alert-success .alert-link, .donemessagelarge .alert-link, .donemessage .alert-link, .donemessagesmall .alert-link { color: #0c440c; } -.alert-info, .infomessagelarge, .infomessage, +.alert-info, +.infomessagelarge, +.infomessage, .infomessagesmall, .ui-form-help .note { color: #11718d; background-color: #b0e7f8; border-color: #60d0f1; } - .alert-info hr, .infomessagelarge hr, .infomessage hr, - .infomessagesmall hr, - .ui-form-help .note hr { + .alert-info hr, .infomessagelarge hr, .infomessage hr, .infomessagesmall hr, .ui-form-help .note hr { border-top-color: #49c9ef; } - .alert-info .alert-link, .infomessagelarge .alert-link, .infomessage .alert-link, - .infomessagesmall .alert-link, - .ui-form-help .note .alert-link { + .alert-info .alert-link, .infomessagelarge .alert-link, .infomessage .alert-link, .infomessagesmall .alert-link, .ui-form-help .note .alert-link { color: #0c4d5f; } -.alert-warning, .warningmessagelarge, .warningmessage, +.alert-warning, +.warningmessagelarge, +.warningmessage, .warningmessagesmall, .ui-form-help .warning { color: #945b0b; background-color: #fcdbac; border-color: #f9b65a; } - .alert-warning hr, .warningmessagelarge hr, .warningmessage hr, - .warningmessagesmall hr, - .ui-form-help .warning hr { + .alert-warning hr, .warningmessagelarge hr, .warningmessage hr, .warningmessagesmall hr, .ui-form-help .warning hr { border-top-color: #f8ab41; } - .alert-warning .alert-link, .warningmessagelarge .alert-link, .warningmessage .alert-link, - .warningmessagesmall .alert-link, - .ui-form-help .warning .alert-link { + .alert-warning .alert-link, .warningmessagelarge .alert-link, .warningmessage .alert-link, .warningmessagesmall .alert-link, .ui-form-help .warning .alert-link { color: #653e07; } .alert-danger, .errormessagelarge, .error-modal .modal-content, .errormessage, @@ -3872,13 +3758,9 @@ button .badge, .action-button .badge, .menubutton .badge { color: #8f3937; background-color: #f9c7c6; border-color: #f48f8c; } - .alert-danger hr, .errormessagelarge hr, .error-modal .modal-content hr, .errormessage hr, - .errormessagesmall hr, - .ui-form-help .error hr { + .alert-danger hr, .errormessagelarge hr, .error-modal .modal-content hr, .errormessage hr, .errormessagesmall hr, .ui-form-help .error hr { border-top-color: #f27875; } - .alert-danger .alert-link, .errormessagelarge .alert-link, .error-modal .modal-content .alert-link, .errormessage .alert-link, - .errormessagesmall .alert-link, - .ui-form-help .error .alert-link { + .alert-danger .alert-link, .errormessagelarge .alert-link, .error-modal .modal-content .alert-link, .errormessage .alert-link, .errormessagesmall .alert-link, .ui-form-help .error .alert-link { color: #6a2a29; } .alert-light { @@ -4345,12 +4227,15 @@ button.close { .bs-popover-top .arrow, .bs-popover-auto[x-placement^="top"] .arrow { bottom: calc((0.5rem + 1px) * -1); } .bs-popover-top .arrow::before, .bs-popover-auto[x-placement^="top"] .arrow::before, - .bs-popover-top .arrow::after, .bs-popover-auto[x-placement^="top"] .arrow::after { + .bs-popover-top .arrow::after, + .bs-popover-auto[x-placement^="top"] .arrow::after { border-width: 0.5rem 0.5rem 0; } .bs-popover-top .arrow::before, .bs-popover-auto[x-placement^="top"] .arrow::before { bottom: 0; border-top-color: rgba(0, 0, 0, 0.25); } - .bs-popover-top .arrow::after, .bs-popover-auto[x-placement^="top"] .arrow::after { + + .bs-popover-top .arrow::after, + .bs-popover-auto[x-placement^="top"] .arrow::after { bottom: 1px; border-top-color: #fff; } @@ -4362,12 +4247,15 @@ button.close { height: 1rem; margin: 0.3rem 0; } .bs-popover-right .arrow::before, .bs-popover-auto[x-placement^="right"] .arrow::before, - .bs-popover-right .arrow::after, .bs-popover-auto[x-placement^="right"] .arrow::after { + .bs-popover-right .arrow::after, + .bs-popover-auto[x-placement^="right"] .arrow::after { border-width: 0.5rem 0.5rem 0.5rem 0; } .bs-popover-right .arrow::before, .bs-popover-auto[x-placement^="right"] .arrow::before { left: 0; border-right-color: rgba(0, 0, 0, 0.25); } - .bs-popover-right .arrow::after, .bs-popover-auto[x-placement^="right"] .arrow::after { + + .bs-popover-right .arrow::after, + .bs-popover-auto[x-placement^="right"] .arrow::after { left: 1px; border-right-color: #fff; } @@ -4376,12 +4264,15 @@ button.close { .bs-popover-bottom .arrow, .bs-popover-auto[x-placement^="bottom"] .arrow { top: calc((0.5rem + 1px) * -1); } .bs-popover-bottom .arrow::before, .bs-popover-auto[x-placement^="bottom"] .arrow::before, - .bs-popover-bottom .arrow::after, .bs-popover-auto[x-placement^="bottom"] .arrow::after { + .bs-popover-bottom .arrow::after, + .bs-popover-auto[x-placement^="bottom"] .arrow::after { border-width: 0 0.5rem 0.5rem 0.5rem; } .bs-popover-bottom .arrow::before, .bs-popover-auto[x-placement^="bottom"] .arrow::before { top: 0; border-bottom-color: rgba(0, 0, 0, 0.25); } - .bs-popover-bottom .arrow::after, .bs-popover-auto[x-placement^="bottom"] .arrow::after { + + .bs-popover-bottom .arrow::after, + .bs-popover-auto[x-placement^="bottom"] .arrow::after { top: 1px; border-bottom-color: #fff; } .bs-popover-bottom .popover-header::before, .bs-popover-auto[x-placement^="bottom"] .popover-header::before { @@ -4402,12 +4293,15 @@ button.close { height: 1rem; margin: 0.3rem 0; } .bs-popover-left .arrow::before, .bs-popover-auto[x-placement^="left"] .arrow::before, - .bs-popover-left .arrow::after, .bs-popover-auto[x-placement^="left"] .arrow::after { + .bs-popover-left .arrow::after, + .bs-popover-auto[x-placement^="left"] .arrow::after { border-width: 0.5rem 0 0.5rem 0.5rem; } .bs-popover-left .arrow::before, .bs-popover-auto[x-placement^="left"] .arrow::before { right: 0; border-left-color: rgba(0, 0, 0, 0.25); } - .bs-popover-left .arrow::after, .bs-popover-auto[x-placement^="left"] .arrow::after { + + .bs-popover-left .arrow::after, + .bs-popover-auto[x-placement^="left"] .arrow::after { right: 1px; border-left-color: #fff; } @@ -6619,18 +6513,10 @@ a.text-dark:hover, .quota-meter-text a:hover, a.text-dark:focus, .quota-meter-te img { page-break-inside: avoid; } p, - h2, - h3, - .upload-view-default .upload-box .upload-helper, - .upload-view-composite .upload-box .upload-helper, - .ui-form-composite .ui-form-header { + h2, h3, .upload-view-default .upload-box .upload-helper, .upload-view-composite .upload-box .upload-helper, .ui-form-composite .ui-form-header { orphans: 3; widows: 3; } - h2, - h3, - .upload-view-default .upload-box .upload-helper, - .upload-view-composite .upload-box .upload-helper, - .ui-form-composite .ui-form-header { + h2, h3, .upload-view-default .upload-box .upload-helper, .upload-view-composite .upload-box .upload-helper, .ui-form-composite .ui-form-header { page-break-after: avoid; } @page { size: a3; } @@ -6644,8 +6530,9 @@ a.text-dark:hover, .quota-meter-text a:hover, a.text-dark:focus, .quota-meter-te border: 1px solid #000; } .table, .ui-table-striped { border-collapse: collapse !important; } - .table td, .ui-table-striped td, - .table th, .ui-table-striped th { + + .table td, + .ui-table-striped td, .table th, .ui-table-striped th { background-color: #fff !important; } .table-bordered th, .table-bordered td { @@ -10653,17 +10540,23 @@ html[dir="rtl"] .select2-container-multi .select2-search-choice-close { .ui-portlet .portlet-content .portlet-buttons, .ui-portlet-repeat .portlet-content .portlet-buttons, .ui-portlet-panel .portlet-content .portlet-buttons, .ui-portlet-section .portlet-content .portlet-buttons, .ui-portlet-limited .portlet-content .portlet-buttons, .ui-portlet-narrow .portlet-content .portlet-buttons, .ui-portlet-plain .portlet-content .portlet-buttons { margin-top: 10px; margin-bottom: 10px; } - .ui-portlet .portlet-content .portlet-buttons > .btn, .ui-portlet-repeat .portlet-content .portlet-buttons > .btn, .ui-portlet-panel .portlet-content .portlet-buttons > .btn, .ui-portlet-section .portlet-content .portlet-buttons > .btn, .ui-portlet-limited .portlet-content .portlet-buttons > .btn, .ui-portlet-narrow .portlet-content .portlet-buttons > .btn, .ui-portlet-plain .portlet-content .portlet-buttons > .btn, .ui-portlet .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-repeat .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-panel .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-section .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-limited .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-narrow .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-plain .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet .portlet-content .portlet-buttons > - button, .ui-portlet-repeat .portlet-content .portlet-buttons > - button, .ui-portlet-panel .portlet-content .portlet-buttons > - button, .ui-portlet-section .portlet-content .portlet-buttons > - button, .ui-portlet-limited .portlet-content .portlet-buttons > - button, .ui-portlet-narrow .portlet-content .portlet-buttons > - button, .ui-portlet-plain .portlet-content .portlet-buttons > - button, .ui-portlet .portlet-content .portlet-buttons > .action-button, .ui-portlet-repeat .portlet-content .portlet-buttons > .action-button, .ui-portlet-panel .portlet-content .portlet-buttons > .action-button, .ui-portlet-section .portlet-content .portlet-buttons > .action-button, .ui-portlet-limited .portlet-content .portlet-buttons > .action-button, .ui-portlet-narrow .portlet-content .portlet-buttons > .action-button, .ui-portlet-plain .portlet-content .portlet-buttons > .action-button, .ui-portlet .portlet-content .portlet-buttons > .menubutton, .ui-portlet-repeat .portlet-content .portlet-buttons > .menubutton, .ui-portlet-panel .portlet-content .portlet-buttons > .menubutton, .ui-portlet-section .portlet-content .portlet-buttons > .menubutton, .ui-portlet-limited .portlet-content .portlet-buttons > .menubutton, .ui-portlet-narrow .portlet-content .portlet-buttons > .menubutton, .ui-portlet-plain .portlet-content .portlet-buttons > .menubutton { + .ui-portlet .portlet-content .portlet-buttons > .btn, .ui-portlet-repeat .portlet-content .portlet-buttons > .btn, .ui-portlet-panel .portlet-content .portlet-buttons > .btn, .ui-portlet-section .portlet-content .portlet-buttons > .btn, .ui-portlet-limited .portlet-content .portlet-buttons > .btn, .ui-portlet-narrow .portlet-content .portlet-buttons > .btn, .ui-portlet-plain .portlet-content .portlet-buttons > .btn, .ui-portlet .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-repeat .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-panel .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-section .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-limited .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-narrow .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-plain .portlet-content .portlet-buttons > input[type="submit"], + .ui-portlet .portlet-content .portlet-buttons > button, + .ui-portlet-repeat .portlet-content .portlet-buttons > button, + .ui-portlet-panel .portlet-content .portlet-buttons > button, + .ui-portlet-section .portlet-content .portlet-buttons > button, + .ui-portlet-limited .portlet-content .portlet-buttons > button, + .ui-portlet-narrow .portlet-content .portlet-buttons > button, + .ui-portlet-plain .portlet-content .portlet-buttons > button, .ui-portlet .portlet-content .portlet-buttons > .action-button, .ui-portlet-repeat .portlet-content .portlet-buttons > .action-button, .ui-portlet-panel .portlet-content .portlet-buttons > .action-button, .ui-portlet-section .portlet-content .portlet-buttons > .action-button, .ui-portlet-limited .portlet-content .portlet-buttons > .action-button, .ui-portlet-narrow .portlet-content .portlet-buttons > .action-button, .ui-portlet-plain .portlet-content .portlet-buttons > .action-button, .ui-portlet .portlet-content .portlet-buttons > .menubutton, .ui-portlet-repeat .portlet-content .portlet-buttons > .menubutton, .ui-portlet-panel .portlet-content .portlet-buttons > .menubutton, .ui-portlet-section .portlet-content .portlet-buttons > .menubutton, .ui-portlet-limited .portlet-content .portlet-buttons > .menubutton, .ui-portlet-narrow .portlet-content .portlet-buttons > .menubutton, .ui-portlet-plain .portlet-content .portlet-buttons > .menubutton { margin-right: 5px; } .ui-portlet .portlet-content.nopadding, .ui-portlet-repeat .portlet-content.nopadding, .ui-portlet-panel .portlet-content.nopadding, .ui-portlet-section .portlet-content.nopadding, .ui-portlet-limited .portlet-content.nopadding, .ui-portlet-narrow .portlet-content.nopadding, .ui-portlet-plain .portlet-content.nopadding, - .ui-portlet .portlet-body.nopadding, .ui-portlet-repeat .portlet-body.nopadding, .ui-portlet-panel .portlet-body.nopadding, .ui-portlet-section .portlet-body.nopadding, .ui-portlet-limited .portlet-body.nopadding, .ui-portlet-narrow .portlet-body.nopadding, .ui-portlet-plain .portlet-body.nopadding { + .ui-portlet .portlet-body.nopadding, + .ui-portlet-repeat .portlet-body.nopadding, + .ui-portlet-panel .portlet-body.nopadding, + .ui-portlet-section .portlet-body.nopadding, + .ui-portlet-limited .portlet-body.nopadding, + .ui-portlet-narrow .portlet-body.nopadding, + .ui-portlet-plain .portlet-body.nopadding { padding: 0px; } .ui-portlet .portlet-backdrop, .ui-portlet-repeat .portlet-backdrop, .ui-portlet-panel .portlet-backdrop, .ui-portlet-section .portlet-backdrop, .ui-portlet-limited .portlet-backdrop, .ui-portlet-narrow .portlet-backdrop, .ui-portlet-plain .portlet-backdrop { display: none; @@ -10825,24 +10718,24 @@ html[dir="rtl"] .select2-container-multi .select2-search-choice-close { background: transparent; -webkit-appearance: none; -moz-border-radius: 0.1875rem; } - .ui-input[disabled], .ui-form-element [disabled].ui-form-preview, .ui-form-element-disabled [disabled].ui-form-preview, [disabled].ui-input-basic, .ui-select select[disabled], .ui-select .select2-container [disabled].select2-choice, [disabled].ui-textarea, .ui-options [disabled].ui-options-list, .ui-form-slider [disabled].ui-form-slider-text, .ui-select .select2-container-multi [disabled].select2-choices, .ui-input[readonly], .ui-form-element [readonly].ui-form-preview, .ui-form-element-disabled [readonly].ui-form-preview, [readonly].ui-input-basic, .ui-select select[readonly], .ui-select .select2-container [readonly].select2-choice, [readonly].ui-textarea, .ui-options [readonly].ui-options-list, .ui-form-slider [readonly].ui-form-slider-text, .ui-select .select2-container-multi [readonly].select2-choices, + .ui-input[disabled], .ui-form-element .ui-form-preview[disabled], .ui-form-element-disabled .ui-form-preview[disabled], .ui-input-basic[disabled], .ui-select select[disabled], .ui-select .select2-container .select2-choice[disabled], .ui-textarea[disabled], .ui-options .ui-options-list[disabled], .ui-form-slider .ui-form-slider-text[disabled], .ui-select .select2-container-multi .select2-choices[disabled], .ui-input[readonly], .ui-form-element .ui-form-preview[readonly], .ui-form-element-disabled .ui-form-preview[readonly], .ui-input-basic[readonly], .ui-select select[readonly], .ui-select .select2-container .select2-choice[readonly], .ui-textarea[readonly], .ui-options .ui-options-list[readonly], .ui-form-slider .ui-form-slider-text[readonly], .ui-select .select2-container-multi .select2-choices[readonly], fieldset[disabled] .ui-input, - fieldset[disabled] .ui-form-element .ui-form-preview, .ui-form-element - fieldset[disabled] .ui-form-preview, - fieldset[disabled] .ui-form-element-disabled .ui-form-preview, .ui-form-element-disabled - fieldset[disabled] .ui-form-preview, + fieldset[disabled] .ui-form-element .ui-form-preview, + .ui-form-element fieldset[disabled] .ui-form-preview, + fieldset[disabled] .ui-form-element-disabled .ui-form-preview, + .ui-form-element-disabled fieldset[disabled] .ui-form-preview, fieldset[disabled] .ui-input-basic, - fieldset[disabled] .ui-select select, .ui-select - fieldset[disabled] select, - fieldset[disabled] .ui-select .select2-container .select2-choice, .ui-select .select2-container - fieldset[disabled] .select2-choice, + fieldset[disabled] .ui-select select, + .ui-select fieldset[disabled] select, + fieldset[disabled] .ui-select .select2-container .select2-choice, + .ui-select .select2-container fieldset[disabled] .select2-choice, fieldset[disabled] .ui-textarea, - fieldset[disabled] .ui-options .ui-options-list, .ui-options - fieldset[disabled] .ui-options-list, - fieldset[disabled] .ui-form-slider .ui-form-slider-text, .ui-form-slider - fieldset[disabled] .ui-form-slider-text, - fieldset[disabled] .ui-select .select2-container-multi .select2-choices, .ui-select .select2-container-multi - fieldset[disabled] .select2-choices { + fieldset[disabled] .ui-options .ui-options-list, + .ui-options fieldset[disabled] .ui-options-list, + fieldset[disabled] .ui-form-slider .ui-form-slider-text, + .ui-form-slider fieldset[disabled] .ui-form-slider-text, + fieldset[disabled] .ui-select .select2-container-multi .select2-choices, + .ui-select .select2-container-multi fieldset[disabled] .select2-choices { cursor: not-allowed; background-color: #e9ecef; } @@ -12348,10 +12241,7 @@ button.toast-close-button { .flex-horizontal-container { flex-flow: row nowrap; } -.flex-row, -.flex-column, -.history-panel > .list-items, -.dataset-collection-panel > .list-items { +.flex-row, .flex-column, .history-panel > .list-items, .dataset-collection-panel > .list-items { -webkit-flex: 1 1 auto; -ms-flex: 1 1 auto; property: 1 1 auto; @@ -12954,7 +12844,8 @@ div.form-row-error-message { .ui-form-info { color: #666; } .form-row .help a, - .toolParamHelp a, .ui-form-info a { + .toolParamHelp a, + .ui-form-info a { color: #666; } .form-row.form-actions { @@ -13777,8 +13668,8 @@ div.permissionContainer { margin-bottom: 8px; } .list-panel .controls .list-actions { display: none; } - .list-panel .controls .list-actions .btn, .list-panel .controls .list-actions input[type="submit"], .list-panel .controls .list-actions - button, .list-panel .controls .list-actions .action-button, .list-panel .controls .list-actions .menubutton { + .list-panel .controls .list-actions .btn, .list-panel .controls .list-actions input[type="submit"], + .list-panel .controls .list-actions button, .list-panel .controls .list-actions .action-button, .list-panel .controls .list-actions .menubutton { padding-top: 2px; padding-bottom: 2px; font-size: 90%; } @@ -14440,20 +14331,19 @@ pre.peek { .multi-panel-history > .footer { width: 100%; background-color: lightgrey; } - .multi-panel-history > .header .btn, .multi-panel-history > .header input[type="submit"], .multi-panel-history > .header - button, .multi-panel-history > .header .action-button, .multi-panel-history > .header .menubutton, + .multi-panel-history > .header .btn, .multi-panel-history > .header input[type="submit"], + .multi-panel-history > .header button, .multi-panel-history > .header .action-button, .multi-panel-history > .header .menubutton, .multi-panel-history > .footer .btn, .multi-panel-history > .footer input[type="submit"], - .multi-panel-history > .footer - button, + .multi-panel-history > .footer button, .multi-panel-history > .footer .action-button, .multi-panel-history > .footer .menubutton { height: 21px; } .multi-panel-history > .header { min-height: 29px; max-height: 29px; } - .multi-panel-history > .header .more-options .btn, .multi-panel-history > .header .more-options input[type="submit"], .multi-panel-history > .header .more-options - button, .multi-panel-history > .header .more-options .action-button, .multi-panel-history > .header .more-options .menubutton { + .multi-panel-history > .header .more-options .btn, .multi-panel-history > .header .more-options input[type="submit"], + .multi-panel-history > .header .more-options button, .multi-panel-history > .header .more-options .action-button, .multi-panel-history > .header .more-options .menubutton { width: 100%; height: 26px; } .multi-panel-history > .header .more-options input[type="checkbox"] { @@ -14642,8 +14532,8 @@ pre.peek { margin: 0; } .collection-creator > *:not(.popover) { padding: 0px 8px 0px 8px; } - .collection-creator .btn, .collection-creator input[type="submit"], .collection-creator - button, .collection-creator .action-button, .collection-creator .menubutton { + .collection-creator .btn, .collection-creator input[type="submit"], + .collection-creator button, .collection-creator .action-button, .collection-creator .menubutton { border-color: #bfbfbf; } .collection-creator .vertically-spaced { margin-top: 8px; } @@ -14701,41 +14591,22 @@ pre.peek { .collection-creator .choose-filters button { width: 100%; margin-top: 2px; } - .collection-creator .header .alert, .collection-creator .header .errormessagelarge, .collection-creator .header .error-modal .modal-content, .error-modal .collection-creator .header .modal-content, .collection-creator .header - .warningmessagelarge, .collection-creator .header - .donemessagelarge, .collection-creator .header - .infomessagelarge, .collection-creator .header - .ui-form-help .error, - .ui-form-help .collection-creator .header .error, .collection-creator .header - .ui-form-help .warning, - .ui-form-help .collection-creator .header .warning, .collection-creator .header - .ui-form-help .note, - .ui-form-help .collection-creator .header .note, .collection-creator .header .errormessage, .collection-creator .header - .warningmessage, .collection-creator .header - .donemessage, .collection-creator .header - .infomessage, .collection-creator .header - .errormessagesmall, .collection-creator .header - .warningmessagesmall, .collection-creator .header - .donemessagesmall, .collection-creator .header - .infomessagesmall { + .collection-creator .header .alert, .collection-creator .header .errormessagelarge, .collection-creator .header .error-modal .modal-content, .error-modal .collection-creator .header .modal-content, + .collection-creator .header .warningmessagelarge, + .collection-creator .header .donemessagelarge, + .collection-creator .header .infomessagelarge, + .collection-creator .header .ui-form-help .error, .ui-form-help .collection-creator .header .error, + .collection-creator .header .ui-form-help .warning, .ui-form-help .collection-creator .header .warning, + .collection-creator .header .ui-form-help .note, .ui-form-help .collection-creator .header .note, .collection-creator .header .errormessage, + .collection-creator .header .warningmessage, + .collection-creator .header .donemessage, + .collection-creator .header .infomessage, + .collection-creator .header .errormessagesmall, + .collection-creator .header .warningmessagesmall, + .collection-creator .header .donemessagesmall, + .collection-creator .header .infomessagesmall { display: none; } - .collection-creator .header .alert li, .collection-creator .header .errormessagelarge li, .collection-creator .header .error-modal .modal-content li, .error-modal .collection-creator .header .modal-content li, .collection-creator .header - .warningmessagelarge li, .collection-creator .header - .donemessagelarge li, .collection-creator .header - .infomessagelarge li, .collection-creator .header - .ui-form-help .error li, - .ui-form-help .collection-creator .header .error li, .collection-creator .header - .ui-form-help .warning li, - .ui-form-help .collection-creator .header .warning li, .collection-creator .header - .ui-form-help .note li, - .ui-form-help .collection-creator .header .note li, .collection-creator .header .errormessage li, .collection-creator .header - .warningmessage li, .collection-creator .header - .donemessage li, .collection-creator .header - .infomessage li, .collection-creator .header - .errormessagesmall li, .collection-creator .header - .warningmessagesmall li, .collection-creator .header - .donemessagesmall li, .collection-creator .header - .infomessagesmall li { + .collection-creator .header .alert li, .collection-creator .header .errormessagelarge li, .collection-creator .header .error-modal .modal-content li, .error-modal .collection-creator .header .modal-content li, .collection-creator .header .warningmessagelarge li, .collection-creator .header .donemessagelarge li, .collection-creator .header .infomessagelarge li, .collection-creator .header .ui-form-help .error li, .ui-form-help .collection-creator .header .error li, .collection-creator .header .ui-form-help .warning li, .ui-form-help .collection-creator .header .warning li, .collection-creator .header .ui-form-help .note li, .ui-form-help .collection-creator .header .note li, .collection-creator .header .errormessage li, .collection-creator .header .warningmessage li, .collection-creator .header .donemessage li, .collection-creator .header .infomessage li, .collection-creator .header .errormessagesmall li, .collection-creator .header .warningmessagesmall li, .collection-creator .header .donemessagesmall li, .collection-creator .header .infomessagesmall li { list-style: circle; margin-left: 32px; } .collection-creator .column { From 4cde6b9b814acae02393c9fc6b432506cb9c429d Mon Sep 17 00:00:00 2001 From: guerler Date: Thu, 29 Nov 2018 13:23:06 -0500 Subject: [PATCH 031/260] Escape dataset names in job submission success message --- client/galaxy/scripts/mvc/tool/tool-form.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/galaxy/scripts/mvc/tool/tool-form.js b/client/galaxy/scripts/mvc/tool/tool-form.js index 38c71aa3a1a..d4f6f3ad0fa 100644 --- a/client/galaxy/scripts/mvc/tool/tool-form.js +++ b/client/galaxy/scripts/mvc/tool/tool-form.js @@ -353,7 +353,7 @@ var View = Backbone.View.extend({ if (list.length > 0) { blurb += `

${title}:

`; for (let item of list) { - let rowString = max > 0 ? `${item.hid}: ${item.name}` : "..."; + let rowString = max > 0 ? `${item.hid}: ${_.escape(item.name)}` : "..."; blurb += `

${rowString}

`; From d3679e2541045f25e6007efbaed5d9d11f91a8c4 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Fri, 30 Nov 2018 10:55:14 -0500 Subject: [PATCH 032/260] revert the changes to css, probably remnant from working on dev --- static/style/blue/base.css | 631 ++++++++++++++++++++++--------------- 1 file changed, 380 insertions(+), 251 deletions(-) diff --git a/static/style/blue/base.css b/static/style/blue/base.css index 3848129a62c..d93ba458ad1 100644 --- a/static/style/blue/base.css +++ b/static/style/blue/base.css @@ -1039,8 +1039,7 @@ pre { margin-bottom: 1rem; background-color: transparent; } .table th, .ui-table-striped th, - .table td, - .ui-table-striped td { + .table td, .ui-table-striped td { padding: 0.75rem; vertical-align: top; border-top: 1px solid #bfbfbf; } @@ -1325,26 +1324,50 @@ select.form-control:focus::-ms-value { .form-control-plaintext.form-control-sm, .input-group-sm > .form-control-plaintext.form-control, .input-group-sm > .input-group-prepend > .form-control-plaintext.input-group-text, .input-group-sm > .input-group-append > .form-control-plaintext.input-group-text, - .input-group-sm > .input-group-prepend > .form-control-plaintext.btn, .input-group-sm > .input-group-prepend > input.form-control-plaintext[type="submit"], - .input-group-sm > .input-group-prepend > button.form-control-plaintext, .input-group-sm > .input-group-prepend > .form-control-plaintext.action-button, .input-group-sm > .input-group-prepend > .form-control-plaintext.menubutton, - .input-group-sm > .input-group-append > .form-control-plaintext.btn, .input-group-sm > .input-group-append > input.form-control-plaintext[type="submit"], - .input-group-sm > .input-group-append > button.form-control-plaintext, .input-group-sm > .input-group-append > .form-control-plaintext.action-button, .input-group-sm > .input-group-append > .form-control-plaintext.menubutton, .form-control-plaintext.form-control-lg, .input-group-lg > .form-control-plaintext.form-control, + .input-group-sm > .input-group-prepend > .form-control-plaintext.btn, + .input-group-sm > .input-group-prepend > input.form-control-plaintext[type="submit"], + .input-group-sm > .input-group-prepend > + button.form-control-plaintext, + .input-group-sm > .input-group-prepend > .form-control-plaintext.action-button, + .input-group-sm > .input-group-prepend > .form-control-plaintext.menubutton, + .input-group-sm > .input-group-append > .form-control-plaintext.btn, + .input-group-sm > .input-group-append > input.form-control-plaintext[type="submit"], + .input-group-sm > .input-group-append > + button.form-control-plaintext, + .input-group-sm > .input-group-append > .form-control-plaintext.action-button, + .input-group-sm > .input-group-append > .form-control-plaintext.menubutton, .form-control-plaintext.form-control-lg, .input-group-lg > .form-control-plaintext.form-control, .input-group-lg > .input-group-prepend > .form-control-plaintext.input-group-text, .input-group-lg > .input-group-append > .form-control-plaintext.input-group-text, - .input-group-lg > .input-group-prepend > .form-control-plaintext.btn, .input-group-lg > .input-group-prepend > input.form-control-plaintext[type="submit"], - .input-group-lg > .input-group-prepend > button.form-control-plaintext, .input-group-lg > .input-group-prepend > .form-control-plaintext.action-button, .input-group-lg > .input-group-prepend > .form-control-plaintext.menubutton, - .input-group-lg > .input-group-append > .form-control-plaintext.btn, .input-group-lg > .input-group-append > input.form-control-plaintext[type="submit"], - .input-group-lg > .input-group-append > button.form-control-plaintext, .input-group-lg > .input-group-append > .form-control-plaintext.action-button, .input-group-lg > .input-group-append > .form-control-plaintext.menubutton { + .input-group-lg > .input-group-prepend > .form-control-plaintext.btn, + .input-group-lg > .input-group-prepend > input.form-control-plaintext[type="submit"], + .input-group-lg > .input-group-prepend > + button.form-control-plaintext, + .input-group-lg > .input-group-prepend > .form-control-plaintext.action-button, + .input-group-lg > .input-group-prepend > .form-control-plaintext.menubutton, + .input-group-lg > .input-group-append > .form-control-plaintext.btn, + .input-group-lg > .input-group-append > input.form-control-plaintext[type="submit"], + .input-group-lg > .input-group-append > + button.form-control-plaintext, + .input-group-lg > .input-group-append > .form-control-plaintext.action-button, + .input-group-lg > .input-group-append > .form-control-plaintext.menubutton { padding-right: 0; padding-left: 0; } .form-control-sm, .input-group-sm > .form-control, .input-group-sm > .input-group-prepend > .input-group-text, .input-group-sm > .input-group-append > .input-group-text, -.input-group-sm > .input-group-prepend > .btn, .input-group-sm > .input-group-prepend > input[type="submit"], -.input-group-sm > .input-group-prepend > button, .input-group-sm > .input-group-prepend > .action-button, .input-group-sm > .input-group-prepend > .menubutton, -.input-group-sm > .input-group-append > .btn, .input-group-sm > .input-group-append > input[type="submit"], -.input-group-sm > .input-group-append > button, .input-group-sm > .input-group-append > .action-button, .input-group-sm > .input-group-append > .menubutton { +.input-group-sm > .input-group-prepend > .btn, +.input-group-sm > .input-group-prepend > input[type="submit"], +.input-group-sm > .input-group-prepend > +button, +.input-group-sm > .input-group-prepend > .action-button, +.input-group-sm > .input-group-prepend > .menubutton, +.input-group-sm > .input-group-append > .btn, +.input-group-sm > .input-group-append > input[type="submit"], +.input-group-sm > .input-group-append > +button, +.input-group-sm > .input-group-append > .action-button, +.input-group-sm > .input-group-append > .menubutton { padding: 0.25rem 0.5rem; font-size: 0.65625rem; line-height: 1.5; @@ -1353,17 +1376,29 @@ select.form-control:focus::-ms-value { select.form-control-sm:not([size]):not([multiple]), .input-group-sm > select.form-control:not([size]):not([multiple]), .input-group-sm > .input-group-prepend > select.input-group-text:not([size]):not([multiple]), .input-group-sm > .input-group-append > select.input-group-text:not([size]):not([multiple]), -.input-group-sm > .input-group-prepend > select.btn:not([size]):not([multiple]), .input-group-sm > .input-group-prepend > select.action-button:not([size]):not([multiple]), .input-group-sm > .input-group-prepend > select.menubutton:not([size]):not([multiple]), -.input-group-sm > .input-group-append > select.btn:not([size]):not([multiple]), .input-group-sm > .input-group-append > select.action-button:not([size]):not([multiple]), .input-group-sm > .input-group-append > select.menubutton:not([size]):not([multiple]) { +.input-group-sm > .input-group-prepend > select.btn:not([size]):not([multiple]), +.input-group-sm > .input-group-prepend > select.action-button:not([size]):not([multiple]), +.input-group-sm > .input-group-prepend > select.menubutton:not([size]):not([multiple]), +.input-group-sm > .input-group-append > select.btn:not([size]):not([multiple]), +.input-group-sm > .input-group-append > select.action-button:not([size]):not([multiple]), +.input-group-sm > .input-group-append > select.menubutton:not([size]):not([multiple]) { height: calc(1.48438rem + 2px); } .form-control-lg, .input-group-lg > .form-control, .input-group-lg > .input-group-prepend > .input-group-text, .input-group-lg > .input-group-append > .input-group-text, -.input-group-lg > .input-group-prepend > .btn, .input-group-lg > .input-group-prepend > input[type="submit"], -.input-group-lg > .input-group-prepend > button, .input-group-lg > .input-group-prepend > .action-button, .input-group-lg > .input-group-prepend > .menubutton, -.input-group-lg > .input-group-append > .btn, .input-group-lg > .input-group-append > input[type="submit"], -.input-group-lg > .input-group-append > button, .input-group-lg > .input-group-append > .action-button, .input-group-lg > .input-group-append > .menubutton { +.input-group-lg > .input-group-prepend > .btn, +.input-group-lg > .input-group-prepend > input[type="submit"], +.input-group-lg > .input-group-prepend > +button, +.input-group-lg > .input-group-prepend > .action-button, +.input-group-lg > .input-group-prepend > .menubutton, +.input-group-lg > .input-group-append > .btn, +.input-group-lg > .input-group-append > input[type="submit"], +.input-group-lg > .input-group-append > +button, +.input-group-lg > .input-group-append > .action-button, +.input-group-lg > .input-group-append > .menubutton { padding: 0.5rem 1rem; font-size: 0.9375rem; line-height: 1.5; @@ -1372,8 +1407,12 @@ select.form-control-sm:not([size]):not([multiple]), .input-group-sm > select.for select.form-control-lg:not([size]):not([multiple]), .input-group-lg > select.form-control:not([size]):not([multiple]), .input-group-lg > .input-group-prepend > select.input-group-text:not([size]):not([multiple]), .input-group-lg > .input-group-append > select.input-group-text:not([size]):not([multiple]), -.input-group-lg > .input-group-prepend > select.btn:not([size]):not([multiple]), .input-group-lg > .input-group-prepend > select.action-button:not([size]):not([multiple]), .input-group-lg > .input-group-prepend > select.menubutton:not([size]):not([multiple]), -.input-group-lg > .input-group-append > select.btn:not([size]):not([multiple]), .input-group-lg > .input-group-append > select.action-button:not([size]):not([multiple]), .input-group-lg > .input-group-append > select.menubutton:not([size]):not([multiple]) { +.input-group-lg > .input-group-prepend > select.btn:not([size]):not([multiple]), +.input-group-lg > .input-group-prepend > select.action-button:not([size]):not([multiple]), +.input-group-lg > .input-group-prepend > select.menubutton:not([size]):not([multiple]), +.input-group-lg > .input-group-append > select.btn:not([size]):not([multiple]), +.input-group-lg > .input-group-append > select.action-button:not([size]):not([multiple]), +.input-group-lg > .input-group-append > select.menubutton:not([size]):not([multiple]) { height: calc(2.40625rem + 2px); } .form-group { @@ -1645,24 +1684,24 @@ button, .action-button, .menubutton { .btn, input[type="submit"], button, .action-button, .menubutton { transition: none; } } - .btn:hover, input:hover[type="submit"], - button:hover, .action-button:hover, .menubutton:hover, .btn:focus, input:focus[type="submit"], + .btn:hover, input[type="submit"]:hover, + button:hover, .action-button:hover, .menubutton:hover, .btn:focus, input[type="submit"]:focus, button:focus, .action-button:focus, .menubutton:focus { text-decoration: none; } - .btn:focus, input:focus[type="submit"], + .btn:focus, input[type="submit"]:focus, button:focus, .action-button:focus, .menubutton:focus, .btn.focus, input.focus[type="submit"], button.focus, .focus.action-button, .focus.menubutton { outline: 0; box-shadow: 0 0 0 0.2rem rgba(37, 83, 123, 0.25); } .btn.disabled, input.disabled[type="submit"], - button.disabled, .disabled.action-button, .disabled.menubutton, .btn:disabled, input:disabled[type="submit"], + button.disabled, .disabled.action-button, .disabled.menubutton, .btn:disabled, input[type="submit"]:disabled, button:disabled, .action-button:disabled, .menubutton:disabled { opacity: 0.65; } - .btn:not(:disabled):not(.disabled), input:not(:disabled):not(.disabled)[type="submit"], + .btn:not(:disabled):not(.disabled), input[type="submit"]:not(:disabled):not(.disabled), button:not(:disabled):not(.disabled), .action-button:not(:disabled):not(.disabled), .menubutton:not(:disabled):not(.disabled) { cursor: pointer; } - .btn:not(:disabled):not(.disabled):active, input:not(:disabled):not(.disabled):active[type="submit"], - button:not(:disabled):not(.disabled):active, .action-button:not(:disabled):not(.disabled):active, .menubutton:not(:disabled):not(.disabled):active, .btn:not(:disabled):not(.disabled).active, input:not(:disabled):not(.disabled).active[type="submit"], + .btn:not(:disabled):not(.disabled):active, input[type="submit"]:not(:disabled):not(.disabled):active, + button:not(:disabled):not(.disabled):active, .action-button:not(:disabled):not(.disabled):active, .menubutton:not(:disabled):not(.disabled):active, .btn:not(:disabled):not(.disabled).active, input[type="submit"]:not(:disabled):not(.disabled).active, button:not(:disabled):not(.disabled).active, .action-button:not(:disabled):not(.disabled).active, .action-button:not(:disabled):not(.disabled):active, .menubutton:not(:disabled):not(.disabled).active, .menubutton:not(:disabled):not(.disabled):active { background-image: none; } @@ -1672,18 +1711,18 @@ fieldset:disabled a.action-button, fieldset:disabled a.menubutton { pointer-events: none; } -.btn-primary, input.btn-primary[type="submit"], +.btn-primary, input[type="submit"].btn-primary, button.btn-primary { color: #fff; background-color: #25537b; border-color: #25537b; } - .btn-primary:hover, input.btn-primary:hover[type="submit"] { + .btn-primary:hover, input[type="submit"].btn-primary:hover { color: #fff; background-color: #1c3f5e; border-color: #193954; } - .btn-primary:focus, input.btn-primary:focus[type="submit"], .btn-primary.focus, input.focus.btn-primary[type="submit"] { + .btn-primary:focus, input[type="submit"].btn-primary:focus, .btn-primary.focus, input.focus[type="submit"].btn-primary { box-shadow: 0 0 0 0.2rem rgba(37, 83, 123, 0.5); } - .btn-primary.disabled, input.disabled.btn-primary[type="submit"], .btn-primary:disabled, input.btn-primary:disabled[type="submit"] { + .btn-primary.disabled, input.disabled[type="submit"].btn-primary, .btn-primary:disabled, input[type="submit"].btn-primary:disabled { color: #fff; background-color: #25537b; border-color: #25537b; } @@ -1701,39 +1740,41 @@ button, .action-button, .btn-default, .menubutton { color: #212529; background-color: #dee2e6; border-color: #dee2e6; } - .btn-secondary:hover, input:hover[type="submit"], + .btn-secondary:hover, input[type="submit"]:hover, button:hover, .action-button:hover, .btn-default:hover, .menubutton:hover { color: #212529; background-color: #c8cfd6; border-color: #c1c9d0; } - .btn-secondary:focus, input:focus[type="submit"], + .btn-secondary:focus, input[type="submit"]:focus, button:focus, .action-button:focus, .btn-default:focus, .menubutton:focus, .btn-secondary.focus, input.focus[type="submit"], button.focus, .focus.action-button, .focus.btn-default, .focus.menubutton { box-shadow: 0 0 0 0.2rem rgba(222, 226, 230, 0.5); } .btn-secondary.disabled, input.disabled[type="submit"], - button.disabled, .disabled.action-button, .disabled.btn-default, .disabled.menubutton, .btn-secondary:disabled, input:disabled[type="submit"], + button.disabled, .disabled.action-button, .disabled.btn-default, .disabled.menubutton, .btn-secondary:disabled, input[type="submit"]:disabled, button:disabled, .action-button:disabled, .btn-default:disabled, .menubutton:disabled { color: #212529; background-color: #dee2e6; border-color: #dee2e6; } - .btn-secondary:not(:disabled):not(.disabled):active, input:not(:disabled):not(.disabled):active[type="submit"], - button:not(:disabled):not(.disabled):active, .action-button:not(:disabled):not(.disabled):active, .btn-default:not(:disabled):not(.disabled):active, .menubutton:not(:disabled):not(.disabled):active, .btn-secondary:not(:disabled):not(.disabled).active, input:not(:disabled):not(.disabled).active[type="submit"], + .btn-secondary:not(:disabled):not(.disabled):active, input[type="submit"]:not(:disabled):not(.disabled):active, + button:not(:disabled):not(.disabled):active, .action-button:not(:disabled):not(.disabled):active, .btn-default:not(:disabled):not(.disabled):active, .menubutton:not(:disabled):not(.disabled):active, .btn-secondary:not(:disabled):not(.disabled).active, input[type="submit"]:not(:disabled):not(.disabled).active, button:not(:disabled):not(.disabled).active, .action-button:not(:disabled):not(.disabled).active, .btn-default:not(:disabled):not(.disabled).active, .menubutton:not(:disabled):not(.disabled).active, .show > .btn-secondary.dropdown-toggle, .show > input.dropdown-toggle[type="submit"], - .show > button.dropdown-toggle, + .show > + button.dropdown-toggle, .show > .dropdown-toggle.action-button, .show > .dropdown-toggle.btn-default, .show > .dropdown-toggle.menubutton { color: #212529; background-color: #c1c9d0; border-color: #bac2cb; } - .btn-secondary:not(:disabled):not(.disabled):active:focus, input:not(:disabled):not(.disabled):active:focus[type="submit"], - button:not(:disabled):not(.disabled):active:focus, .action-button:not(:disabled):not(.disabled):active:focus, .btn-default:not(:disabled):not(.disabled):active:focus, .menubutton:not(:disabled):not(.disabled):active:focus, .btn-secondary:not(:disabled):not(.disabled).active:focus, input:not(:disabled):not(.disabled).active:focus[type="submit"], + .btn-secondary:not(:disabled):not(.disabled):active:focus, input[type="submit"]:not(:disabled):not(.disabled):active:focus, + button:not(:disabled):not(.disabled):active:focus, .action-button:not(:disabled):not(.disabled):active:focus, .btn-default:not(:disabled):not(.disabled):active:focus, .menubutton:not(:disabled):not(.disabled):active:focus, .btn-secondary:not(:disabled):not(.disabled).active:focus, input[type="submit"]:not(:disabled):not(.disabled).active:focus, button:not(:disabled):not(.disabled).active:focus, .action-button:not(:disabled):not(.disabled).active:focus, .btn-default:not(:disabled):not(.disabled).active:focus, .menubutton:not(:disabled):not(.disabled).active:focus, .show > .btn-secondary.dropdown-toggle:focus, - .show > input.dropdown-toggle:focus[type="submit"], - .show > button.dropdown-toggle:focus, + .show > input.dropdown-toggle[type="submit"]:focus, + .show > + button.dropdown-toggle:focus, .show > .dropdown-toggle.action-button:focus, .show > .dropdown-toggle.btn-default:focus, .show > .dropdown-toggle.menubutton:focus { @@ -2078,17 +2119,17 @@ button, .action-button, .btn-default, .menubutton { color: #868e96; pointer-events: none; } -.btn-lg, .btn-group-lg > .btn, .btn-group-lg > input[type="submit"], -.btn-group-lg > button, .btn-group-lg > .action-button, .btn-group-lg > .menubutton { +.btn-lg, .btn-group-lg > .btn, .btn-group-lg > input[type="submit"], .btn-group-lg > +button, .btn-group-lg > .action-button, .btn-group-lg > .menubutton { padding: 0.5rem 1rem; font-size: 0.9375rem; line-height: 1.5; border-radius: 0.3rem; } -.btn-sm, .btn-group-sm > .btn, .btn-group-sm > input[type="submit"], -.btn-group-sm > button, .btn-group-sm > .action-button, .btn-group-sm > .menubutton, .btn-xs, .multi-panel-history .control-column .btn, .multi-panel-history .control-column input[type="submit"], -.multi-panel-history .control-column button, .multi-panel-history .control-column .action-button, .multi-panel-history .control-column .menubutton, .multi-panel-history .history-column .panel-controls .btn, .multi-panel-history .history-column .panel-controls input[type="submit"], -.multi-panel-history .history-column .panel-controls button, .multi-panel-history .history-column .panel-controls .action-button, .multi-panel-history .history-column .panel-controls .menubutton { +.btn-sm, .btn-group-sm > .btn, .btn-group-sm > input[type="submit"], .btn-group-sm > +button, .btn-group-sm > .action-button, .btn-group-sm > .menubutton, .btn-xs, .multi-panel-history .control-column .btn, .multi-panel-history .control-column input[type="submit"], .multi-panel-history .control-column +button, .multi-panel-history .control-column .action-button, .multi-panel-history .control-column .menubutton, .multi-panel-history .history-column .panel-controls .btn, .multi-panel-history .history-column .panel-controls input[type="submit"], .multi-panel-history .history-column .panel-controls +button, .multi-panel-history .history-column .panel-controls .action-button, .multi-panel-history .history-column .panel-controls .menubutton { padding: 0.25rem 0.5rem; font-size: 0.65625rem; line-height: 1.5; @@ -2304,95 +2345,119 @@ input[type="button"].btn-block { position: relative; display: inline-flex; vertical-align: middle; } - .btn-group > .btn, .btn-group > input[type="submit"], - .btn-group > button, .btn-group > .action-button, .btn-group > .menubutton, + .btn-group > .btn, .btn-group > input[type="submit"], .btn-group > + button, .btn-group > .action-button, .btn-group > .menubutton, .btn-group-vertical > .btn, .btn-group-vertical > input[type="submit"], - .btn-group-vertical > button, + .btn-group-vertical > + button, .btn-group-vertical > .action-button, .btn-group-vertical > .menubutton { position: relative; flex: 0 1 auto; } - .btn-group > .btn:hover, .btn-group > input:hover[type="submit"], - .btn-group > button:hover, .btn-group > .action-button:hover, .btn-group > .menubutton:hover, + .btn-group > .btn:hover, .btn-group > input[type="submit"]:hover, .btn-group > + button:hover, .btn-group > .action-button:hover, .btn-group > .menubutton:hover, .btn-group-vertical > .btn:hover, - .btn-group-vertical > input:hover[type="submit"], - .btn-group-vertical > button:hover, + .btn-group-vertical > input[type="submit"]:hover, + .btn-group-vertical > + button:hover, .btn-group-vertical > .action-button:hover, .btn-group-vertical > .menubutton:hover { z-index: 1; } - .btn-group > .btn:focus, .btn-group > input:focus[type="submit"], - .btn-group > button:focus, .btn-group > .action-button:focus, .btn-group > .menubutton:focus, .btn-group > .btn:active, .btn-group > input:active[type="submit"], - .btn-group > button:active, .btn-group > .action-button:active, .btn-group > .menubutton:active, .btn-group > .btn.active, .btn-group > input.active[type="submit"], - .btn-group > button.active, .btn-group > .active.action-button, .btn-group > .action-button:active, .btn-group > .active.menubutton, .btn-group > .menubutton:active, + .btn-group > .btn:focus, .btn-group > input[type="submit"]:focus, .btn-group > + button:focus, .btn-group > .action-button:focus, .btn-group > .menubutton:focus, .btn-group > .btn:active, .btn-group > input[type="submit"]:active, .btn-group > + button:active, .btn-group > .action-button:active, .btn-group > .menubutton:active, .btn-group > .btn.active, .btn-group > input.active[type="submit"], .btn-group > + button.active, .btn-group > .active.action-button, .btn-group > .action-button:active, .btn-group > .active.menubutton, .btn-group > .menubutton:active, .btn-group-vertical > .btn:focus, - .btn-group-vertical > input:focus[type="submit"], - .btn-group-vertical > button:focus, + .btn-group-vertical > input[type="submit"]:focus, + .btn-group-vertical > + button:focus, .btn-group-vertical > .action-button:focus, .btn-group-vertical > .menubutton:focus, .btn-group-vertical > .btn:active, - .btn-group-vertical > input:active[type="submit"], - .btn-group-vertical > button:active, + .btn-group-vertical > input[type="submit"]:active, + .btn-group-vertical > + button:active, .btn-group-vertical > .action-button:active, .btn-group-vertical > .menubutton:active, .btn-group-vertical > .btn.active, .btn-group-vertical > input.active[type="submit"], - .btn-group-vertical > button.active, + .btn-group-vertical > + button.active, .btn-group-vertical > .active.action-button, .btn-group-vertical > .action-button:active, .btn-group-vertical > .active.menubutton, .btn-group-vertical > .menubutton:active { z-index: 1; } - .btn-group .btn + .btn, .btn-group input[type="submit"] + .btn, .btn-group button + .btn, .btn-group .action-button + .btn, .btn-group .menubutton + .btn, .btn-group .btn + input[type="submit"], .btn-group input[type="submit"] + input[type="submit"], .btn-group button + input[type="submit"], .btn-group .action-button + input[type="submit"], .btn-group .menubutton + input[type="submit"], - .btn-group .btn + button, - .btn-group input[type="submit"] + button, - .btn-group button + button, - .btn-group .action-button + button, - .btn-group .menubutton + button, .btn-group .btn + .action-button, .btn-group input[type="submit"] + .action-button, .btn-group button + .action-button, .btn-group .action-button + .action-button, .btn-group .menubutton + .action-button, .btn-group .btn + .menubutton, .btn-group input[type="submit"] + .menubutton, .btn-group button + .menubutton, .btn-group .action-button + .menubutton, .btn-group .menubutton + .menubutton, + .btn-group .btn + .btn, .btn-group input[type="submit"] + .btn, .btn-group + button + .btn, .btn-group .action-button + .btn, .btn-group .menubutton + .btn, .btn-group .btn + input[type="submit"], .btn-group input[type="submit"] + input[type="submit"], .btn-group + button + input[type="submit"], .btn-group .action-button + input[type="submit"], .btn-group .menubutton + input[type="submit"], .btn-group .btn + + button, .btn-group input[type="submit"] + + button, .btn-group + button + + button, .btn-group .action-button + + button, .btn-group .menubutton + + button, .btn-group .btn + .action-button, .btn-group input[type="submit"] + .action-button, .btn-group + button + .action-button, .btn-group .action-button + .action-button, .btn-group .menubutton + .action-button, .btn-group .btn + .menubutton, .btn-group input[type="submit"] + .menubutton, .btn-group + button + .menubutton, .btn-group .action-button + .menubutton, .btn-group .menubutton + .menubutton, .btn-group .btn + .btn-group, .btn-group input[type="submit"] + .btn-group, - .btn-group button + .btn-group, + .btn-group + button + .btn-group, .btn-group .action-button + .btn-group, .btn-group .menubutton + .btn-group, .btn-group .btn-group + .btn, .btn-group .btn-group + input[type="submit"], - .btn-group .btn-group + button, + .btn-group .btn-group + + button, .btn-group .btn-group + .action-button, .btn-group .btn-group + .menubutton, .btn-group .btn-group + .btn-group, .btn-group-vertical .btn + .btn, .btn-group-vertical input[type="submit"] + .btn, - .btn-group-vertical button + .btn, + .btn-group-vertical + button + .btn, .btn-group-vertical .action-button + .btn, .btn-group-vertical .menubutton + .btn, .btn-group-vertical .btn + input[type="submit"], .btn-group-vertical input[type="submit"] + input[type="submit"], - .btn-group-vertical button + input[type="submit"], + .btn-group-vertical + button + input[type="submit"], .btn-group-vertical .action-button + input[type="submit"], .btn-group-vertical .menubutton + input[type="submit"], - .btn-group-vertical .btn + button, - .btn-group-vertical input[type="submit"] + button, - .btn-group-vertical button + button, - .btn-group-vertical .action-button + button, - .btn-group-vertical .menubutton + button, + .btn-group-vertical .btn + + button, + .btn-group-vertical input[type="submit"] + + button, + .btn-group-vertical + button + + button, + .btn-group-vertical .action-button + + button, + .btn-group-vertical .menubutton + + button, .btn-group-vertical .btn + .action-button, .btn-group-vertical input[type="submit"] + .action-button, - .btn-group-vertical button + .action-button, + .btn-group-vertical + button + .action-button, .btn-group-vertical .action-button + .action-button, .btn-group-vertical .menubutton + .action-button, .btn-group-vertical .btn + .menubutton, .btn-group-vertical input[type="submit"] + .menubutton, - .btn-group-vertical button + .menubutton, + .btn-group-vertical + button + .menubutton, .btn-group-vertical .action-button + .menubutton, .btn-group-vertical .menubutton + .menubutton, .btn-group-vertical .btn + .btn-group, .btn-group-vertical input[type="submit"] + .btn-group, - .btn-group-vertical button + .btn-group, + .btn-group-vertical + button + .btn-group, .btn-group-vertical .action-button + .btn-group, .btn-group-vertical .menubutton + .btn-group, .btn-group-vertical .btn-group + .btn, .btn-group-vertical .btn-group + input[type="submit"], - .btn-group-vertical .btn-group + button, + .btn-group-vertical .btn-group + + button, .btn-group-vertical .btn-group + .action-button, .btn-group-vertical .btn-group + .menubutton, .btn-group-vertical .btn-group + .btn-group { @@ -2405,25 +2470,27 @@ input[type="button"].btn-block { .btn-toolbar .input-group { width: auto; } -.btn-group > .btn:first-child, .btn-group > input:first-child[type="submit"], -.btn-group > button:first-child, .btn-group > .action-button:first-child, .btn-group > .menubutton:first-child { +.btn-group > .btn:first-child, .btn-group > input[type="submit"]:first-child, .btn-group > +button:first-child, .btn-group > .action-button:first-child, .btn-group > .menubutton:first-child { margin-left: 0; } -.btn-group > .btn:not(:last-child):not(.dropdown-toggle), .btn-group > input:not(:last-child):not(.dropdown-toggle)[type="submit"], -.btn-group > button:not(:last-child):not(.dropdown-toggle), .btn-group > .action-button:not(:last-child):not(.dropdown-toggle), .btn-group > .menubutton:not(:last-child):not(.dropdown-toggle), +.btn-group > .btn:not(:last-child):not(.dropdown-toggle), .btn-group > input[type="submit"]:not(:last-child):not(.dropdown-toggle), .btn-group > +button:not(:last-child):not(.dropdown-toggle), .btn-group > .action-button:not(:last-child):not(.dropdown-toggle), .btn-group > .menubutton:not(:last-child):not(.dropdown-toggle), .btn-group > .btn-group:not(:last-child) > .btn, .btn-group > .btn-group:not(:last-child) > input[type="submit"], -.btn-group > .btn-group:not(:last-child) > button, +.btn-group > .btn-group:not(:last-child) > +button, .btn-group > .btn-group:not(:last-child) > .action-button, .btn-group > .btn-group:not(:last-child) > .menubutton { border-top-right-radius: 0; border-bottom-right-radius: 0; } -.btn-group > .btn:not(:first-child), .btn-group > input:not(:first-child)[type="submit"], -.btn-group > button:not(:first-child), .btn-group > .action-button:not(:first-child), .btn-group > .menubutton:not(:first-child), +.btn-group > .btn:not(:first-child), .btn-group > input[type="submit"]:not(:first-child), .btn-group > +button:not(:first-child), .btn-group > .action-button:not(:first-child), .btn-group > .menubutton:not(:first-child), .btn-group > .btn-group:not(:first-child) > .btn, .btn-group > .btn-group:not(:first-child) > input[type="submit"], -.btn-group > .btn-group:not(:first-child) > button, +.btn-group > .btn-group:not(:first-child) > +button, .btn-group > .btn-group:not(:first-child) > .action-button, .btn-group > .btn-group:not(:first-child) > .menubutton { border-top-left-radius: 0; @@ -2439,11 +2506,15 @@ input[type="button"].btn-block { .dropleft .dropdown-toggle-split::before { margin-right: 0; } -.btn-sm + .dropdown-toggle-split, .btn-group-sm > .btn + .dropdown-toggle-split, .btn-group-sm > input[type="submit"] + .dropdown-toggle-split, .btn-group-sm > button + .dropdown-toggle-split, .btn-group-sm > .action-button + .dropdown-toggle-split, .btn-group-sm > .menubutton + .dropdown-toggle-split, .btn-xs + .dropdown-toggle-split, .multi-panel-history .control-column .btn + .dropdown-toggle-split, .multi-panel-history .control-column input[type="submit"] + .dropdown-toggle-split, .multi-panel-history .control-column button + .dropdown-toggle-split, .multi-panel-history .control-column .action-button + .dropdown-toggle-split, .multi-panel-history .control-column .menubutton + .dropdown-toggle-split, .multi-panel-history .history-column .panel-controls .btn + .dropdown-toggle-split, .multi-panel-history .history-column .panel-controls input[type="submit"] + .dropdown-toggle-split, .multi-panel-history .history-column .panel-controls button + .dropdown-toggle-split, .multi-panel-history .history-column .panel-controls .action-button + .dropdown-toggle-split, .multi-panel-history .history-column .panel-controls .menubutton + .dropdown-toggle-split { +.btn-sm + .dropdown-toggle-split, .btn-group-sm > .btn + .dropdown-toggle-split, .btn-group-sm > input[type="submit"] + .dropdown-toggle-split, .btn-group-sm > +button + .dropdown-toggle-split, .btn-group-sm > .action-button + .dropdown-toggle-split, .btn-group-sm > .menubutton + .dropdown-toggle-split, .btn-xs + .dropdown-toggle-split, .multi-panel-history .control-column .btn + .dropdown-toggle-split, .multi-panel-history .control-column input[type="submit"] + .dropdown-toggle-split, .multi-panel-history .control-column +button + .dropdown-toggle-split, .multi-panel-history .control-column .action-button + .dropdown-toggle-split, .multi-panel-history .control-column .menubutton + .dropdown-toggle-split, .multi-panel-history .history-column .panel-controls .btn + .dropdown-toggle-split, .multi-panel-history .history-column .panel-controls input[type="submit"] + .dropdown-toggle-split, .multi-panel-history .history-column .panel-controls +button + .dropdown-toggle-split, .multi-panel-history .history-column .panel-controls .action-button + .dropdown-toggle-split, .multi-panel-history .history-column .panel-controls .menubutton + .dropdown-toggle-split { padding-right: 0.375rem; padding-left: 0.375rem; } -.btn-lg + .dropdown-toggle-split, .btn-group-lg > .btn + .dropdown-toggle-split, .btn-group-lg > input[type="submit"] + .dropdown-toggle-split, .btn-group-lg > button + .dropdown-toggle-split, .btn-group-lg > .action-button + .dropdown-toggle-split, .btn-group-lg > .menubutton + .dropdown-toggle-split { +.btn-lg + .dropdown-toggle-split, .btn-group-lg > .btn + .dropdown-toggle-split, .btn-group-lg > input[type="submit"] + .dropdown-toggle-split, .btn-group-lg > +button + .dropdown-toggle-split, .btn-group-lg > .action-button + .dropdown-toggle-split, .btn-group-lg > .menubutton + .dropdown-toggle-split { padding-right: 0.75rem; padding-left: 0.75rem; } @@ -2451,70 +2522,84 @@ input[type="button"].btn-block { flex-direction: column; align-items: flex-start; justify-content: center; } - .btn-group-vertical .btn, .btn-group-vertical input[type="submit"], - .btn-group-vertical button, .btn-group-vertical .action-button, .btn-group-vertical .menubutton, + .btn-group-vertical .btn, .btn-group-vertical input[type="submit"], .btn-group-vertical + button, .btn-group-vertical .action-button, .btn-group-vertical .menubutton, .btn-group-vertical .btn-group { width: 100%; } - .btn-group-vertical > .btn + .btn, .btn-group-vertical > input[type="submit"] + .btn, .btn-group-vertical > button + .btn, .btn-group-vertical > .action-button + .btn, .btn-group-vertical > .menubutton + .btn, .btn-group-vertical > .btn + input[type="submit"], .btn-group-vertical > input[type="submit"] + input[type="submit"], .btn-group-vertical > button + input[type="submit"], .btn-group-vertical > .action-button + input[type="submit"], .btn-group-vertical > .menubutton + input[type="submit"], - .btn-group-vertical > .btn + button, - .btn-group-vertical > input[type="submit"] + button, - .btn-group-vertical > button + button, - .btn-group-vertical > .action-button + button, - .btn-group-vertical > .menubutton + button, .btn-group-vertical > .btn + .action-button, .btn-group-vertical > input[type="submit"] + .action-button, .btn-group-vertical > button + .action-button, .btn-group-vertical > .action-button + .action-button, .btn-group-vertical > .menubutton + .action-button, .btn-group-vertical > .btn + .menubutton, .btn-group-vertical > input[type="submit"] + .menubutton, .btn-group-vertical > button + .menubutton, .btn-group-vertical > .action-button + .menubutton, .btn-group-vertical > .menubutton + .menubutton, + .btn-group-vertical > .btn + .btn, .btn-group-vertical > input[type="submit"] + .btn, .btn-group-vertical > + button + .btn, .btn-group-vertical > .action-button + .btn, .btn-group-vertical > .menubutton + .btn, .btn-group-vertical > .btn + input[type="submit"], .btn-group-vertical > input[type="submit"] + input[type="submit"], .btn-group-vertical > + button + input[type="submit"], .btn-group-vertical > .action-button + input[type="submit"], .btn-group-vertical > .menubutton + input[type="submit"], .btn-group-vertical > .btn + + button, .btn-group-vertical > input[type="submit"] + + button, .btn-group-vertical > + button + + button, .btn-group-vertical > .action-button + + button, .btn-group-vertical > .menubutton + + button, .btn-group-vertical > .btn + .action-button, .btn-group-vertical > input[type="submit"] + .action-button, .btn-group-vertical > + button + .action-button, .btn-group-vertical > .action-button + .action-button, .btn-group-vertical > .menubutton + .action-button, .btn-group-vertical > .btn + .menubutton, .btn-group-vertical > input[type="submit"] + .menubutton, .btn-group-vertical > + button + .menubutton, .btn-group-vertical > .action-button + .menubutton, .btn-group-vertical > .menubutton + .menubutton, .btn-group-vertical > .btn + .btn-group, .btn-group-vertical > input[type="submit"] + .btn-group, - .btn-group-vertical > button + .btn-group, + .btn-group-vertical > + button + .btn-group, .btn-group-vertical > .action-button + .btn-group, .btn-group-vertical > .menubutton + .btn-group, .btn-group-vertical > .btn-group + .btn, .btn-group-vertical > .btn-group + input[type="submit"], - .btn-group-vertical > .btn-group + button, + .btn-group-vertical > .btn-group + + button, .btn-group-vertical > .btn-group + .action-button, .btn-group-vertical > .btn-group + .menubutton, .btn-group-vertical > .btn-group + .btn-group { margin-top: -1px; margin-left: 0; } - .btn-group-vertical > .btn:not(:last-child):not(.dropdown-toggle), .btn-group-vertical > input:not(:last-child):not(.dropdown-toggle)[type="submit"], - .btn-group-vertical > button:not(:last-child):not(.dropdown-toggle), .btn-group-vertical > .action-button:not(:last-child):not(.dropdown-toggle), .btn-group-vertical > .menubutton:not(:last-child):not(.dropdown-toggle), + .btn-group-vertical > .btn:not(:last-child):not(.dropdown-toggle), .btn-group-vertical > input[type="submit"]:not(:last-child):not(.dropdown-toggle), .btn-group-vertical > + button:not(:last-child):not(.dropdown-toggle), .btn-group-vertical > .action-button:not(:last-child):not(.dropdown-toggle), .btn-group-vertical > .menubutton:not(:last-child):not(.dropdown-toggle), .btn-group-vertical > .btn-group:not(:last-child) > .btn, .btn-group-vertical > .btn-group:not(:last-child) > input[type="submit"], - .btn-group-vertical > .btn-group:not(:last-child) > button, + .btn-group-vertical > .btn-group:not(:last-child) > + button, .btn-group-vertical > .btn-group:not(:last-child) > .action-button, .btn-group-vertical > .btn-group:not(:last-child) > .menubutton { border-bottom-right-radius: 0; border-bottom-left-radius: 0; } - .btn-group-vertical > .btn:not(:first-child), .btn-group-vertical > input:not(:first-child)[type="submit"], - .btn-group-vertical > button:not(:first-child), .btn-group-vertical > .action-button:not(:first-child), .btn-group-vertical > .menubutton:not(:first-child), + .btn-group-vertical > .btn:not(:first-child), .btn-group-vertical > input[type="submit"]:not(:first-child), .btn-group-vertical > + button:not(:first-child), .btn-group-vertical > .action-button:not(:first-child), .btn-group-vertical > .menubutton:not(:first-child), .btn-group-vertical > .btn-group:not(:first-child) > .btn, .btn-group-vertical > .btn-group:not(:first-child) > input[type="submit"], - .btn-group-vertical > .btn-group:not(:first-child) > button, + .btn-group-vertical > .btn-group:not(:first-child) > + button, .btn-group-vertical > .btn-group:not(:first-child) > .action-button, .btn-group-vertical > .btn-group:not(:first-child) > .menubutton { border-top-left-radius: 0; border-top-right-radius: 0; } -.btn-group-toggle > .btn, .btn-group-toggle > input[type="submit"], -.btn-group-toggle > button, .btn-group-toggle > .action-button, .btn-group-toggle > .menubutton, +.btn-group-toggle > .btn, .btn-group-toggle > input[type="submit"], .btn-group-toggle > +button, .btn-group-toggle > .action-button, .btn-group-toggle > .menubutton, .btn-group-toggle > .btn-group > .btn, .btn-group-toggle > .btn-group > input[type="submit"], -.btn-group-toggle > .btn-group > button, +.btn-group-toggle > .btn-group > +button, .btn-group-toggle > .btn-group > .action-button, .btn-group-toggle > .btn-group > .menubutton { margin-bottom: 0; } - .btn-group-toggle > .btn input[type="radio"], .btn-group-toggle > input[type="submit"] input[type="radio"], .btn-group-toggle > button input[type="radio"], .btn-group-toggle > .action-button input[type="radio"], .btn-group-toggle > .menubutton input[type="radio"], + .btn-group-toggle > .btn input[type="radio"], .btn-group-toggle > input[type="submit"] input[type="radio"], .btn-group-toggle > + button input[type="radio"], .btn-group-toggle > .action-button input[type="radio"], .btn-group-toggle > .menubutton input[type="radio"], .btn-group-toggle > .btn input[type="checkbox"], .btn-group-toggle > input[type="submit"] input[type="checkbox"], - .btn-group-toggle > button input[type="checkbox"], + .btn-group-toggle > + button input[type="checkbox"], .btn-group-toggle > .action-button input[type="checkbox"], .btn-group-toggle > .menubutton input[type="checkbox"], .btn-group-toggle > .btn-group > .btn input[type="radio"], .btn-group-toggle > .btn-group > input[type="submit"] input[type="radio"], - .btn-group-toggle > .btn-group > button input[type="radio"], + .btn-group-toggle > .btn-group > + button input[type="radio"], .btn-group-toggle > .btn-group > .action-button input[type="radio"], .btn-group-toggle > .btn-group > .menubutton input[type="radio"], .btn-group-toggle > .btn-group > .btn input[type="checkbox"], .btn-group-toggle > .btn-group > input[type="submit"] input[type="checkbox"], - .btn-group-toggle > .btn-group > button input[type="checkbox"], + .btn-group-toggle > .btn-group > + button input[type="checkbox"], .btn-group-toggle > .btn-group > .action-button input[type="checkbox"], .btn-group-toggle > .btn-group > .menubutton input[type="checkbox"] { position: absolute; @@ -2570,66 +2655,86 @@ input[type="button"].btn-block { .input-group-prepend, .input-group-append { display: flex; } - .input-group-prepend .btn, .input-group-prepend input[type="submit"], - .input-group-prepend button, .input-group-prepend .action-button, .input-group-prepend .menubutton, + .input-group-prepend .btn, .input-group-prepend input[type="submit"], .input-group-prepend + button, .input-group-prepend .action-button, .input-group-prepend .menubutton, .input-group-append .btn, .input-group-append input[type="submit"], - .input-group-append button, + .input-group-append + button, .input-group-append .action-button, .input-group-append .menubutton { position: relative; z-index: 2; } - .input-group-prepend .btn + .btn, .input-group-prepend input[type="submit"] + .btn, .input-group-prepend button + .btn, .input-group-prepend .action-button + .btn, .input-group-prepend .menubutton + .btn, .input-group-prepend .btn + input[type="submit"], .input-group-prepend input[type="submit"] + input[type="submit"], .input-group-prepend button + input[type="submit"], .input-group-prepend .action-button + input[type="submit"], .input-group-prepend .menubutton + input[type="submit"], - .input-group-prepend .btn + button, - .input-group-prepend input[type="submit"] + button, - .input-group-prepend button + button, - .input-group-prepend .action-button + button, - .input-group-prepend .menubutton + button, .input-group-prepend .btn + .action-button, .input-group-prepend input[type="submit"] + .action-button, .input-group-prepend button + .action-button, .input-group-prepend .action-button + .action-button, .input-group-prepend .menubutton + .action-button, .input-group-prepend .btn + .menubutton, .input-group-prepend input[type="submit"] + .menubutton, .input-group-prepend button + .menubutton, .input-group-prepend .action-button + .menubutton, .input-group-prepend .menubutton + .menubutton, + .input-group-prepend .btn + .btn, .input-group-prepend input[type="submit"] + .btn, .input-group-prepend + button + .btn, .input-group-prepend .action-button + .btn, .input-group-prepend .menubutton + .btn, .input-group-prepend .btn + input[type="submit"], .input-group-prepend input[type="submit"] + input[type="submit"], .input-group-prepend + button + input[type="submit"], .input-group-prepend .action-button + input[type="submit"], .input-group-prepend .menubutton + input[type="submit"], .input-group-prepend .btn + + button, .input-group-prepend input[type="submit"] + + button, .input-group-prepend + button + + button, .input-group-prepend .action-button + + button, .input-group-prepend .menubutton + + button, .input-group-prepend .btn + .action-button, .input-group-prepend input[type="submit"] + .action-button, .input-group-prepend + button + .action-button, .input-group-prepend .action-button + .action-button, .input-group-prepend .menubutton + .action-button, .input-group-prepend .btn + .menubutton, .input-group-prepend input[type="submit"] + .menubutton, .input-group-prepend + button + .menubutton, .input-group-prepend .action-button + .menubutton, .input-group-prepend .menubutton + .menubutton, .input-group-prepend .btn + .input-group-text, .input-group-prepend input[type="submit"] + .input-group-text, - .input-group-prepend button + .input-group-text, + .input-group-prepend + button + .input-group-text, .input-group-prepend .action-button + .input-group-text, .input-group-prepend .menubutton + .input-group-text, .input-group-prepend .input-group-text + .input-group-text, .input-group-prepend .input-group-text + .btn, .input-group-prepend .input-group-text + input[type="submit"], - .input-group-prepend .input-group-text + button, + .input-group-prepend .input-group-text + + button, .input-group-prepend .input-group-text + .action-button, .input-group-prepend .input-group-text + .menubutton, .input-group-append .btn + .btn, .input-group-append input[type="submit"] + .btn, - .input-group-append button + .btn, + .input-group-append + button + .btn, .input-group-append .action-button + .btn, .input-group-append .menubutton + .btn, .input-group-append .btn + input[type="submit"], .input-group-append input[type="submit"] + input[type="submit"], - .input-group-append button + input[type="submit"], + .input-group-append + button + input[type="submit"], .input-group-append .action-button + input[type="submit"], .input-group-append .menubutton + input[type="submit"], - .input-group-append .btn + button, - .input-group-append input[type="submit"] + button, - .input-group-append button + button, - .input-group-append .action-button + button, - .input-group-append .menubutton + button, + .input-group-append .btn + + button, + .input-group-append input[type="submit"] + + button, + .input-group-append + button + + button, + .input-group-append .action-button + + button, + .input-group-append .menubutton + + button, .input-group-append .btn + .action-button, .input-group-append input[type="submit"] + .action-button, - .input-group-append button + .action-button, + .input-group-append + button + .action-button, .input-group-append .action-button + .action-button, .input-group-append .menubutton + .action-button, .input-group-append .btn + .menubutton, .input-group-append input[type="submit"] + .menubutton, - .input-group-append button + .menubutton, + .input-group-append + button + .menubutton, .input-group-append .action-button + .menubutton, .input-group-append .menubutton + .menubutton, .input-group-append .btn + .input-group-text, .input-group-append input[type="submit"] + .input-group-text, - .input-group-append button + .input-group-text, + .input-group-append + button + .input-group-text, .input-group-append .action-button + .input-group-text, .input-group-append .menubutton + .input-group-text, .input-group-append .input-group-text + .input-group-text, .input-group-append .input-group-text + .btn, .input-group-append .input-group-text + input[type="submit"], - .input-group-append .input-group-text + button, + .input-group-append .input-group-text + + button, .input-group-append .input-group-text + .action-button, .input-group-append .input-group-text + .menubutton { margin-left: -1px; } @@ -2658,36 +2763,40 @@ input[type="button"].btn-block { .input-group-text input[type="checkbox"] { margin-top: 0; } -.input-group > .input-group-prepend > .btn, .input-group > .input-group-prepend > input[type="submit"], -.input-group > .input-group-prepend > button, .input-group > .input-group-prepend > .action-button, .input-group > .input-group-prepend > .menubutton, +.input-group > .input-group-prepend > .btn, .input-group > .input-group-prepend > input[type="submit"], .input-group > .input-group-prepend > +button, .input-group > .input-group-prepend > .action-button, .input-group > .input-group-prepend > .menubutton, .input-group > .input-group-prepend > .input-group-text, .input-group > .input-group-append:not(:last-child) > .btn, .input-group > .input-group-append:not(:last-child) > input[type="submit"], -.input-group > .input-group-append:not(:last-child) > button, +.input-group > .input-group-append:not(:last-child) > +button, .input-group > .input-group-append:not(:last-child) > .action-button, .input-group > .input-group-append:not(:last-child) > .menubutton, .input-group > .input-group-append:not(:last-child) > .input-group-text, .input-group > .input-group-append:last-child > .btn:not(:last-child):not(.dropdown-toggle), -.input-group > .input-group-append:last-child > input:not(:last-child):not(.dropdown-toggle)[type="submit"], -.input-group > .input-group-append:last-child > button:not(:last-child):not(.dropdown-toggle), +.input-group > .input-group-append:last-child > input[type="submit"]:not(:last-child):not(.dropdown-toggle), +.input-group > .input-group-append:last-child > +button:not(:last-child):not(.dropdown-toggle), .input-group > .input-group-append:last-child > .action-button:not(:last-child):not(.dropdown-toggle), .input-group > .input-group-append:last-child > .menubutton:not(:last-child):not(.dropdown-toggle), .input-group > .input-group-append:last-child > .input-group-text:not(:last-child) { border-top-right-radius: 0; border-bottom-right-radius: 0; } -.input-group > .input-group-append > .btn, .input-group > .input-group-append > input[type="submit"], -.input-group > .input-group-append > button, .input-group > .input-group-append > .action-button, .input-group > .input-group-append > .menubutton, +.input-group > .input-group-append > .btn, .input-group > .input-group-append > input[type="submit"], .input-group > .input-group-append > +button, .input-group > .input-group-append > .action-button, .input-group > .input-group-append > .menubutton, .input-group > .input-group-append > .input-group-text, .input-group > .input-group-prepend:not(:first-child) > .btn, .input-group > .input-group-prepend:not(:first-child) > input[type="submit"], -.input-group > .input-group-prepend:not(:first-child) > button, +.input-group > .input-group-prepend:not(:first-child) > +button, .input-group > .input-group-prepend:not(:first-child) > .action-button, .input-group > .input-group-prepend:not(:first-child) > .menubutton, .input-group > .input-group-prepend:not(:first-child) > .input-group-text, .input-group > .input-group-prepend:first-child > .btn:not(:first-child), -.input-group > .input-group-prepend:first-child > input:not(:first-child)[type="submit"], -.input-group > .input-group-prepend:first-child > button:not(:first-child), +.input-group > .input-group-prepend:first-child > input[type="submit"]:not(:first-child), +.input-group > .input-group-prepend:first-child > +button:not(:first-child), .input-group > .input-group-prepend:first-child > .action-button:not(:first-child), .input-group > .input-group-prepend:first-child > .menubutton:not(:first-child), .input-group > .input-group-prepend:first-child > .input-group-text:not(:first-child) { @@ -3574,7 +3683,8 @@ input[type="button"].btn-block { .badge:empty { display: none; } -.btn .badge, input[type="submit"] .badge, button .badge, .action-button .badge, .menubutton .badge { +.btn .badge, input[type="submit"] .badge, +button .badge, .action-button .badge, .menubutton .badge { position: relative; top: -1px; } @@ -3610,7 +3720,7 @@ input[type="button"].btn-block { .badge-info, .badge-new { color: #fff; background-color: #1cbceb; } - .badge-info[href]:hover, .badge-new[href]:hover, .badge-info[href]:focus, .badge-new[href]:focus { + .badge-info[href]:hover, [href].badge-new:hover, .badge-info[href]:focus, [href].badge-new:focus { color: #fff; text-decoration: none; background-color: #119ac3; } @@ -3618,7 +3728,7 @@ input[type="button"].btn-block { .badge-warning, .badge-beta { color: #212529; background-color: #f69713; } - .badge-warning[href]:hover, .badge-beta[href]:hover, .badge-warning[href]:focus, .badge-beta[href]:focus { + .badge-warning[href]:hover, [href].badge-beta:hover, .badge-warning[href]:focus, [href].badge-beta:focus { color: #212529; text-decoration: none; background-color: #ce7b08; } @@ -3714,42 +3824,46 @@ input[type="button"].btn-block { .alert-secondary .alert-link { color: #6c6e70; } -.alert-success, -.donemessagelarge, -.donemessage, +.alert-success, .donemessagelarge, .donemessage, .donemessagesmall { color: #146f14; background-color: #b1e7b1; border-color: #64ce64; } - .alert-success hr, .donemessagelarge hr, .donemessage hr, .donemessagesmall hr { + .alert-success hr, .donemessagelarge hr, .donemessage hr, + .donemessagesmall hr { border-top-color: #51c851; } - .alert-success .alert-link, .donemessagelarge .alert-link, .donemessage .alert-link, .donemessagesmall .alert-link { + .alert-success .alert-link, .donemessagelarge .alert-link, .donemessage .alert-link, + .donemessagesmall .alert-link { color: #0c440c; } -.alert-info, -.infomessagelarge, -.infomessage, +.alert-info, .infomessagelarge, .infomessage, .infomessagesmall, .ui-form-help .note { color: #11718d; background-color: #b0e7f8; border-color: #60d0f1; } - .alert-info hr, .infomessagelarge hr, .infomessage hr, .infomessagesmall hr, .ui-form-help .note hr { + .alert-info hr, .infomessagelarge hr, .infomessage hr, + .infomessagesmall hr, + .ui-form-help .note hr { border-top-color: #49c9ef; } - .alert-info .alert-link, .infomessagelarge .alert-link, .infomessage .alert-link, .infomessagesmall .alert-link, .ui-form-help .note .alert-link { + .alert-info .alert-link, .infomessagelarge .alert-link, .infomessage .alert-link, + .infomessagesmall .alert-link, + .ui-form-help .note .alert-link { color: #0c4d5f; } -.alert-warning, -.warningmessagelarge, -.warningmessage, +.alert-warning, .warningmessagelarge, .warningmessage, .warningmessagesmall, .ui-form-help .warning { color: #945b0b; background-color: #fcdbac; border-color: #f9b65a; } - .alert-warning hr, .warningmessagelarge hr, .warningmessage hr, .warningmessagesmall hr, .ui-form-help .warning hr { + .alert-warning hr, .warningmessagelarge hr, .warningmessage hr, + .warningmessagesmall hr, + .ui-form-help .warning hr { border-top-color: #f8ab41; } - .alert-warning .alert-link, .warningmessagelarge .alert-link, .warningmessage .alert-link, .warningmessagesmall .alert-link, .ui-form-help .warning .alert-link { + .alert-warning .alert-link, .warningmessagelarge .alert-link, .warningmessage .alert-link, + .warningmessagesmall .alert-link, + .ui-form-help .warning .alert-link { color: #653e07; } .alert-danger, .errormessagelarge, .error-modal .modal-content, .errormessage, @@ -3758,9 +3872,13 @@ input[type="button"].btn-block { color: #8f3937; background-color: #f9c7c6; border-color: #f48f8c; } - .alert-danger hr, .errormessagelarge hr, .error-modal .modal-content hr, .errormessage hr, .errormessagesmall hr, .ui-form-help .error hr { + .alert-danger hr, .errormessagelarge hr, .error-modal .modal-content hr, .errormessage hr, + .errormessagesmall hr, + .ui-form-help .error hr { border-top-color: #f27875; } - .alert-danger .alert-link, .errormessagelarge .alert-link, .error-modal .modal-content .alert-link, .errormessage .alert-link, .errormessagesmall .alert-link, .ui-form-help .error .alert-link { + .alert-danger .alert-link, .errormessagelarge .alert-link, .error-modal .modal-content .alert-link, .errormessage .alert-link, + .errormessagesmall .alert-link, + .ui-form-help .error .alert-link { color: #6a2a29; } .alert-light { @@ -4227,15 +4345,12 @@ button.close { .bs-popover-top .arrow, .bs-popover-auto[x-placement^="top"] .arrow { bottom: calc((0.5rem + 1px) * -1); } .bs-popover-top .arrow::before, .bs-popover-auto[x-placement^="top"] .arrow::before, - .bs-popover-top .arrow::after, - .bs-popover-auto[x-placement^="top"] .arrow::after { + .bs-popover-top .arrow::after, .bs-popover-auto[x-placement^="top"] .arrow::after { border-width: 0.5rem 0.5rem 0; } .bs-popover-top .arrow::before, .bs-popover-auto[x-placement^="top"] .arrow::before { bottom: 0; border-top-color: rgba(0, 0, 0, 0.25); } - - .bs-popover-top .arrow::after, - .bs-popover-auto[x-placement^="top"] .arrow::after { + .bs-popover-top .arrow::after, .bs-popover-auto[x-placement^="top"] .arrow::after { bottom: 1px; border-top-color: #fff; } @@ -4247,15 +4362,12 @@ button.close { height: 1rem; margin: 0.3rem 0; } .bs-popover-right .arrow::before, .bs-popover-auto[x-placement^="right"] .arrow::before, - .bs-popover-right .arrow::after, - .bs-popover-auto[x-placement^="right"] .arrow::after { + .bs-popover-right .arrow::after, .bs-popover-auto[x-placement^="right"] .arrow::after { border-width: 0.5rem 0.5rem 0.5rem 0; } .bs-popover-right .arrow::before, .bs-popover-auto[x-placement^="right"] .arrow::before { left: 0; border-right-color: rgba(0, 0, 0, 0.25); } - - .bs-popover-right .arrow::after, - .bs-popover-auto[x-placement^="right"] .arrow::after { + .bs-popover-right .arrow::after, .bs-popover-auto[x-placement^="right"] .arrow::after { left: 1px; border-right-color: #fff; } @@ -4264,15 +4376,12 @@ button.close { .bs-popover-bottom .arrow, .bs-popover-auto[x-placement^="bottom"] .arrow { top: calc((0.5rem + 1px) * -1); } .bs-popover-bottom .arrow::before, .bs-popover-auto[x-placement^="bottom"] .arrow::before, - .bs-popover-bottom .arrow::after, - .bs-popover-auto[x-placement^="bottom"] .arrow::after { + .bs-popover-bottom .arrow::after, .bs-popover-auto[x-placement^="bottom"] .arrow::after { border-width: 0 0.5rem 0.5rem 0.5rem; } .bs-popover-bottom .arrow::before, .bs-popover-auto[x-placement^="bottom"] .arrow::before { top: 0; border-bottom-color: rgba(0, 0, 0, 0.25); } - - .bs-popover-bottom .arrow::after, - .bs-popover-auto[x-placement^="bottom"] .arrow::after { + .bs-popover-bottom .arrow::after, .bs-popover-auto[x-placement^="bottom"] .arrow::after { top: 1px; border-bottom-color: #fff; } .bs-popover-bottom .popover-header::before, .bs-popover-auto[x-placement^="bottom"] .popover-header::before { @@ -4293,15 +4402,12 @@ button.close { height: 1rem; margin: 0.3rem 0; } .bs-popover-left .arrow::before, .bs-popover-auto[x-placement^="left"] .arrow::before, - .bs-popover-left .arrow::after, - .bs-popover-auto[x-placement^="left"] .arrow::after { + .bs-popover-left .arrow::after, .bs-popover-auto[x-placement^="left"] .arrow::after { border-width: 0.5rem 0 0.5rem 0.5rem; } .bs-popover-left .arrow::before, .bs-popover-auto[x-placement^="left"] .arrow::before { right: 0; border-left-color: rgba(0, 0, 0, 0.25); } - - .bs-popover-left .arrow::after, - .bs-popover-auto[x-placement^="left"] .arrow::after { + .bs-popover-left .arrow::after, .bs-popover-auto[x-placement^="left"] .arrow::after { right: 1px; border-left-color: #fff; } @@ -6513,10 +6619,18 @@ a.text-dark:hover, .quota-meter-text a:hover, a.text-dark:focus, .quota-meter-te img { page-break-inside: avoid; } p, - h2, h3, .upload-view-default .upload-box .upload-helper, .upload-view-composite .upload-box .upload-helper, .ui-form-composite .ui-form-header { + h2, + h3, + .upload-view-default .upload-box .upload-helper, + .upload-view-composite .upload-box .upload-helper, + .ui-form-composite .ui-form-header { orphans: 3; widows: 3; } - h2, h3, .upload-view-default .upload-box .upload-helper, .upload-view-composite .upload-box .upload-helper, .ui-form-composite .ui-form-header { + h2, + h3, + .upload-view-default .upload-box .upload-helper, + .upload-view-composite .upload-box .upload-helper, + .ui-form-composite .ui-form-header { page-break-after: avoid; } @page { size: a3; } @@ -6530,9 +6644,8 @@ a.text-dark:hover, .quota-meter-text a:hover, a.text-dark:focus, .quota-meter-te border: 1px solid #000; } .table, .ui-table-striped { border-collapse: collapse !important; } - - .table td, - .ui-table-striped td, .table th, .ui-table-striped th { + .table td, .ui-table-striped td, + .table th, .ui-table-striped th { background-color: #fff !important; } .table-bordered th, .table-bordered td { @@ -10540,23 +10653,17 @@ html[dir="rtl"] .select2-container-multi .select2-search-choice-close { .ui-portlet .portlet-content .portlet-buttons, .ui-portlet-repeat .portlet-content .portlet-buttons, .ui-portlet-panel .portlet-content .portlet-buttons, .ui-portlet-section .portlet-content .portlet-buttons, .ui-portlet-limited .portlet-content .portlet-buttons, .ui-portlet-narrow .portlet-content .portlet-buttons, .ui-portlet-plain .portlet-content .portlet-buttons { margin-top: 10px; margin-bottom: 10px; } - .ui-portlet .portlet-content .portlet-buttons > .btn, .ui-portlet-repeat .portlet-content .portlet-buttons > .btn, .ui-portlet-panel .portlet-content .portlet-buttons > .btn, .ui-portlet-section .portlet-content .portlet-buttons > .btn, .ui-portlet-limited .portlet-content .portlet-buttons > .btn, .ui-portlet-narrow .portlet-content .portlet-buttons > .btn, .ui-portlet-plain .portlet-content .portlet-buttons > .btn, .ui-portlet .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-repeat .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-panel .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-section .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-limited .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-narrow .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-plain .portlet-content .portlet-buttons > input[type="submit"], - .ui-portlet .portlet-content .portlet-buttons > button, - .ui-portlet-repeat .portlet-content .portlet-buttons > button, - .ui-portlet-panel .portlet-content .portlet-buttons > button, - .ui-portlet-section .portlet-content .portlet-buttons > button, - .ui-portlet-limited .portlet-content .portlet-buttons > button, - .ui-portlet-narrow .portlet-content .portlet-buttons > button, - .ui-portlet-plain .portlet-content .portlet-buttons > button, .ui-portlet .portlet-content .portlet-buttons > .action-button, .ui-portlet-repeat .portlet-content .portlet-buttons > .action-button, .ui-portlet-panel .portlet-content .portlet-buttons > .action-button, .ui-portlet-section .portlet-content .portlet-buttons > .action-button, .ui-portlet-limited .portlet-content .portlet-buttons > .action-button, .ui-portlet-narrow .portlet-content .portlet-buttons > .action-button, .ui-portlet-plain .portlet-content .portlet-buttons > .action-button, .ui-portlet .portlet-content .portlet-buttons > .menubutton, .ui-portlet-repeat .portlet-content .portlet-buttons > .menubutton, .ui-portlet-panel .portlet-content .portlet-buttons > .menubutton, .ui-portlet-section .portlet-content .portlet-buttons > .menubutton, .ui-portlet-limited .portlet-content .portlet-buttons > .menubutton, .ui-portlet-narrow .portlet-content .portlet-buttons > .menubutton, .ui-portlet-plain .portlet-content .portlet-buttons > .menubutton { + .ui-portlet .portlet-content .portlet-buttons > .btn, .ui-portlet-repeat .portlet-content .portlet-buttons > .btn, .ui-portlet-panel .portlet-content .portlet-buttons > .btn, .ui-portlet-section .portlet-content .portlet-buttons > .btn, .ui-portlet-limited .portlet-content .portlet-buttons > .btn, .ui-portlet-narrow .portlet-content .portlet-buttons > .btn, .ui-portlet-plain .portlet-content .portlet-buttons > .btn, .ui-portlet .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-repeat .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-panel .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-section .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-limited .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-narrow .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet-plain .portlet-content .portlet-buttons > input[type="submit"], .ui-portlet .portlet-content .portlet-buttons > + button, .ui-portlet-repeat .portlet-content .portlet-buttons > + button, .ui-portlet-panel .portlet-content .portlet-buttons > + button, .ui-portlet-section .portlet-content .portlet-buttons > + button, .ui-portlet-limited .portlet-content .portlet-buttons > + button, .ui-portlet-narrow .portlet-content .portlet-buttons > + button, .ui-portlet-plain .portlet-content .portlet-buttons > + button, .ui-portlet .portlet-content .portlet-buttons > .action-button, .ui-portlet-repeat .portlet-content .portlet-buttons > .action-button, .ui-portlet-panel .portlet-content .portlet-buttons > .action-button, .ui-portlet-section .portlet-content .portlet-buttons > .action-button, .ui-portlet-limited .portlet-content .portlet-buttons > .action-button, .ui-portlet-narrow .portlet-content .portlet-buttons > .action-button, .ui-portlet-plain .portlet-content .portlet-buttons > .action-button, .ui-portlet .portlet-content .portlet-buttons > .menubutton, .ui-portlet-repeat .portlet-content .portlet-buttons > .menubutton, .ui-portlet-panel .portlet-content .portlet-buttons > .menubutton, .ui-portlet-section .portlet-content .portlet-buttons > .menubutton, .ui-portlet-limited .portlet-content .portlet-buttons > .menubutton, .ui-portlet-narrow .portlet-content .portlet-buttons > .menubutton, .ui-portlet-plain .portlet-content .portlet-buttons > .menubutton { margin-right: 5px; } .ui-portlet .portlet-content.nopadding, .ui-portlet-repeat .portlet-content.nopadding, .ui-portlet-panel .portlet-content.nopadding, .ui-portlet-section .portlet-content.nopadding, .ui-portlet-limited .portlet-content.nopadding, .ui-portlet-narrow .portlet-content.nopadding, .ui-portlet-plain .portlet-content.nopadding, - .ui-portlet .portlet-body.nopadding, - .ui-portlet-repeat .portlet-body.nopadding, - .ui-portlet-panel .portlet-body.nopadding, - .ui-portlet-section .portlet-body.nopadding, - .ui-portlet-limited .portlet-body.nopadding, - .ui-portlet-narrow .portlet-body.nopadding, - .ui-portlet-plain .portlet-body.nopadding { + .ui-portlet .portlet-body.nopadding, .ui-portlet-repeat .portlet-body.nopadding, .ui-portlet-panel .portlet-body.nopadding, .ui-portlet-section .portlet-body.nopadding, .ui-portlet-limited .portlet-body.nopadding, .ui-portlet-narrow .portlet-body.nopadding, .ui-portlet-plain .portlet-body.nopadding { padding: 0px; } .ui-portlet .portlet-backdrop, .ui-portlet-repeat .portlet-backdrop, .ui-portlet-panel .portlet-backdrop, .ui-portlet-section .portlet-backdrop, .ui-portlet-limited .portlet-backdrop, .ui-portlet-narrow .portlet-backdrop, .ui-portlet-plain .portlet-backdrop { display: none; @@ -10718,24 +10825,24 @@ html[dir="rtl"] .select2-container-multi .select2-search-choice-close { background: transparent; -webkit-appearance: none; -moz-border-radius: 0.1875rem; } - .ui-input[disabled], .ui-form-element .ui-form-preview[disabled], .ui-form-element-disabled .ui-form-preview[disabled], .ui-input-basic[disabled], .ui-select select[disabled], .ui-select .select2-container .select2-choice[disabled], .ui-textarea[disabled], .ui-options .ui-options-list[disabled], .ui-form-slider .ui-form-slider-text[disabled], .ui-select .select2-container-multi .select2-choices[disabled], .ui-input[readonly], .ui-form-element .ui-form-preview[readonly], .ui-form-element-disabled .ui-form-preview[readonly], .ui-input-basic[readonly], .ui-select select[readonly], .ui-select .select2-container .select2-choice[readonly], .ui-textarea[readonly], .ui-options .ui-options-list[readonly], .ui-form-slider .ui-form-slider-text[readonly], .ui-select .select2-container-multi .select2-choices[readonly], + .ui-input[disabled], .ui-form-element [disabled].ui-form-preview, .ui-form-element-disabled [disabled].ui-form-preview, [disabled].ui-input-basic, .ui-select select[disabled], .ui-select .select2-container [disabled].select2-choice, [disabled].ui-textarea, .ui-options [disabled].ui-options-list, .ui-form-slider [disabled].ui-form-slider-text, .ui-select .select2-container-multi [disabled].select2-choices, .ui-input[readonly], .ui-form-element [readonly].ui-form-preview, .ui-form-element-disabled [readonly].ui-form-preview, [readonly].ui-input-basic, .ui-select select[readonly], .ui-select .select2-container [readonly].select2-choice, [readonly].ui-textarea, .ui-options [readonly].ui-options-list, .ui-form-slider [readonly].ui-form-slider-text, .ui-select .select2-container-multi [readonly].select2-choices, fieldset[disabled] .ui-input, - fieldset[disabled] .ui-form-element .ui-form-preview, - .ui-form-element fieldset[disabled] .ui-form-preview, - fieldset[disabled] .ui-form-element-disabled .ui-form-preview, - .ui-form-element-disabled fieldset[disabled] .ui-form-preview, + fieldset[disabled] .ui-form-element .ui-form-preview, .ui-form-element + fieldset[disabled] .ui-form-preview, + fieldset[disabled] .ui-form-element-disabled .ui-form-preview, .ui-form-element-disabled + fieldset[disabled] .ui-form-preview, fieldset[disabled] .ui-input-basic, - fieldset[disabled] .ui-select select, - .ui-select fieldset[disabled] select, - fieldset[disabled] .ui-select .select2-container .select2-choice, - .ui-select .select2-container fieldset[disabled] .select2-choice, + fieldset[disabled] .ui-select select, .ui-select + fieldset[disabled] select, + fieldset[disabled] .ui-select .select2-container .select2-choice, .ui-select .select2-container + fieldset[disabled] .select2-choice, fieldset[disabled] .ui-textarea, - fieldset[disabled] .ui-options .ui-options-list, - .ui-options fieldset[disabled] .ui-options-list, - fieldset[disabled] .ui-form-slider .ui-form-slider-text, - .ui-form-slider fieldset[disabled] .ui-form-slider-text, - fieldset[disabled] .ui-select .select2-container-multi .select2-choices, - .ui-select .select2-container-multi fieldset[disabled] .select2-choices { + fieldset[disabled] .ui-options .ui-options-list, .ui-options + fieldset[disabled] .ui-options-list, + fieldset[disabled] .ui-form-slider .ui-form-slider-text, .ui-form-slider + fieldset[disabled] .ui-form-slider-text, + fieldset[disabled] .ui-select .select2-container-multi .select2-choices, .ui-select .select2-container-multi + fieldset[disabled] .select2-choices { cursor: not-allowed; background-color: #e9ecef; } @@ -12241,7 +12348,10 @@ button.toast-close-button { .flex-horizontal-container { flex-flow: row nowrap; } -.flex-row, .flex-column, .history-panel > .list-items, .dataset-collection-panel > .list-items { +.flex-row, +.flex-column, +.history-panel > .list-items, +.dataset-collection-panel > .list-items { -webkit-flex: 1 1 auto; -ms-flex: 1 1 auto; property: 1 1 auto; @@ -12844,8 +12954,7 @@ div.form-row-error-message { .ui-form-info { color: #666; } .form-row .help a, - .toolParamHelp a, - .ui-form-info a { + .toolParamHelp a, .ui-form-info a { color: #666; } .form-row.form-actions { @@ -13668,8 +13777,8 @@ div.permissionContainer { margin-bottom: 8px; } .list-panel .controls .list-actions { display: none; } - .list-panel .controls .list-actions .btn, .list-panel .controls .list-actions input[type="submit"], - .list-panel .controls .list-actions button, .list-panel .controls .list-actions .action-button, .list-panel .controls .list-actions .menubutton { + .list-panel .controls .list-actions .btn, .list-panel .controls .list-actions input[type="submit"], .list-panel .controls .list-actions + button, .list-panel .controls .list-actions .action-button, .list-panel .controls .list-actions .menubutton { padding-top: 2px; padding-bottom: 2px; font-size: 90%; } @@ -14331,19 +14440,20 @@ pre.peek { .multi-panel-history > .footer { width: 100%; background-color: lightgrey; } - .multi-panel-history > .header .btn, .multi-panel-history > .header input[type="submit"], - .multi-panel-history > .header button, .multi-panel-history > .header .action-button, .multi-panel-history > .header .menubutton, + .multi-panel-history > .header .btn, .multi-panel-history > .header input[type="submit"], .multi-panel-history > .header + button, .multi-panel-history > .header .action-button, .multi-panel-history > .header .menubutton, .multi-panel-history > .footer .btn, .multi-panel-history > .footer input[type="submit"], - .multi-panel-history > .footer button, + .multi-panel-history > .footer + button, .multi-panel-history > .footer .action-button, .multi-panel-history > .footer .menubutton { height: 21px; } .multi-panel-history > .header { min-height: 29px; max-height: 29px; } - .multi-panel-history > .header .more-options .btn, .multi-panel-history > .header .more-options input[type="submit"], - .multi-panel-history > .header .more-options button, .multi-panel-history > .header .more-options .action-button, .multi-panel-history > .header .more-options .menubutton { + .multi-panel-history > .header .more-options .btn, .multi-panel-history > .header .more-options input[type="submit"], .multi-panel-history > .header .more-options + button, .multi-panel-history > .header .more-options .action-button, .multi-panel-history > .header .more-options .menubutton { width: 100%; height: 26px; } .multi-panel-history > .header .more-options input[type="checkbox"] { @@ -14532,8 +14642,8 @@ pre.peek { margin: 0; } .collection-creator > *:not(.popover) { padding: 0px 8px 0px 8px; } - .collection-creator .btn, .collection-creator input[type="submit"], - .collection-creator button, .collection-creator .action-button, .collection-creator .menubutton { + .collection-creator .btn, .collection-creator input[type="submit"], .collection-creator + button, .collection-creator .action-button, .collection-creator .menubutton { border-color: #bfbfbf; } .collection-creator .vertically-spaced { margin-top: 8px; } @@ -14591,22 +14701,41 @@ pre.peek { .collection-creator .choose-filters button { width: 100%; margin-top: 2px; } - .collection-creator .header .alert, .collection-creator .header .errormessagelarge, .collection-creator .header .error-modal .modal-content, .error-modal .collection-creator .header .modal-content, - .collection-creator .header .warningmessagelarge, - .collection-creator .header .donemessagelarge, - .collection-creator .header .infomessagelarge, - .collection-creator .header .ui-form-help .error, .ui-form-help .collection-creator .header .error, - .collection-creator .header .ui-form-help .warning, .ui-form-help .collection-creator .header .warning, - .collection-creator .header .ui-form-help .note, .ui-form-help .collection-creator .header .note, .collection-creator .header .errormessage, - .collection-creator .header .warningmessage, - .collection-creator .header .donemessage, - .collection-creator .header .infomessage, - .collection-creator .header .errormessagesmall, - .collection-creator .header .warningmessagesmall, - .collection-creator .header .donemessagesmall, - .collection-creator .header .infomessagesmall { + .collection-creator .header .alert, .collection-creator .header .errormessagelarge, .collection-creator .header .error-modal .modal-content, .error-modal .collection-creator .header .modal-content, .collection-creator .header + .warningmessagelarge, .collection-creator .header + .donemessagelarge, .collection-creator .header + .infomessagelarge, .collection-creator .header + .ui-form-help .error, + .ui-form-help .collection-creator .header .error, .collection-creator .header + .ui-form-help .warning, + .ui-form-help .collection-creator .header .warning, .collection-creator .header + .ui-form-help .note, + .ui-form-help .collection-creator .header .note, .collection-creator .header .errormessage, .collection-creator .header + .warningmessage, .collection-creator .header + .donemessage, .collection-creator .header + .infomessage, .collection-creator .header + .errormessagesmall, .collection-creator .header + .warningmessagesmall, .collection-creator .header + .donemessagesmall, .collection-creator .header + .infomessagesmall { display: none; } - .collection-creator .header .alert li, .collection-creator .header .errormessagelarge li, .collection-creator .header .error-modal .modal-content li, .error-modal .collection-creator .header .modal-content li, .collection-creator .header .warningmessagelarge li, .collection-creator .header .donemessagelarge li, .collection-creator .header .infomessagelarge li, .collection-creator .header .ui-form-help .error li, .ui-form-help .collection-creator .header .error li, .collection-creator .header .ui-form-help .warning li, .ui-form-help .collection-creator .header .warning li, .collection-creator .header .ui-form-help .note li, .ui-form-help .collection-creator .header .note li, .collection-creator .header .errormessage li, .collection-creator .header .warningmessage li, .collection-creator .header .donemessage li, .collection-creator .header .infomessage li, .collection-creator .header .errormessagesmall li, .collection-creator .header .warningmessagesmall li, .collection-creator .header .donemessagesmall li, .collection-creator .header .infomessagesmall li { + .collection-creator .header .alert li, .collection-creator .header .errormessagelarge li, .collection-creator .header .error-modal .modal-content li, .error-modal .collection-creator .header .modal-content li, .collection-creator .header + .warningmessagelarge li, .collection-creator .header + .donemessagelarge li, .collection-creator .header + .infomessagelarge li, .collection-creator .header + .ui-form-help .error li, + .ui-form-help .collection-creator .header .error li, .collection-creator .header + .ui-form-help .warning li, + .ui-form-help .collection-creator .header .warning li, .collection-creator .header + .ui-form-help .note li, + .ui-form-help .collection-creator .header .note li, .collection-creator .header .errormessage li, .collection-creator .header + .warningmessage li, .collection-creator .header + .donemessage li, .collection-creator .header + .infomessage li, .collection-creator .header + .errormessagesmall li, .collection-creator .header + .warningmessagesmall li, .collection-creator .header + .donemessagesmall li, .collection-creator .header + .infomessagesmall li { list-style: circle; margin-left: 32px; } .collection-creator .column { From 2a3b22cf497055a1afc578a1f7aead35f51a0ba2 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 30 Nov 2018 18:10:52 +0000 Subject: [PATCH 033/260] Update galaxy.yml.sample and galaxy_options.rst with `make config-rebuild` . Follow-up on https://github.com/galaxyproject/galaxy/pull/6776 . --- config/galaxy.yml.sample | 3 +++ doc/source/admin/galaxy_options.rst | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample index 82f8e524aa0..fb4351ba4be 100644 --- a/config/galaxy.yml.sample +++ b/config/galaxy.yml.sample @@ -1366,6 +1366,9 @@ galaxy: # of Galaxy's stable API. #enable_beta_workflow_modules: false + # Enable import and export of workflows as Galaxy Format 2 workflows. + #enable_beta_workflow_format: false + # Following options only apply to workflows scheduled using the legacy # workflow run API (running workflows via a POST to /api/workflows). # Force usage of Galaxy's beta workflow scheduler under certain diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 5940a9f1a9f..f96ee544854 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -2836,6 +2836,17 @@ :Type: bool +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``enable_beta_workflow_format`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Enable import and export of workflows as Galaxy Format 2 + workflows. +:Default: ``false`` +:Type: bool + + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``force_beta_workflow_scheduled_min_steps`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 6703584714507f129fee6e967df875bfa6027fc0 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 30 Nov 2018 19:26:41 -0500 Subject: [PATCH 034/260] Fix link to requirements.txt Dependency files rearranged (commit 1b0ccf6, Nov. 1, 2017) --- doc/source/admin/framework_dependencies.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/admin/framework_dependencies.rst b/doc/source/admin/framework_dependencies.rst index a0b70c2afc2..1f9ccd68948 100644 --- a/doc/source/admin/framework_dependencies.rst +++ b/doc/source/admin/framework_dependencies.rst @@ -12,7 +12,7 @@ In addition to framework dependencies, as of Galaxy 18.01, the client (UI) is no **in the development (dev) branch of the source repository**. The built client is still provided in ``release_YY.MM`` branches and the ``master`` branch. -.. _Python module dependencies: https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/dependencies/requirements.txt +.. _Python module dependencies: https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt .. _Python packaging best practices: https://packaging.python.org .. _virtualenv: https://packaging.python.org/tutorials/installing-packages/#creating-virtual-environments .. _pip: https://packaging.python.org/tutorials/installing-packages/#use-pip-for-installing From 66fae561b29507a69f5c32690be4791f86f5cdd7 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Sun, 2 Dec 2018 22:04:45 -0500 Subject: [PATCH 035/260] Don't use pushd/popd --- scripts/common_startup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index a3dfde519fe..ff9f853bdbf 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -241,7 +241,7 @@ if [ $SKIP_CLIENT_BUILD -eq 0 ]; then fi # Build client - pushd client + cd client if yarn install --network-timeout 120000 --check-files; then if ! yarn run build-production-maps; then echo "ERROR: Galaxy client build failed. See ./client/README.md for more information, including how to get help." @@ -251,7 +251,7 @@ if [ $SKIP_CLIENT_BUILD -eq 0 ]; then echo "ERROR: Galaxy client dependency installation failed. See ./client/README.md for more information, including how to get help." exit 1 fi - popd + cd - else echo "Regenerating static plugin directories." python ./scripts/plugin_staging.py From 118cd15186fc3d8d4784f8847355542ee07a8a5a Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Mon, 3 Dec 2018 14:16:30 -0500 Subject: [PATCH 036/260] downgrade bdbag, 1.5.0 requires python:>=2.7.9 which centos7 does not have --- .../dependencies/pipfiles/default/pinned-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 63e18ec1de7..737b810c102 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -20,7 +20,7 @@ azure-storage-nspkg==3.0.0 babel==2.6.0 bagit==1.7.0 bcrypt==3.1.4 -bdbag==1.5.0 +bdbag==1.4.1 beaker==1.10.0 bioblend==0.11.0 bleach==3.0.2 From 3931b8ec355df64e780f640d39ac0928d7c8afde Mon Sep 17 00:00:00 2001 From: M Bernt Date: Sun, 2 Dec 2018 18:55:09 +0100 Subject: [PATCH 037/260] improved parsing of sizes (bytes) before floats like without byte extension were not recognized (eg 1.0) --- lib/galaxy/util/__init__.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index f1bfbe75bd0..c42ba68c5f5 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -1351,6 +1351,8 @@ def size_to_bytes(size): >>> size_to_bytes('1024') 1024 + >>> size_to_bytes('1.0') + 1 >>> size_to_bytes('10 bytes') 10 >>> size_to_bytes('4k') @@ -1360,16 +1362,16 @@ def size_to_bytes(size): """ # Assume input in bytes if we can convert directly to an int try: - return int(size) + return int(float(size)) except ValueError: pass # Otherwise it must have non-numeric characters - size_re = re.compile(r'([\d\.]+)\s*([eptgmk]b?|b|bytes?)$') + size_re = re.compile(r'(?P[\d\.]+)\s*(?P[eptgmk]?(b|bytes?)?)?$') size_match = size_re.match(size.lower()) assert size_match is not None - size = float(size_match.group(1)) - multiple = size_match.group(2) - if multiple.startswith('b'): + size = float(size_match.group("size")) + multiple = size_match.group("multiple") + if multiple == "" or multiple.startswith('b'): return int(size) elif multiple.startswith('k'): return int(size * 1024) From cf25f7dcec6f7f7b13aac93a18ced291d5e74444 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 3 Dec 2018 09:31:59 +0100 Subject: [PATCH 038/260] Fix nesting of repeats and error when repeat is provided literally. This fixex #7048, where in addition to repeat values ('repeat_0|param: some_value') there are also values for the repeat param itself ('repeat': []). --- lib/galaxy/tools/parameters/meta.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/tools/parameters/meta.py b/lib/galaxy/tools/parameters/meta.py index d5fb5f53590..7440b1bf64c 100644 --- a/lib/galaxy/tools/parameters/meta.py +++ b/lib/galaxy/tools/parameters/meta.py @@ -72,26 +72,30 @@ def expand_workflow_inputs(inputs): return params, params_keys -def process_key(incoming_key, d): +def process_key(incoming_key, incoming_value, d): key_parts = incoming_key.split('|') if len(key_parts) == 1: # Regular parameter - d[incoming_key] = object() + d[incoming_key] = incoming_value elif key_parts[0].rsplit('_', 1)[-1].isdigit(): # Repeat input_name_index = key_parts[0].rsplit('_', 1) input_name, index = input_name_index + index = int(index) if input_name not in d: d[input_name] = [] - subdict = {} - d[input_name].append(subdict) - process_key("|".join(key_parts[1:]), d=subdict) + if len(d[input_name]) > index: + subdict = d[input_name][index] + else: + subdict = {} + d[input_name].append(subdict) + process_key("|".join(key_parts[1:]), incoming_value=incoming_value, d=subdict) else: # Section / Conditional input_name = key_parts[0] subdict = {} d[input_name] = subdict - process_key("|".join(key_parts[1:]), d=subdict) + process_key("|".join(key_parts[1:]), incoming_value=incoming_value, d=subdict) def expand_meta_parameters(trans, tool, incoming): @@ -113,9 +117,9 @@ def expand_meta_parameters(trans, tool, incoming): # according to tool.inputs (which is ordered). incoming_copy = incoming.copy() nested_dict = {} - for incoming_key in incoming_copy: + for incoming_key, incoming_value in incoming_copy.items(): if not incoming_key.startswith('__'): - process_key(incoming_key, d=nested_dict) + process_key(incoming_key, incoming_value=incoming_value, d=nested_dict) reordered_incoming = OrderedDict() From 3a93fd4f2321bf141b2ccf34ea80c42ee517b33d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 3 Dec 2018 09:31:36 +0100 Subject: [PATCH 039/260] Add test for proper nesting in process_keys and a unit test for https://github.com/galaxyproject/galaxy/issues/7048 --- test/unit/tools/test_parameter_parsing.py | 24 +++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/unit/tools/test_parameter_parsing.py b/test/unit/tools/test_parameter_parsing.py index 79b3ac35afd..047222a26be 100644 --- a/test/unit/tools/test_parameter_parsing.py +++ b/test/unit/tools/test_parameter_parsing.py @@ -3,10 +3,34 @@ from xml.etree.ElementTree import XML from galaxy import model from galaxy.tools.parameters import basic +from galaxy.tools.parameters.meta import process_key from galaxy.util import bunch from ..tools_support import UsesApp +def test_process_key(): + nested_dict = {} + d = { + 'repeat_0|inner_repeat_0|data_table_column_value': u'bla', + 'repeat_1|inner_repeat_1|data_table_column_value': u'bla2', + } + for key, value in d.items(): + process_key(key, value, nested_dict) + assert len(nested_dict) == 1 + + +def test_process_key_2(): + nested_dict = {} + d = { + 'data_tables_0|columns_0|data_table_column_value': 'Amel_HAv3.1', + 'data_tables': [], + 'directory_content': [], + } + for key, value in d.items(): + process_key(key, value, nested_dict) + assert len(nested_dict) == 2 + + class BaseParameterTestCase(TestCase, UsesApp): def setUp(self): From 27cc49d3f785c643b4dcdbf0da09d1bf63ee44f9 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 4 Dec 2018 14:12:41 +0100 Subject: [PATCH 040/260] size_to_bytes: use only regex, remove assert, add exceptions --- lib/galaxy/util/__init__.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index c42ba68c5f5..2c998a8a227 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -1347,7 +1347,7 @@ def nice_size(size): def size_to_bytes(size): """ - Returns a number of bytes if given a reasonably formatted string with the size + Returns a number of bytes (as integer) if given a reasonably formatted string with the size >>> size_to_bytes('1024') 1024 @@ -1360,32 +1360,28 @@ def size_to_bytes(size): >>> size_to_bytes('2.2 TB') 2418925581107 """ - # Assume input in bytes if we can convert directly to an int - try: - return int(float(size)) - except ValueError: - pass - # Otherwise it must have non-numeric characters size_re = re.compile(r'(?P[\d\.]+)\s*(?P[eptgmk]?(b|bytes?)?)?$') size_match = size_re.match(size.lower()) - assert size_match is not None - size = float(size_match.group("size")) + if size_match is None: + raise Exception("size_to_int: could not parse %s"%(size)) + number = float(size_match.group("size")) multiple = size_match.group("multiple") if multiple == "" or multiple.startswith('b'): - return int(size) + return int(number) elif multiple.startswith('k'): - return int(size * 1024) + return int(number * 1024) elif multiple.startswith('m'): - return int(size * 1024 ** 2) + return int(number * 1024 ** 2) elif multiple.startswith('g'): - return int(size * 1024 ** 3) + return int(number * 1024 ** 3) elif multiple.startswith('t'): - return int(size * 1024 ** 4) + return int(number * 1024 ** 4) elif multiple.startswith('p'): - return int(size * 1024 ** 5) + return int(number * 1024 ** 5) elif multiple.startswith('e'): - return int(size * 1024 ** 6) - + return int(number * 1024 ** 6) + else: + raise Exception("size_to_bytes: unknown multipier %s in %s"%(multiple, size)) def send_mail(frm, to, subject, body, config, html=None): """ From c90d71dd44794f8499813804290d08d3fdae6a6f Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 4 Dec 2018 13:35:41 +0000 Subject: [PATCH 041/260] Raise ValueError, fix spacing --- lib/galaxy/util/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 2c998a8a227..1229d390527 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -1363,7 +1363,7 @@ def size_to_bytes(size): size_re = re.compile(r'(?P[\d\.]+)\s*(?P[eptgmk]?(b|bytes?)?)?$') size_match = size_re.match(size.lower()) if size_match is None: - raise Exception("size_to_int: could not parse %s"%(size)) + raise ValueError("Could not parse string '%s'" % size) number = float(size_match.group("size")) multiple = size_match.group("multiple") if multiple == "" or multiple.startswith('b'): @@ -1381,7 +1381,7 @@ def size_to_bytes(size): elif multiple.startswith('e'): return int(number * 1024 ** 6) else: - raise Exception("size_to_bytes: unknown multipier %s in %s"%(multiple, size)) + raise ValueError("Unknown multiplier '%s' in '%s'" % (multiple, size)) def send_mail(frm, to, subject, body, config, html=None): """ From df51c468898e6bbc236e5a30a2758de4d06c9642 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 4 Dec 2018 13:36:53 +0000 Subject: [PATCH 042/260] Add missing empty line --- lib/galaxy/util/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 1229d390527..a20c2e04811 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -1383,6 +1383,7 @@ def size_to_bytes(size): else: raise ValueError("Unknown multiplier '%s' in '%s'" % (multiple, size)) + def send_mail(frm, to, subject, body, config, html=None): """ Sends an email. From 26a6b1c83381d1c75131f330d8b8f4c23680f1dd Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 4 Dec 2018 13:50:31 +0000 Subject: [PATCH 043/260] Stricter regexp --- lib/galaxy/util/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index a20c2e04811..470a90d5491 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -1360,11 +1360,11 @@ def size_to_bytes(size): >>> size_to_bytes('2.2 TB') 2418925581107 """ - size_re = re.compile(r'(?P[\d\.]+)\s*(?P[eptgmk]?(b|bytes?)?)?$') + size_re = re.compile(r'(?P\d+(\.\d+)?)\s*(?P[eptgmk]?(b|bytes?)?)?$') size_match = size_re.match(size.lower()) if size_match is None: raise ValueError("Could not parse string '%s'" % size) - number = float(size_match.group("size")) + number = float(size_match.group("number")) multiple = size_match.group("multiple") if multiple == "" or multiple.startswith('b'): return int(number) From eeffb26423a2b6380087c01af5d3390c2fd9b0df Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Tue, 4 Dec 2018 17:26:59 +0100 Subject: [PATCH 044/260] Implement colourful tags --- .../scripts/mvc/history/history-item-li.js | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/client/galaxy/scripts/mvc/history/history-item-li.js b/client/galaxy/scripts/mvc/history/history-item-li.js index bd67502fc73..316da595826 100644 --- a/client/galaxy/scripts/mvc/history/history-item-li.js +++ b/client/galaxy/scripts/mvc/history/history-item-li.js @@ -1,5 +1,14 @@ function _templateNametag(tag) { - return `${_.escape(tag.slice(5))}`; + return `${_.escape(tag.slice(5))}`; +} + +function generateTagColor(tagValue) { + var hash = hashFnv32a(tagValue); + var r = ((hash & 0xf) % 10), + g = (((hash & 0xf0) >> 4) % 10), + b = (((hash & 0xf00) >> 8) % 10); + + return `#${r}${g}${b}`; } function nametagTemplate(historyItem) { @@ -14,3 +23,23 @@ function nametagTemplate(historyItem) { export default { nametagTemplate: nametagTemplate }; + +/** + * Calculate a 32 bit FNV-1a hash + * Found here: https://gist.github.com/vaiorabbit/5657561 + * Ref.: http://isthe.com/chongo/tech/comp/fnv/ + * + * @param {string} str the input value + * @returns {integer} + */ +function hashFnv32a(str) { + /*jshint bitwise:false */ + var i, l, + hval = 0x811c9dc5; + + for (i = 0, l = str.length; i < l; i++) { + hval ^= str.charCodeAt(i); + hval += (hval << 1) + (hval << 4) + (hval << 7) + (hval << 8) + (hval << 24); + } + return hval >>> 0; +} From e6c4292fd5a3182f01f16dfd445359f5dcb87e0c Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Tue, 4 Dec 2018 17:50:50 +0100 Subject: [PATCH 045/260] Move to correct location --- .../scripts/mvc/history/history-item-li.js | 33 ++--------------- .../scripts/mvc/ui/ui-select-default.js | 2 +- client/galaxy/scripts/utils/utils.js | 37 ++++++++++++++++++- 3 files changed, 40 insertions(+), 32 deletions(-) diff --git a/client/galaxy/scripts/mvc/history/history-item-li.js b/client/galaxy/scripts/mvc/history/history-item-li.js index 316da595826..b57ef7135c3 100644 --- a/client/galaxy/scripts/mvc/history/history-item-li.js +++ b/client/galaxy/scripts/mvc/history/history-item-li.js @@ -1,14 +1,7 @@ +import Utils from "utils/utils"; + function _templateNametag(tag) { - return `${_.escape(tag.slice(5))}`; -} - -function generateTagColor(tagValue) { - var hash = hashFnv32a(tagValue); - var r = ((hash & 0xf) % 10), - g = (((hash & 0xf0) >> 4) % 10), - b = (((hash & 0xf00) >> 8) % 10); - - return `#${r}${g}${b}`; + return `${_.escape(tag.slice(5))}`; } function nametagTemplate(historyItem) { @@ -23,23 +16,3 @@ function nametagTemplate(historyItem) { export default { nametagTemplate: nametagTemplate }; - -/** - * Calculate a 32 bit FNV-1a hash - * Found here: https://gist.github.com/vaiorabbit/5657561 - * Ref.: http://isthe.com/chongo/tech/comp/fnv/ - * - * @param {string} str the input value - * @returns {integer} - */ -function hashFnv32a(str) { - /*jshint bitwise:false */ - var i, l, - hval = 0x811c9dc5; - - for (i = 0, l = str.length; i < l; i++) { - hval ^= str.charCodeAt(i); - hval += (hval << 1) + (hval << 4) + (hval << 7) + (hval << 8) + (hval << 24); - } - return hval >>> 0; -} diff --git a/client/galaxy/scripts/mvc/ui/ui-select-default.js b/client/galaxy/scripts/mvc/ui/ui-select-default.js index 67e3370b415..95f867d97fc 100644 --- a/client/galaxy/scripts/mvc/ui/ui-select-default.js +++ b/client/galaxy/scripts/mvc/ui/ui-select-default.js @@ -208,7 +208,7 @@ var View = Backbone.View.extend({ ${_.reduce( filteredTags.slice(0, 5), (memo, tag) => { - return `${memo} 
${_.escape( + return `${memo} 
${_.escape( tag )}
`; }, diff --git a/client/galaxy/scripts/utils/utils.js b/client/galaxy/scripts/utils/utils.js index e9c45c4e952..6b98e520aec 100644 --- a/client/galaxy/scripts/utils/utils.js +++ b/client/galaxy/scripts/utils/utils.js @@ -349,6 +349,40 @@ export function setWindowTitle(title) { } } +/** + * Calculate a 32 bit FNV-1a hash + * Found here: https://gist.github.com/vaiorabbit/5657561 + * Ref.: http://isthe.com/chongo/tech/comp/fnv/ + * + * @param {string} str the input value + * @returns {integer} + */ +function hashFnv32a(str) { + /*jshint bitwise:false */ + var i, l, + hval = 0x811c9dc5; + + for (i = 0, l = str.length; i < l; i++) { + hval ^= str.charCodeAt(i); + hval += (hval << 1) + (hval << 4) + (hval << 7) + (hval << 8) + (hval << 24); + } + return hval >>> 0; +} + +export function generateTagColor(tagValue) { + //var r = (15 - ((hash & 0xf) % 6)).toString(16), + //g = (15 - (((hash & 0xf0) >> 4) % 6)).toString(16), + //b = (15 - (((hash & 0xf00) >> 8) % 6)).toString(16); + + var hash = hashFnv32a(tagValue); + var r = (((hash & 0x00f) >> 0) % 10), + g = (((hash & 0x0f0) >> 4) % 10), + b = (((hash & 0xf00) >> 8) % 10); + + return `#${r}${g}${b}`; +} + + export default { cssLoadFile: cssLoadFile, cssGetAttribute: cssGetAttribute, @@ -368,5 +402,6 @@ export default { linkify: linkify, appendScriptStyle: appendScriptStyle, getQueryString: getQueryString, - setWindowTitle: setWindowTitle + setWindowTitle: setWindowTitle, + generateTagColor: generateTagColor }; From 5865fe3513232fe893dd2a638a54218599be11a9 Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Tue, 4 Dec 2018 18:56:45 +0100 Subject: [PATCH 046/260] Add colorFg function --- .../scripts/mvc/history/history-item-li.js | 2 +- .../scripts/mvc/ui/ui-select-default.js | 2 +- client/galaxy/scripts/utils/utils.js | 49 +++++++++++++++---- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/client/galaxy/scripts/mvc/history/history-item-li.js b/client/galaxy/scripts/mvc/history/history-item-li.js index b57ef7135c3..805bde01ef7 100644 --- a/client/galaxy/scripts/mvc/history/history-item-li.js +++ b/client/galaxy/scripts/mvc/history/history-item-li.js @@ -1,7 +1,7 @@ import Utils from "utils/utils"; function _templateNametag(tag) { - return `${_.escape(tag.slice(5))}`; + return `${_.escape(tag.slice(5))}`; } function nametagTemplate(historyItem) { diff --git a/client/galaxy/scripts/mvc/ui/ui-select-default.js b/client/galaxy/scripts/mvc/ui/ui-select-default.js index 95f867d97fc..9821756171b 100644 --- a/client/galaxy/scripts/mvc/ui/ui-select-default.js +++ b/client/galaxy/scripts/mvc/ui/ui-select-default.js @@ -208,7 +208,7 @@ var View = Backbone.View.extend({ ${_.reduce( filteredTags.slice(0, 5), (memo, tag) => { - return `${memo} 
${_.escape( + return `${memo} 
${_.escape( tag )}
`; }, diff --git a/client/galaxy/scripts/utils/utils.js b/client/galaxy/scripts/utils/utils.js index 6b98e520aec..92352f9b450 100644 --- a/client/galaxy/scripts/utils/utils.js +++ b/client/galaxy/scripts/utils/utils.js @@ -369,17 +369,47 @@ function hashFnv32a(str) { return hval >>> 0; } -export function generateTagColor(tagValue) { - //var r = (15 - ((hash & 0xf) % 6)).toString(16), - //g = (15 - (((hash & 0xf0) >> 4) % 6)).toString(16), - //b = (15 - (((hash & 0xf00) >> 8) % 6)).toString(16); +function contrastingColor(r, g, b) { + // Expects r, g, b as floats on range [0, 1] + // http://www.w3.org/TR/AERT#color-contrast + var o = ((r * 255 * 299) + + (g * 255 * 587) + + (b * 255 * 114)) / 1000; + return (o > 125) ? 'black' : 'white'; +} +export function hashToTagColor(tagValue) { var hash = hashFnv32a(tagValue); - var r = (((hash & 0x00f) >> 0) % 10), - g = (((hash & 0x0f0) >> 4) % 10), - b = (((hash & 0xf00) >> 8) % 10); + var r = ((hash & 0x00f) >> 0), + g = ((hash & 0x0f0) >> 4), + b = ((hash & 0xf00) >> 8); - return `#${r}${g}${b}`; + // Pastels + var r2 = 15 - (r % 6), + g2 = 15 - (g % 6), + b2 = 15 - (b % 6); + + // All colours + //var r2 = r % 0xf, + //g2 = g % 0xf, + //b2 = b % 0xf; + + // Not horribly dark + //var r2 = r % 0xa, + //g2 = g % 0xa, + //b2 = b % 0xa; + + return [r2, g2, b2]; +} + +export function generateTagColor(tag) { + var [r, g, b] = hashToTagColor(tag) + return `#${r.toString(16)}${g.toString(16)}${b.toString(16)}`; +} + +export function generateTagColorFg(tag) { + var [r, g, b] = hashToTagColor(tag) + return contrastingColor(r / 0xf, g / 0xf, b / 0xf) } @@ -403,5 +433,6 @@ export default { appendScriptStyle: appendScriptStyle, getQueryString: getQueryString, setWindowTitle: setWindowTitle, - generateTagColor: generateTagColor + generateTagColor: generateTagColor, + generateTagColorFg: generateTagColorFg }; From 12576ef49477f51137cf0f223600a00c2c7b90e3 Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Tue, 4 Dec 2018 19:59:39 +0100 Subject: [PATCH 047/260] Replace with style function --- .../scripts/mvc/history/history-item-li.js | 2 +- .../scripts/mvc/ui/ui-select-default.js | 2 +- client/galaxy/scripts/utils/utils.js | 72 +++++++++++-------- 3 files changed, 46 insertions(+), 30 deletions(-) diff --git a/client/galaxy/scripts/mvc/history/history-item-li.js b/client/galaxy/scripts/mvc/history/history-item-li.js index 805bde01ef7..42ab15671d0 100644 --- a/client/galaxy/scripts/mvc/history/history-item-li.js +++ b/client/galaxy/scripts/mvc/history/history-item-li.js @@ -1,7 +1,7 @@ import Utils from "utils/utils"; function _templateNametag(tag) { - return `${_.escape(tag.slice(5))}`; + return `${_.escape(tag.slice(5))}`; } function nametagTemplate(historyItem) { diff --git a/client/galaxy/scripts/mvc/ui/ui-select-default.js b/client/galaxy/scripts/mvc/ui/ui-select-default.js index 9821756171b..e3b29a6bd66 100644 --- a/client/galaxy/scripts/mvc/ui/ui-select-default.js +++ b/client/galaxy/scripts/mvc/ui/ui-select-default.js @@ -208,7 +208,7 @@ var View = Backbone.View.extend({ ${_.reduce( filteredTags.slice(0, 5), (memo, tag) => { - return `${memo} 
${_.escape( + return `${memo} 
${_.escape( tag )}
`; }, diff --git a/client/galaxy/scripts/utils/utils.js b/client/galaxy/scripts/utils/utils.js index 92352f9b450..0c14d34c80e 100644 --- a/client/galaxy/scripts/utils/utils.js +++ b/client/galaxy/scripts/utils/utils.js @@ -378,38 +378,55 @@ function contrastingColor(r, g, b) { return (o > 125) ? 'black' : 'white'; } -export function hashToTagColor(tagValue) { - var hash = hashFnv32a(tagValue); - var r = ((hash & 0x00f) >> 0), - g = ((hash & 0x0f0) >> 4), - b = ((hash & 0xf00) >> 8); +/** + * Converts an HSL color value to RGB. Conversion formula + * adapted from http://en.wikipedia.org/wiki/HSL_color_space. + * Assumes h, s, and l are contained in the set [0, 1] and + * returns r, g, and b in the set [0, 1]. + * + * @param {number} h The hue + * @param {number} s The saturation + * @param {number} l The lightness + * @return {Array} The RGB representation + */ +function hslToRgb(h, s, l){ + var r, g, b; - // Pastels - var r2 = 15 - (r % 6), - g2 = 15 - (g % 6), - b2 = 15 - (b % 6); + if(s == 0){ + r = g = b = l; // achromatic + }else{ + var hue2rgb = function hue2rgb(p, q, t){ + if(t < 0) t += 1; + if(t > 1) t -= 1; + if(t < 1/6) return p + (q - p) * 6 * t; + if(t < 1/2) return q; + if(t < 2/3) return p + (q - p) * (2/3 - t) * 6; + return p; + } - // All colours - //var r2 = r % 0xf, - //g2 = g % 0xf, - //b2 = b % 0xf; + var q = l < 0.5 ? l * (1 + s) : l + s - l * s; + var p = 2 * l - q; + r = hue2rgb(p, q, h + 1/3); + g = hue2rgb(p, q, h); + b = hue2rgb(p, q, h - 1/3); + } - // Not horribly dark - //var r2 = r % 0xa, - //g2 = g % 0xa, - //b2 = b % 0xa; - - return [r2, g2, b2]; + return [r, g, b]; } -export function generateTagColor(tag) { - var [r, g, b] = hashToTagColor(tag) - return `#${r.toString(16)}${g.toString(16)}${b.toString(16)}`; -} +export function generateTagStyle(tag) { + var hash = hashFnv32a(tag); + var hue = (hash >> 4) & 360; + //var lightnessOffset = 20; // dark + var lightnessOffset = 75; // pastel + var lightness = lightnessOffset + (hash & 0xf); + var bgColor = `hsl(${hue}, 100%, ${lightness}%)`; + var brColor = `hsl(${hue}, 100%, 30%)`; // darker -export function generateTagColorFg(tag) { - var [r, g, b] = hashToTagColor(tag) - return contrastingColor(r / 0xf, g / 0xf, b / 0xf) + var [r, g, b] = hslToRgb(hue, 1.0, lightness / 100); + var fgColor = contrastingColor(r, g, b) + + return `background-color: ${bgColor}; color: ${fgColor};` } @@ -433,6 +450,5 @@ export default { appendScriptStyle: appendScriptStyle, getQueryString: getQueryString, setWindowTitle: setWindowTitle, - generateTagColor: generateTagColor, - generateTagColorFg: generateTagColorFg + generateTagStyle: generateTagStyle }; From 0e7f2b3595a4235b1672205bd13b73153014f139 Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Tue, 4 Dec 2018 20:07:43 +0100 Subject: [PATCH 048/260] revert to grey border --- client/galaxy/scripts/utils/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/galaxy/scripts/utils/utils.js b/client/galaxy/scripts/utils/utils.js index 0c14d34c80e..a50132ca2d1 100644 --- a/client/galaxy/scripts/utils/utils.js +++ b/client/galaxy/scripts/utils/utils.js @@ -426,7 +426,7 @@ export function generateTagStyle(tag) { var [r, g, b] = hslToRgb(hue, 1.0, lightness / 100); var fgColor = contrastingColor(r, g, b) - return `background-color: ${bgColor}; color: ${fgColor};` + return `background-color: ${bgColor}; color: ${fgColor}; border: 1px solid gray` } From ad9787ef7050e3747b1b862ba857b1a2557174a0 Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Wed, 5 Dec 2018 15:54:21 +0100 Subject: [PATCH 049/260] cleanup/docs --- client/galaxy/scripts/utils/utils.js | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/client/galaxy/scripts/utils/utils.js b/client/galaxy/scripts/utils/utils.js index a50132ca2d1..76e052121f0 100644 --- a/client/galaxy/scripts/utils/utils.js +++ b/client/galaxy/scripts/utils/utils.js @@ -358,7 +358,6 @@ export function setWindowTitle(title) { * @returns {integer} */ function hashFnv32a(str) { - /*jshint bitwise:false */ var i, l, hval = 0x811c9dc5; @@ -369,9 +368,18 @@ function hashFnv32a(str) { return hval >>> 0; } +/** + * Implement W3C contrasting color algorithm + * http://www.w3.org/TR/AERT#color-contrast + * + * @param {number} r Red + * @param {number} g Green + * @param {number} b Blue + * @return {string} Either 'white' or 'black' + * + * Assumes r, g, b are in the set [0, 1] + */ function contrastingColor(r, g, b) { - // Expects r, g, b as floats on range [0, 1] - // http://www.w3.org/TR/AERT#color-contrast var o = ((r * 255 * 299) + (g * 255 * 587) + (b * 255 * 114)) / 1000; @@ -417,16 +425,15 @@ function hslToRgb(h, s, l){ export function generateTagStyle(tag) { var hash = hashFnv32a(tag); var hue = (hash >> 4) & 360; - //var lightnessOffset = 20; // dark - var lightnessOffset = 75; // pastel + var lightnessOffset = 75; var lightness = lightnessOffset + (hash & 0xf); var bgColor = `hsl(${hue}, 100%, ${lightness}%)`; - var brColor = `hsl(${hue}, 100%, 30%)`; // darker + var brColor = `hsl(${hue}, 100%, ${lightness - 40}%)`; var [r, g, b] = hslToRgb(hue, 1.0, lightness / 100); var fgColor = contrastingColor(r, g, b) - return `background-color: ${bgColor}; color: ${fgColor}; border: 1px solid gray` + return `background-color: ${bgColor}; color: ${fgColor}; border: 1px solid ${brColor}` } From 480143af57163b416f56566153e55d32b90e195b Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 5 Dec 2018 10:53:19 -0500 Subject: [PATCH 050/260] Handle extra cruft sent along by facebook login redirects --- client/galaxy/scripts/apps/analysis.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/galaxy/scripts/apps/analysis.js b/client/galaxy/scripts/apps/analysis.js index 14e5001eedb..ab5a0ac2d36 100644 --- a/client/galaxy/scripts/apps/analysis.js +++ b/client/galaxy/scripts/apps/analysis.js @@ -51,7 +51,7 @@ window.app = function app(options, bootstrapped) { /** Routes */ var AnalysisRouter = Router.extend({ routes: { - "(/)": "home", + "(/)(#)(_=_)": "home", "(/)root*": "home", "(/)tours(/)(:tour_id)": "show_tours", "(/)user(/)": "show_user", From f8ed3e73253e9ad2b893c14409173f9b504f00cc Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 5 Dec 2018 10:54:53 -0500 Subject: [PATCH 051/260] Client format. --- client/galaxy/scripts/layout/menu.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/galaxy/scripts/layout/menu.js b/client/galaxy/scripts/layout/menu.js index e8c22996f8c..2dc1b410985 100644 --- a/client/galaxy/scripts/layout/menu.js +++ b/client/galaxy/scripts/layout/menu.js @@ -221,9 +221,9 @@ var Collection = Backbone.Collection.extend({ } if (options.helpsite_url) { helpTab.menu.unshift({ - title: _l("Galaxy Help"), - url: options.helpsite_url, - target: "_blank" + title: _l("Galaxy Help"), + url: options.helpsite_url, + target: "_blank" }); } this.add(helpTab); From f5301c264592f5a6d3f7747b1f930f82384c9495 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 5 Dec 2018 13:12:42 +0000 Subject: [PATCH 052/260] Dependencies fixes, follow up on #7070 --- lib/galaxy/dependencies/pipfiles/default/Pipfile | 2 +- .../pipfiles/default/pinned-dev-requirements.txt | 8 ++++---- .../pipfiles/default/pinned-requirements.txt | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index aae0334053b..29414b29d87 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -37,7 +37,7 @@ nodeenv = "*" pycryptodome = "*" uWSGI = "*" pysam = "==0.15.1" -bdbag = "*" +bdbag = "==1.4.1" # 1.5.0 requires Python >=2.7.9 bleach = "*" "bz2file" = {version = "*", markers = "python_version < '3.3'"} ipaddress = {version = "*", markers = "python_version < '3.3'"} diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt index c33871500c5..32f6e262da8 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt @@ -5,7 +5,7 @@ argh==0.26.2 atomicwrites==1.2.1 attrs==18.2.0 babel==2.6.0 -certifi==2018.10.15 +certifi==2018.11.29 chardet==3.0.4 commonmark==0.5.4 docutils==0.14 @@ -31,10 +31,10 @@ pytest-html==1.19.0 pytest-metadata==1.7.0 pytest-pythonpath==0.7.3 pytest==3.9.2 -pytz==2018.6 +pytz==2018.7 pyyaml==3.13 recommonmark==0.4.0 -requests==2.20.0 +requests==2.20.1 scandir==1.9.0 ; python_version < '3.5' selenium==3.14.1 six==1.11.0 @@ -45,5 +45,5 @@ sphinxcontrib-websupport==1.1.0 testfixtures==6.3.0 twill==0.9.1 ; python_version < '3' typing==3.6.6 ; python_version < '3.5' -urllib3==1.24 +urllib3==1.24.1 watchdog==0.9.0 diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 737b810c102..e45d6f49a0a 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -18,7 +18,7 @@ azure-storage-blob==1.3.1 azure-storage-common==1.3.0 azure-storage-nspkg==3.0.0 babel==2.6.0 -bagit==1.7.0 +bagit==1.6.4 bcrypt==3.1.4 bdbag==1.4.1 beaker==1.10.0 @@ -32,7 +32,7 @@ bunch==1.0.1 bx-python==0.8.2 bz2file==0.98 ; python_version < '3.3' cachetools==2.1.0 -certifi==2018.10.15 +certifi==2018.11.29 cffi==1.11.5 chardet==3.0.4 cheetah3==3.1.0 @@ -127,12 +127,12 @@ python-neutronclient==6.9.0 python-novaclient==11.0.0 python-openid==2.2.5 python-swiftclient==3.6.0 -pytz==2018.6 +pytz==2018.7 pyyaml==3.13 repoze.lru==0.7 requests-oauthlib==1.0.0 requests-toolbelt==0.8.0 -requests==2.20.0 +requests==2.20.1 requestsexceptions==1.4.0 rfc3986==1.1.0 routes==2.4.1 @@ -152,7 +152,7 @@ tenacity==4.12.0 typing==3.6.6 ; python_version < '3.5' tzlocal==1.5.1 unicodecsv==0.14.1 ; python_version < '3.0' -urllib3==1.24 +urllib3==1.24.1 uwsgi==2.0.17.1 vine==1.1.4 warlock==1.3.0 From 7eb436cebc39363330e9b0503c36b00cb763dbc5 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 5 Dec 2018 19:04:58 +0000 Subject: [PATCH 053/260] Update all dependencies with `make update-dependencies` using pipenv 2018.11.26 . Also remove oscillating `python_version < '3.0'` environment marker for `azure-storage-nspkg` package. --- .../default/pinned-dev-requirements.txt | 12 ++-- .../pipfiles/default/pinned-requirements.txt | 68 +++++++++---------- lib/galaxy/dependencies/pipfiles/update.sh | 3 +- 3 files changed, 42 insertions(+), 41 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt index 32f6e262da8..4d68d12801c 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt @@ -15,28 +15,28 @@ idna==2.7 imagesize==1.1.0 jinja2==2.10 lxml==4.2.5 -markupsafe==1.0 +markupsafe==1.1.0 mock==2.0.0 more-itertools==4.3.0 nose==1.3.7 nosehtml==0.4.5 -pathlib2==2.3.2 ; python_version < '3.6' +pathlib2==2.3.3 ; python_version < '3.6' pathtools==0.1.2 -pbr==5.1.0 +pbr==5.1.1 pluggy==0.8.0 py==1.7.0 pygithub3==0.5.1 ; python_version < '3' -pygments==2.2.0 +pygments==2.3.0 pytest-html==1.19.0 pytest-metadata==1.7.0 pytest-pythonpath==0.7.3 -pytest==3.9.2 +pytest==4.0.1 pytz==2018.7 pyyaml==3.13 recommonmark==0.4.0 requests==2.20.1 scandir==1.9.0 ; python_version < '3.5' -selenium==3.14.1 +selenium==3.141.0 six==1.11.0 snowballstemmer==1.2.1 sphinx-rtd-theme==0.4.2 diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index e45d6f49a0a..b4ae3fc1e29 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -15,8 +15,8 @@ azure-mgmt-resource==2.0.0 azure-mgmt-storage==2.0.0 azure-nspkg==3.0.2 azure-storage-blob==1.3.1 -azure-storage-common==1.3.0 -azure-storage-nspkg==3.0.0 +azure-storage-common==1.4.0 +azure-storage-nspkg==3.1.0 babel==2.6.0 bagit==1.6.4 bcrypt==3.1.4 @@ -31,29 +31,29 @@ botocore==1.10.84 bunch==1.0.1 bx-python==0.8.2 bz2file==0.98 ; python_version < '3.3' -cachetools==2.1.0 +cachetools==3.0.0 certifi==2018.11.29 cffi==1.11.5 chardet==3.0.4 cheetah3==3.1.0 -cliff==2.13.0 +cliff==2.14.0 cloudauthz==0.2.0 cloudbridge==1.0.2 cmd2==0.8.9 contextlib2==0.5.5 ; python_version < '3.5' -cryptography==2.3.1 +cryptography==2.4.2 debtcollector==1.20.0 decorator==4.3.0 deprecation==2.0.6 dictobj==0.4 docopt==0.6.2 docutils==0.14 -dogpile.cache==0.6.7 +dogpile.cache==0.6.8 enum34==1.1.6 ; python_version < '3.4' fabric3==1.14.post1 funcsigs==1.0.2 ; python_version < '3.3' functools32==3.2.3.post2 ; python_version == '2.7' -future==0.16.0 +future==0.17.1 futures==3.2.0 ; python_version == '2.6' or python_version == '2.7' galaxy-sequence-utils==1.1.3 gxformat2==0.8.0 @@ -67,59 +67,59 @@ jmespath==0.9.3 jsonpatch==1.23 jsonpointer==2.0 jsonschema==2.6.0 -keystoneauth1==3.11.0 +keystoneauth1==3.11.1 kombu==4.2.1 mako==1.0.7 -markupsafe==1.0 +markupsafe==1.1.0 mercurial==3.7.3 ; python_version < '3' monotonic==1.5 -msgpack==0.5.6 +msgpack==0.6.0 msrest==0.5.5 msrestazure==0.5.0 munch==2.3.2 netaddr==0.7.19 netifaces==0.10.7 networkx==1.11 -nodeenv==1.3.2 +nodeenv==1.3.3 nose==1.3.7 -numpy==1.15.3 +numpy==1.15.4 oauthlib==2.1.0 openstacksdk==0.17.0 os-client-config==1.31.2 -os-service-types==1.3.0 +os-service-types==1.4.0 osc-lib==1.11.1 -oslo.config==6.6.2 -oslo.context==2.21.0 -oslo.i18n==3.22.1 -oslo.log==3.40.1 +oslo.config==6.7.0 +oslo.context==2.22.0 +oslo.i18n==3.23.0 +oslo.log==3.42.0 oslo.serialization==2.28.1 -oslo.utils==3.37.1 +oslo.utils==3.38.0 packaging==18.0 paramiko==2.4.2 parsley==1.3 -paste==3.0.4 -pastedeploy==1.5.2 -pastescript==2.0.2 -pbr==5.1.0 +paste==3.0.5 +pastedeploy==2.0.1 +pastescript==3.0.0 +pbr==5.1.1 prettytable==0.7.2 -psutil==5.4.7 +psutil==5.4.8 pulsar-galaxy-lib==0.8.3 pyasn1==0.4.4 pycparser==2.19 -pycryptodome==3.6.6 -pycryptodomex==3.6.6 +pycryptodome==3.7.2 +pycryptodomex==3.7.2 pyinotify==0.9.6 ; sys_platform != 'win32' and sys_platform != 'darwin' and sys_platform != 'sunos5' -pyjwkest==1.4.0 ; extra == 'openidconnect' -pyjwt==1.6.4 +pyjwkest==1.4.0 +pyjwt==1.7.0 pykwalify==1.7.0 pynacl==1.3.0 pyopenssl==18.0.0 -pyparsing==2.2.2 +pyparsing==2.3.0 pyperclip==1.7.0 pysam==0.15.1 pysftp==0.2.9 python-cinderclient==4.0.0 -python-dateutil==2.7.3 +python-dateutil==2.7.5 python-genomespaceclient==1.1.0 python-glanceclient==2.12.0 python-keystoneclient==3.17.0 @@ -134,19 +134,19 @@ requests-oauthlib==1.0.0 requests-toolbelt==0.8.0 requests==2.20.1 requestsexceptions==1.4.0 -rfc3986==1.1.0 +rfc3986==1.2.0 routes==2.4.1 s3transfer==0.1.13 simplejson==3.16.0 six==1.11.0 social-auth-core[openidconnect]==1.5.0 sqlalchemy-migrate==0.11.0 -sqlalchemy-utils==0.33.6 -sqlalchemy==1.2.12 +sqlalchemy-utils==0.33.9 +sqlalchemy==1.2.14 sqlparse==0.2.4 stevedore==1.30.0 subprocess32==3.5.3 ; python_version < '3.0' -svgwrite==1.1.12 +svgwrite==1.2.1 tempita==0.5.2 tenacity==4.12.0 typing==3.6.6 ; python_version < '3.5' @@ -158,6 +158,6 @@ vine==1.1.4 warlock==1.3.0 wcwidth==0.1.7 ; sys_platform != 'win32' webencodings==0.5.1 -webob==1.8.3 +webob==1.8.4 whoosh==2.7.4 wrapt==1.10.11 diff --git a/lib/galaxy/dependencies/pipfiles/update.sh b/lib/galaxy/dependencies/pipfiles/update.sh index 5dbe6ca7b22..711e0595388 100755 --- a/lib/galaxy/dependencies/pipfiles/update.sh +++ b/lib/galaxy/dependencies/pipfiles/update.sh @@ -42,7 +42,8 @@ for env in $ENVS; do pipenv lock -r | sed -e 's/--hash[^[:space:]]*//g' -e 's/[[:space:]]*$//' > pinned-requirements.txt pipenv lock -r --dev | sed -e 's/--hash[^[:space:]]*//g' -e 's/[[:space:]]*$//' > pinned-dev-requirements.txt # Fix oscillating environment markers - sed -i.orig -e "s/^cffi==\([^ ;]\{1,\}\).*$/cffi==\1/" \ + sed -i.orig -e "s/^azure-storage-nspkg==\([^ ;]\{1,\}\).*$/azure-storage-nspkg==\1/" \ + -e "s/^cffi==\([^ ;]\{1,\}\).*$/cffi==\1/" \ -e "s/^cmd2==\([^ ;]\{1,\}\).*$/cmd2==\1/" \ -e "s/^enum34==\([^ ;]\{1,\}\).*$/enum34==\1 ; python_version < '3.4'/" \ -e "s/^funcsigs==\([^ ;]\{1,\}\).*$/funcsigs==\1 ; python_version < '3.3'/" \ From 691fbfe606fba9399f66ffeed916294d3a4e5113 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 6 Dec 2018 10:40:51 +0100 Subject: [PATCH 054/260] Fix set_peek method for copied datasets dataset_assoc is the job to ouptut dataset association that references the HDA created by a job. If the HDA is copied the copies need to be updated as well, so here we need to call `set_peek` on dataset, not dataset_assoc.dataset. Broken in f4343d876f13e55e92ca228582b4430dcd01e4aa Thanks for tracking this down @jmchilton! --- lib/galaxy/jobs/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 0da8c26ad05..310d1ab0d40 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1399,10 +1399,10 @@ class JobWrapper(HasResourceParameters): line_count = context.get('line_count', None) try: # Certain datatype's set_peek methods contain a line_count argument - dataset_assoc.dataset.set_peek(line_count=line_count) + dataset.set_peek(line_count=line_count) except TypeError: # ... and others don't - dataset_assoc.dataset.set_peek() + dataset.set_peek() else: # Handle purged datasets. dataset.blurb = "empty" From 35fd29062587002f70f660e4a7675bd23545fc85 Mon Sep 17 00:00:00 2001 From: jvanbraekel <29703119+jvanbraekel@users.noreply.github.com> Date: Thu, 6 Dec 2018 11:40:50 +0100 Subject: [PATCH 055/260] Fix Error display The template display was broken in case of error. 'in_error_state is never defined it should be 'repository.in_error_state' instead. This was leading to some wrong field type display and not displaying the details about the error. --- .../admin/tool_shed_repository/manage_repository.mako | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/admin/tool_shed_repository/manage_repository.mako b/templates/admin/tool_shed_repository/manage_repository.mako index 59da6400041..a4f3b679070 100644 --- a/templates/admin/tool_shed_repository/manage_repository.mako +++ b/templates/admin/tool_shed_repository/manage_repository.mako @@ -36,7 +36,7 @@ ${render_galaxy_repository_actions( repository )}
- %if in_error_state: + %if repository.in_error_state: ${description|h} %else: @@ -51,7 +51,7 @@ ${render_galaxy_repository_actions( repository )} ${repository.owner|h}
- %if in_error_state: + %if repository.in_error_state:
${repository.error_message|h} @@ -66,7 +66,7 @@ ${render_galaxy_repository_actions( repository )} ${repository.deleted|h}
- %if not in_error_state: + %if not repository.in_error_state:
@@ -76,6 +76,6 @@ ${render_galaxy_repository_actions( repository )}

${render_resolver_dependencies(requirements_status)} -%if not in_error_state: +%if not repository.in_error_state: ${render_repository_items( repository.metadata, containers_dict, can_set_metadata=False, render_repository_actions_for='galaxy' )} %endif From 4c2f95b1f469d9789e65c4c9be97c4e137014ece Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Fri, 7 Dec 2018 10:42:53 +0100 Subject: [PATCH 056/260] Fix hashing, AND was not the right operation --- client/galaxy/scripts/utils/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/galaxy/scripts/utils/utils.js b/client/galaxy/scripts/utils/utils.js index 76e052121f0..89ce6ec3f5d 100644 --- a/client/galaxy/scripts/utils/utils.js +++ b/client/galaxy/scripts/utils/utils.js @@ -424,7 +424,7 @@ function hslToRgb(h, s, l){ export function generateTagStyle(tag) { var hash = hashFnv32a(tag); - var hue = (hash >> 4) & 360; + var hue = Math.abs((hash >> 4) % 360); var lightnessOffset = 75; var lightness = lightnessOffset + (hash & 0xf); var bgColor = `hsl(${hue}, 100%, ${lightness}%)`; From 2de4a0f72e5d51661318e1abeb727fdaa10f7444 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 27 Nov 2018 14:05:14 -0500 Subject: [PATCH 057/260] Remove legacy job handling hacks. These were left in to let Galaxy jobs complete after version upgrades, but the plan was always to remove them eventually. --- lib/galaxy/jobs/__init__.py | 10 ------- lib/galaxy/jobs/runners/pulsar.py | 2 -- lib/galaxy/tools/__init__.py | 5 ---- lib/galaxy/tools/parameters/output_collect.py | 4 --- lib/galaxy_ext/metadata/set_metadata.py | 30 +++++-------------- 5 files changed, 8 insertions(+), 43 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 3b2dee083f4..b58a6af8cd0 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1223,7 +1223,6 @@ class JobWrapper(HasResourceParameters): stderr, tool_exit_code=None, check_output_detected_state=None, - remote_working_directory=None, remote_metadata_directory=None, ): """ @@ -1231,8 +1230,6 @@ class JobWrapper(HasResourceParameters): the output datasets based on stderr and stdout from the command, and the contents of the output files. """ - # remote_working_directory not used with updated (7.0+ pulsar and 16.04+ - # originated Galaxy job - keep for a few releases for older jobs) finish_timer = util.ExecutionTimer() # default post job setup @@ -1382,11 +1379,8 @@ class JobWrapper(HasResourceParameters): def path_rewriter(path): if not path: return path - normalized_remote_working_directory = remote_working_directory and os.path.normpath(remote_working_directory) normalized_remote_metadata_directory = remote_metadata_directory and os.path.normpath(remote_metadata_directory) normalized_path = os.path.normpath(path) - if remote_working_directory and normalized_path.startswith(normalized_remote_working_directory): - return normalized_path.replace(normalized_remote_working_directory, self.working_directory, 1) if remote_metadata_directory and normalized_path.startswith(normalized_remote_metadata_directory): return normalized_path.replace(normalized_remote_metadata_directory, self.working_directory, 1) return path @@ -1465,10 +1459,6 @@ class JobWrapper(HasResourceParameters): param_dict = self.tool.params_from_strings(param_dict, self.app) # Create generated output children and primary datasets and add to param_dict tool_working_directory = self.tool_working_directory - # LEGACY: Remove in 17.XX - if not os.path.exists(tool_working_directory): - # Maybe this is a legacy job, use the job working directory instead - tool_working_directory = self.working_directory collected_datasets = { 'primary': self.tool.collect_primary_datasets(out_data, self.get_tool_provided_job_metadata(), tool_working_directory, input_ext, input_dbkey) } diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 5fc388e126f..dfbebe2c8d4 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -477,7 +477,6 @@ class PulsarJobRunner(AsynchronousJobRunner): try: client = self.get_client_from_state(job_state) run_results = client.full_status() - remote_working_directory = run_results.get("working_directory", None) remote_metadata_directory = run_results.get("metadata_directory", None) stdout = run_results.get('stdout', '') stderr = run_results.get('stderr', '') @@ -510,7 +509,6 @@ class PulsarJobRunner(AsynchronousJobRunner): stdout, stderr, exit_code, - remote_working_directory=remote_working_directory, remote_metadata_directory=remote_metadata_directory, ) except Exception: diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 3a56e10c82a..1463a29f235 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -895,11 +895,6 @@ class Tool(Dictifiable): def tool_provided_metadata(self, job_wrapper): meta_file = os.path.join(job_wrapper.tool_working_directory, self.provided_metadata_file) - # LEGACY: Remove in 17.XX - if not os.path.exists(meta_file): - # Maybe this is a legacy job, use the job working directory instead - meta_file = os.path.join(job_wrapper.working_directory, self.provided_metadata_file) - if not os.path.exists(meta_file): return output_collect.NullToolProvidedMetadata() if self.provided_metadata_style == "legacy": diff --git a/lib/galaxy/tools/parameters/output_collect.py b/lib/galaxy/tools/parameters/output_collect.py index e3adcaa5d25..000d9193df3 100644 --- a/lib/galaxy/tools/parameters/output_collect.py +++ b/lib/galaxy/tools/parameters/output_collect.py @@ -531,10 +531,6 @@ def collect_primary_datasets(tool, output, tool_provided_metadata, job_working_d new_primary_datasets = {} try: galaxy_json_path = os.path.join(job_working_directory, "working", tool.provide_metadata_file) - # LEGACY: Remove in 17.XX - if not os.path.exists(galaxy_json_path): - # Maybe this is a legacy job, use the job working directory instead - galaxy_json_path = os.path.join(job_working_directory, tool.provide_metadata_file) json_file = open(galaxy_json_path, 'r') for line in json_file: line = json.loads(line) diff --git a/lib/galaxy_ext/metadata/set_metadata.py b/lib/galaxy_ext/metadata/set_metadata.py index 0a51d03a4da..5342f90039f 100644 --- a/lib/galaxy_ext/metadata/set_metadata.py +++ b/lib/galaxy_ext/metadata/set_metadata.py @@ -82,15 +82,8 @@ def set_metadata(): # Set up datatypes registry datatypes_config = sys.argv.pop(1) if not os.path.exists(datatypes_config): - # This path should exist, except for jobs that started running on release 17.05, where a global - # datatypes_config (instead of a datatypes_config per job) was used. For a while release 17.05 - # would remove the global datatypes config on shutdown and toolbox reload, which would lead to - # failed metadata jobs. To remedy this we scan jobs at startup for missing registry.xml files, - # and if we detect such a job we write out the current registry.xml file. - datatypes_config = os.path.join(tool_job_working_directory, "registry.xml") - if not os.path.exists(datatypes_config): - print("Metadata setting failed because registry.xml could not be found. You may retry setting metadata.") - sys.exit(1) + print("Metadata setting failed because registry.xml could not be found. You may retry setting metadata.") + sys.exit(1) import galaxy.datatypes.registry datatypes_registry = galaxy.datatypes.registry.Registry() datatypes_registry.load_datatypes(root_dir=galaxy_root, config=datatypes_config) @@ -117,13 +110,7 @@ def set_metadata(): filename_out = fields.pop(0) filename_results_code = fields.pop(0) dataset_filename_override = fields.pop(0) - # Need to be careful with the way that these parameters are populated from the filename splitting, - # because if a job is running when the server is updated, any existing external metadata command-lines - # will not have info about the newly added override_metadata file - if fields: - override_metadata = fields.pop(0) - else: - override_metadata = None + override_metadata = fields.pop(0) set_meta_kwds = stringify_dictionary_keys(json.load(open(filename_kwds))) # load kwds; need to ensure our keywords are not unicode try: dataset = cPickle.load(open(filename_in, 'rb')) # load DatasetInstance @@ -133,12 +120,11 @@ def set_metadata(): if dataset.dataset.id in existing_job_metadata_dict: dataset.extension = existing_job_metadata_dict[dataset.dataset.id].get('ext', dataset.extension) # Metadata FileParameter types may not be writable on a cluster node, and are therefore temporarily substituted with MetadataTempFiles - if override_metadata: - override_metadata = json.load(open(override_metadata)) - for metadata_name, metadata_file_override in override_metadata: - if galaxy.datatypes.metadata.MetadataTempFile.is_JSONified_value(metadata_file_override): - metadata_file_override = galaxy.datatypes.metadata.MetadataTempFile.from_JSON(metadata_file_override) - setattr(dataset.metadata, metadata_name, metadata_file_override) + override_metadata = json.load(open(override_metadata)) + for metadata_name, metadata_file_override in override_metadata: + if galaxy.datatypes.metadata.MetadataTempFile.is_JSONified_value(metadata_file_override): + metadata_file_override = galaxy.datatypes.metadata.MetadataTempFile.from_JSON(metadata_file_override) + setattr(dataset.metadata, metadata_name, metadata_file_override) file_dict = existing_job_metadata_dict.get(dataset.dataset.id, {}) set_meta_with_tool_provided(dataset, file_dict, set_meta_kwds, datatypes_registry) if max_metadata_value_size: From 5c384efb8d2ad42d6dfafa84fc09bc02d3f462e8 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 28 Nov 2018 15:08:25 -0500 Subject: [PATCH 058/260] Fix via refactor set_metadata so more output metadata pruned properly. --- lib/galaxy_ext/metadata/set_metadata.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/galaxy_ext/metadata/set_metadata.py b/lib/galaxy_ext/metadata/set_metadata.py index 0a51d03a4da..c2c81a8e8b1 100644 --- a/lib/galaxy_ext/metadata/set_metadata.py +++ b/lib/galaxy_ext/metadata/set_metadata.py @@ -34,7 +34,7 @@ log = logging.getLogger(__name__) galaxy.model.Job() # this looks REAL stupid, but it is REQUIRED in order for SA to insert parameters into the classes defined by the mappers --> it appears that instantiating ANY mapper'ed class would suffice here -def set_meta_with_tool_provided(dataset_instance, file_dict, set_meta_kwds, datatypes_registry): +def set_meta_with_tool_provided(dataset_instance, file_dict, set_meta_kwds, datatypes_registry, max_metadata_value_size): # This method is somewhat odd, in that we set the metadata attributes from tool, # then call set_meta, then set metadata attributes from tool again. # This is intentional due to interplay of overwrite kwd, the fact that some metadata @@ -62,6 +62,12 @@ def set_meta_with_tool_provided(dataset_instance, file_dict, set_meta_kwds, data for metadata_name, metadata_value in file_dict.get('metadata', {}).items(): setattr(dataset_instance.metadata, metadata_name, metadata_value) + if max_metadata_value_size: + for k, v in list(dataset_instance.metadata.items()): + if total_size(v) > max_metadata_value_size: + log.info("Key %s too large for metadata, discarding" % k) + dataset_instance.metadata.remove_key(k) + def set_metadata(): # locate galaxy_root for loading datatypes @@ -140,12 +146,7 @@ def set_metadata(): metadata_file_override = galaxy.datatypes.metadata.MetadataTempFile.from_JSON(metadata_file_override) setattr(dataset.metadata, metadata_name, metadata_file_override) file_dict = existing_job_metadata_dict.get(dataset.dataset.id, {}) - set_meta_with_tool_provided(dataset, file_dict, set_meta_kwds, datatypes_registry) - if max_metadata_value_size: - for k, v in list(dataset.metadata.items()): - if total_size(v) > max_metadata_value_size: - log.info("Key %s too large for metadata, discarding" % k) - dataset.metadata.remove_key(k) + set_meta_with_tool_provided(dataset, file_dict, set_meta_kwds, datatypes_registry, max_metadata_value_size) dataset.metadata.to_JSON_dict(filename_out) # write out results of set_meta json.dump((True, 'Metadata has been set successfully'), open(filename_results_code, 'wt+')) # setting metadata has succeeded except Exception as e: @@ -159,7 +160,7 @@ def set_metadata(): new_dataset._extra_files_path = os.path.join(tool_job_working_directory, "working", extra_files) new_dataset.state = new_dataset.states.OK new_dataset_instance = galaxy.model.HistoryDatasetAssociation(id=-i, dataset=new_dataset, extension=file_dict.get('ext', 'data')) - set_meta_with_tool_provided(new_dataset_instance, file_dict, set_meta_kwds, datatypes_registry) + set_meta_with_tool_provided(new_dataset_instance, file_dict, set_meta_kwds, datatypes_registry, max_metadata_value_size) file_dict['metadata'] = json.loads(new_dataset_instance.metadata.to_JSON_dict()) # storing metadata in external form, need to turn back into dict, then later jsonify if existing_job_metadata_dict or new_job_metadata_dict: with open(job_metadata, 'wt') as job_metadata_fh: From 4894ab9c80ae39f8b5527860618e973daee00a61 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Fri, 7 Dec 2018 11:26:23 -0500 Subject: [PATCH 059/260] Add pointer to user permissions interface on finish of make datasets private --- client/galaxy/scripts/mvc/user/user-preferences.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client/galaxy/scripts/mvc/user/user-preferences.js b/client/galaxy/scripts/mvc/user/user-preferences.js index 4fd9b0cad85..254d8848744 100644 --- a/client/galaxy/scripts/mvc/user/user-preferences.js +++ b/client/galaxy/scripts/mvc/user/user-preferences.js @@ -62,7 +62,9 @@ var Model = Backbone.Model.extend({ $.post(`${Galaxy.root}history/make_private`, { all_histories: true }, () => { Galaxy.modal.show({ title: _l("Datasets are now private"), - body: "All of your histories and datsets have been made private.", + body: `All of your histories and datsets have been made private. If you'd like to make all *future* histories private please use the User Permissions interface.`, buttons: { Close: function() { Galaxy.modal.hide(); From 4fbaa161232f1b8e12b23788e258def10e6aa156 Mon Sep 17 00:00:00 2001 From: Dan Fornika Date: Fri, 7 Dec 2018 15:23:54 -0800 Subject: [PATCH 060/260] Add simple stub for mash sketch files --- config/datatypes_conf.xml.sample | 1 + lib/galaxy/datatypes/binary.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 29daee09b6f..7fe5ab9c57e 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -538,6 +538,7 @@ + diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 19ec0f80962..b6b57b644a6 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -126,6 +126,18 @@ class Cel(Binary): return False +class MashSketch(Binary): + """ + Mash Sketch file. + Sketches are used by the MinHash algorithm to allow fast distance estimations + with low storage and memory requirements. To make a sketch, each k-mer in a sequence + is hashed, which creates a pseudo-random identifier. By sorting these identifiers (hashes), + a small subset from the top of the sorted list can represent the entire sequence (these are min-hashes). + The more similar another sequence is, the more min-hashes it is likely to share. + """ + file_ext = "msh" + + class CompressedArchive(Binary): """ Class describing an compressed binary file From 4b1d9df4d677a0db7a11f39490671567e95c45be Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 7 Dec 2018 15:53:37 -0800 Subject: [PATCH 061/260] Replace `credentials` with `authz_id` on cloud storage get api doc string. --- lib/galaxy/webapps/galaxy/api/cloud.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/cloud.py b/lib/galaxy/webapps/galaxy/api/cloud.py index 27a27fab218..188e76d3f88 100644 --- a/lib/galaxy/webapps/galaxy/api/cloud.py +++ b/lib/galaxy/webapps/galaxy/api/cloud.py @@ -52,8 +52,10 @@ class CloudController(BaseAPIController): * history_id: the (encoded) id of history to which the object should be received to. * bucket: the name of a bucket from which data should be fetched from (e.g., a bucket name on AWS S3). * objects: a list of the names of objects to be fetched. - * credentials: a dictionary containing all the credentials required to authenticated to the - specified provider (e.g., {"secret_key": YOUR_AWS_SECRET_TOKEN, "access_key": YOUR_AWS_ACCESS_TOKEN}). + * authz_id: the encoded ID of CloudAuthz to be used for authorizing access to the resource + provider. You may get a list of the defined authorizations via + `/api/cloud/authz`. Also, you can use `/api/cloud/authz/create` to define a + new authorization. * input_args [Optional; default value is an empty dict] a dictionary containing the following keys: ** `dbkey`: [Optional; default value: is `?`] From 7d5a006b01ccd37ad65156be95f5e606dc4ea985 Mon Sep 17 00:00:00 2001 From: Dan Fornika Date: Fri, 7 Dec 2018 16:42:35 -0800 Subject: [PATCH 062/260] Removed extra whitespace to appease linter --- lib/galaxy/datatypes/binary.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index b6b57b644a6..a2963ec2914 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -136,8 +136,8 @@ class MashSketch(Binary): The more similar another sequence is, the more min-hashes it is likely to share. """ file_ext = "msh" - - + + class CompressedArchive(Binary): """ Class describing an compressed binary file From 27dc7c42c97db9230657a7db86567ac8b253d6b5 Mon Sep 17 00:00:00 2001 From: Dan Fornika Date: Fri, 7 Dec 2018 16:43:51 -0800 Subject: [PATCH 063/260] Removed extra whitespace to appease linter --- lib/galaxy/datatypes/binary.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index a2963ec2914..8da7603ad4d 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -129,10 +129,10 @@ class Cel(Binary): class MashSketch(Binary): """ Mash Sketch file. - Sketches are used by the MinHash algorithm to allow fast distance estimations - with low storage and memory requirements. To make a sketch, each k-mer in a sequence - is hashed, which creates a pseudo-random identifier. By sorting these identifiers (hashes), - a small subset from the top of the sorted list can represent the entire sequence (these are min-hashes). + Sketches are used by the MinHash algorithm to allow fast distance estimations + with low storage and memory requirements. To make a sketch, each k-mer in a sequence + is hashed, which creates a pseudo-random identifier. By sorting these identifiers (hashes), + a small subset from the top of the sorted list can represent the entire sequence (these are min-hashes). The more similar another sequence is, the more min-hashes it is likely to share. """ file_ext = "msh" From 355b48c361e68a7d9fbe9fe86de7aa2313e5d608 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 5 Dec 2018 16:42:15 -0500 Subject: [PATCH 064/260] Make handler assignment configurable in preparation for new assignment methods. --- lib/galaxy/app.py | 5 + lib/galaxy/config.py | 2 +- lib/galaxy/exceptions/__init__.py | 15 + lib/galaxy/exceptions/error_codes.json | 5 + lib/galaxy/jobs/__init__.py | 55 +--- lib/galaxy/jobs/handler.py | 8 - lib/galaxy/jobs/manager.py | 109 +++++--- lib/galaxy/managers/datasets.py | 2 +- lib/galaxy/model/__init__.py | 40 +++ lib/galaxy/tools/__init__.py | 23 +- lib/galaxy/tools/actions/__init__.py | 10 +- lib/galaxy/tools/actions/history_imp_exp.py | 8 +- lib/galaxy/tools/actions/metadata.py | 3 +- lib/galaxy/tools/actions/upload_common.py | 4 +- lib/galaxy/util/handlers.py | 257 +++++++++++++++--- lib/galaxy/web/stack/__init__.py | 134 ++++++++- lib/galaxy/web/stack/transport.py | 11 +- lib/galaxy/webapps/galaxy/api/jobs.py | 2 +- lib/galaxy/webapps/galaxy/config_schema.yml | 1 + .../webapps/galaxy/controllers/admin.py | 2 +- .../webapps/galaxy/controllers/history.py | 2 +- lib/galaxy/workflow/scheduling_manager.py | 123 ++++++--- test/unit/jobs/test_job_configuration.py | 11 +- test/unit/tools/test_actions.py | 7 - test/unit/unittest_utils/galaxy_mock.py | 4 +- 25 files changed, 616 insertions(+), 227 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 8871407a942..c2005ac1329 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -212,6 +212,11 @@ class UniverseApplication(config.ConfiguresGalaxyMixin): # Must be initialized after job_config. self.workflow_scheduling_manager = scheduling_manager.WorkflowSchedulingManager(self) + # Must be initialized after any component that might make use of stack messaging is configured. Alternatively if + # it becomes more commonly needed we could create a prefork function registration method like we do with + # postfork functions. + self.application_stack.init_late_prefork() + self.containers = {} if self.config.enable_beta_containers_interface: self.containers = build_container_interfaces( diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 4f9b628a5ac..13003a7c2c5 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -154,7 +154,7 @@ def find_root(kwargs): class Configuration(object): - deprecated_options = ('database_file', ) + deprecated_options = ('database_file', 'track_jobs_in_database') def __init__(self, **kwargs): self.config_dict = kwargs diff --git a/lib/galaxy/exceptions/__init__.py b/lib/galaxy/exceptions/__init__.py index db59cd80d74..be9a1d5dee1 100644 --- a/lib/galaxy/exceptions/__init__.py +++ b/lib/galaxy/exceptions/__init__.py @@ -185,6 +185,15 @@ class InternalServerError(MessageException): err_code = error_codes.INTERNAL_SERVER_ERROR +class ToolExecutionError(MessageException): + status_code = 500 + err_code = error_codes.TOOL_EXECUTION_ERROR + + def __init__(self, err_msg, type="error", job=None): + super(ToolExecutionError, self).__init__(err_msg, type) + self.job = job + + class NotImplemented(MessageException): status_code = 501 err_code = error_codes.NOT_IMPLEMENTED @@ -221,3 +230,9 @@ class ContainerRunError(Exception): super(ContainerRunError, self).__init__(msg, **kwargs) self.image = image self.command = command + + +class HandlerAssignmentError(Exception): + def __init__(self, msg=None, obj=None, **kwargs): + super(HandlerAssignmentError, self).__init__(msg, **kwargs) + self.obj = obj diff --git a/lib/galaxy/exceptions/error_codes.json b/lib/galaxy/exceptions/error_codes.json index aa790ec16b8..11908515a1d 100644 --- a/lib/galaxy/exceptions/error_codes.json +++ b/lib/galaxy/exceptions/error_codes.json @@ -144,6 +144,11 @@ "code": 500003, "message": "Error in a configuration file." }, + { + "name": "TOOL_EXECUTION_ERROR", + "code": 500004, + "message": "Tool execution failed due to an internal server error." + }, { "name": "NOT_IMPLEMENTED", "code": 501001, diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 3b2dee083f4..e4b92978113 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -115,6 +115,8 @@ class JobConfiguration(ConfiguresHandlers): These features are configured in the job configuration, by default, ``job_conf.xml`` """ + DEFAULT_BASE_HANDLER_POOLS = ('job-handlers',) + DEFAULT_NWORKERS = 4 JOB_RESOURCE_CONDITIONAL_XML = """ @@ -135,6 +137,8 @@ class JobConfiguration(ConfiguresHandlers): self.handlers = {} self.handler_runner_plugins = {} self.default_handler_id = None + self.handler_assignment_methods = None + self.handler_assignment_methods_configured = False self.destinations = {} self.destination_tags = {} self.default_destination_id = None @@ -150,7 +154,6 @@ class JobConfiguration(ConfiguresHandlers): output_size=None, destination_user_concurrent_jobs={}, destination_total_concurrent_jobs={}) - self._is_handler = None default_resubmits = [] default_resubmit_condition = self.app.config.default_job_resubmission_condition @@ -212,23 +215,12 @@ class JobConfiguration(ConfiguresHandlers): # Parse handlers handlers_conf = root.find('handlers') + self._init_handler_assignment_methods(handlers_conf) self._init_handlers(handlers_conf) - - # Determine the default handler(s) - try: - self.default_handler_id = self._get_default(self.app.config, handlers_conf, list(self.handlers.keys())) - except Exception: - pass - # For tets, this may not exist - try: - base_server_name = self.app.config.base_server_name - except AttributeError: - base_server_name = self.app.config.get('base_server_name', None) - if (self.default_handler_id is None - or (len(self.handlers) == 1 and base_server_name == next(iter(self.handlers.keys())))): - # Shortcut for compatibility with existing job confs that use the default handlers block, - # there are no defined handlers, or there's only one handler and it's this server - self.__set_default_job_handler() + self._set_default_handler_assignment_methods() + log.info("Job handler assignment methods set to: %s", ', '.join(self.handler_assignment_methods)) + for tag, handlers in [(t, h) for t, h in self.handlers.items() if isinstance(h, list)]: + log.info("Tag [%s] handlers: %s", tag, ', '.join(handlers)) # Parse destinations destinations = root.find('destinations') @@ -269,7 +261,8 @@ class JobConfiguration(ConfiguresHandlers): self.destinations[tag].append(job_destination) # Determine the default destination - self.default_destination_id = self._get_default(self.app.config, destinations, list(self.destinations.keys())) + self.default_destination_id = self._get_default( + self.app.config, destinations, list(self.destinations.keys()), auto=True) # Parse resources... resources = root.find('resources') @@ -346,32 +339,14 @@ class JobConfiguration(ConfiguresHandlers): if self.app.config.use_tasked_jobs: self.runner_plugins.append(dict(id='tasks', load='tasks', workers=DEFAULT_LOCAL_WORKERS)) # Set the handlers - self.__set_default_job_handler() + self._init_handler_assignment_methods() + self._init_handlers() + self._set_default_handler_assignment_methods() # Set the destination self.default_destination_id = 'local' self.destinations['local'] = [JobDestination(id='local', runner='local')] log.debug('Done loading job configuration') - def __set_default_job_handler(self): - # Called when self.default_handler_id is None - if self.app.application_stack.has_pool(self.app.application_stack.pools.JOB_HANDLERS): - if (self.app.application_stack.in_pool(self.app.application_stack.pools.JOB_HANDLERS)): - log.info("Found job handler pool managed by application stack, this server (%s) is a member of pool: " - "%s", self.app.config.server_name, self.app.application_stack.pools.JOB_HANDLERS) - self._is_handler = True - else: - log.info("Found job handler pool managed by application stack, this server (%s) will submit jobs to " - "pool: %s", self.app.config.server_name, self.app.application_stack.pools.JOB_HANDLERS) - else: - log.info('Did not find job handler pool managed by application stack, this server (%s) will handle jobs ' - 'submitted to it', self.app.config.server_name) - self._is_handler = True - self.app.application_stack.register_postfork_function(self.make_self_default_handler) - - def make_self_default_handler(self): - self.default_handler_id = self.app.config.server_name - self.handlers[self.app.config.server_name] = [self.app.config.server_name] - def get_tool_resource_xml(self, tool_id, tool_type): """ Given a tool id, return XML elements describing parameters to insert into job resources. @@ -484,7 +459,7 @@ class JobConfiguration(ConfiguresHandlers): :returns: JobToolConfiguration -- a representation of a element that uses the default handler and destination """ - return JobToolConfiguration(id='default', handler=self.default_handler_id, destination=self.default_destination_id) + return JobToolConfiguration(id='_default_', handler=self.default_handler_id, destination=self.default_destination_id) # Called upon instantiation of a Tool object def get_job_tool_configurations(self, ids): diff --git a/lib/galaxy/jobs/handler.py b/lib/galaxy/jobs/handler.py index 2e4b35107c0..1233b2c912e 100644 --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -743,14 +743,6 @@ class JobHandlerQueue(Monitors): if not self.track_jobs_in_database: self.queue.put((job_id, tool_id)) self.sleeper.wake() - else: - # Workflow invocations farmed out to workers will submit jobs through here. If a handler is unassigned, we - # will submit for one, or else claim it ourself. TODO: This should be moved to a higher level as it's now - # implemented here and in MessageJobQueue - job = self.sa_session.query(model.Job).get(job_id) - if job.handler is None and self.app.application_stack.has_pool(self.app.application_stack.pools.JOB_HANDLERS): - msg = JobHandlerMessage(task='setup', job_id=job_id) - self.app.application_stack.send_message(self.app.application_stack.pools.JOB_HANDLERS, msg) def shutdown(self): """Attempts to gracefully shut down the worker thread""" diff --git a/lib/galaxy/jobs/manager.py b/lib/galaxy/jobs/manager.py index 2cb68405502..325e7e26f12 100644 --- a/lib/galaxy/jobs/manager.py +++ b/lib/galaxy/jobs/manager.py @@ -1,11 +1,12 @@ """ Top-level Galaxy job manager, moves jobs to handler(s) """ - import logging +from functools import partial from sqlalchemy.sql.expression import null +from galaxy.exceptions import HandlerAssignmentError, ToolExecutionError from galaxy.jobs import handler, NoopQueue from galaxy.model import Job from galaxy.web.stack.message import JobHandlerMessage @@ -16,35 +17,86 @@ log = logging.getLogger(__name__) class JobManager(object): """ Highest level interface to job management. - - TODO: Currently the app accesses "job_queue" and "job_stop_queue" directly. - This should be decoupled. """ - def __init__(self, app): self.app = app self.job_lock = False if self.app.is_job_handler: log.debug("Initializing job handler") self.job_handler = handler.JobHandler(app) - self.job_stop_queue = self.job_handler.job_stop_queue - elif app.application_stack.has_pool(app.application_stack.pools.JOB_HANDLERS): - log.debug("Initializing job handler messaging interface") - self.job_handler = MessageJobHandler(app) - self.job_stop_queue = NoopQueue() else: self.job_handler = NoopHandler() - self.job_stop_queue = NoopQueue() - self.job_queue = self.job_handler.job_queue + self.__check_jobs_at_startup() + + def __check_jobs_at_startup(self): + """ + TODO: It should be documented that starting two Galaxy uWSGI master processes simultaneously would result in a race condition that *could* cause two handlers to pick up the same job. + + The recommended config for now will be webless handlers if running more than one uWSGI (master) process + """ + # FIXME: test + if self.app.job_config.use_messaging: + jobs_at_startup = self.app.model.context.query(Job).enable_eagerloads(False) \ + .filter((Job.state == Job.states.NEW) & (Job.handler == null())).all() + if jobs_at_startup: + log.info( + 'No handler assigned at startup for the following jobs, will dispatch via message: %s', + ', '.join([str(j.id) for j in jobs_at_startup])) + for job in jobs_at_startup: + tool = self.app.toolbox.get_tool(job.tool_id, job.tool_version, exact=True) + self.enqueue(job, tool) def start(self): self.job_handler.start() + def _queue_callback(self, job, tool_id): + self.job_handler.job_queue.put(job.id, tool_id) + + def _message_callback(self, job): + return JobHandlerMessage(task='setup', job_id=job.id) + + def enqueue(self, job, tool=None): + tool_id = None + configured_handler = None + if tool: + tool_id = tool.id + configured_handler = tool.get_configured_job_handler(job.params) + if configured_handler is not None: + p = " (with job params: %s)" % str(job.params) if job.params else "" + log.debug("(%s) Configured job handler for tool '%s'%s is: %s", job.log_str(), tool_id, p, configured_handler) + queue_callback = partial(self._queue_callback, job, tool_id) + message_callback = partial(self._message_callback, job) + try: + return self.app.job_config.assign_handler( + job, configured=configured_handler, queue_callback=queue_callback, message_callback=message_callback) + except HandlerAssignmentError as exc: + raise ToolExecutionError(exc.args[0], job=exc.obj) + + def stop(self, job, message=None): + self.job_handler.job_stop_queue.put(job.id, error_msg=message) + def shutdown(self): self.job_handler.shutdown() +class NoopManager(object): + """ + Implements the JobManager interface but does nothing + """ + def __init__(self, *args, **kwargs): + self.job_handler = NoopHandler() + + def enqueue(self, *args, **kwargs): + pass + + def stop(self, *args, **kwargs): + pass + + class NoopHandler(object): + """ + Implements the JobHandler interface but does nothing + """ def __init__(self, *args, **kwargs): self.job_queue = NoopQueue() self.job_stop_queue = NoopQueue() @@ -54,36 +106,3 @@ class NoopHandler(object): def shutdown(self, *args): pass - - -class MessageJobHandler(NoopHandler): - """ - Implements the JobHandler interface but just to send setup messages on startup - - TODO: It should be documented that starting two Galaxy uWSGI master processes simultaneously would result in a race condition that *could* cause two handlers to pick up the same job. - - The recommended config for now will be webless handlers if running more than one uWSGI (master) process - """ - def __init__(self, app): - # This runs in the web (main) process pre-fork - self.app = app - self.job_queue = MessageJobQueue(app) - self.job_stop_queue = NoopQueue() - jobs_at_startup = self.app.model.context.query(Job).enable_eagerloads(False) \ - .filter((Job.state == Job.states.NEW) & (Job.handler == null())).all() - if jobs_at_startup: - log.info('No handler assigned at startup for the following jobs, will dispatch via message: %s', ', '.join([str(j.id) for j in jobs_at_startup])) - for job in jobs_at_startup: - self.job_queue.put(job.id, job.tool_id) - - -class MessageJobQueue(NoopQueue): - """ - Implements the JobQueue / JobStopQueue interface but only sends messages to the actual job queue - """ - def __init__(self, app): - self.app = app - - def put(self, job_id, tool_id): - msg = JobHandlerMessage(task='setup', job_id=job_id) - self.app.application_stack.send_message(self.app.application_stack.pools.JOB_HANDLERS, msg) diff --git a/lib/galaxy/managers/datasets.py b/lib/galaxy/managers/datasets.py index 7c78ea8f85e..b83d6f4337f 100644 --- a/lib/galaxy/managers/datasets.py +++ b/lib/galaxy/managers/datasets.py @@ -296,7 +296,7 @@ class DatasetAssociationManager(base.ModelManager, # Are *all* of the job's other output datasets deleted? if job.check_if_output_datasets_deleted(): job.mark_deleted(self.app.config.track_jobs_in_database) - self.app.job_manager.job_stop_queue.put(job.id) + self.app.job_manager.stop(job) return True return False diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 1130b9e93e1..a2a2537b4c9 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -561,6 +561,7 @@ class Job(JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): RUNNING='running', OK='ok', ERROR='error', + FAILED='failed', PAUSED='paused', DELETED='deleted', DELETED_NEW='deleted_new') @@ -862,6 +863,22 @@ class Job(JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): dataset.peek = 'Job deleted' dataset.info = 'Job output deleted by user before job completed' + def mark_failed(self, info="Job execution failed", blurb=None, peek=None): + """ + Mark this job as failed, and mark any output datasets as errored. + """ + self.state = self.states.FAILED + self.info = info + for jtod in self.output_datasets: + jtod.dataset.state = jtod.dataset.states.ERROR + for hda in jtod.dataset.dataset.history_associations: + hda.state = hda.states.ERROR + if blurb: + hda.blurb = blurb + if peek: + hda.peek = peek + hda.info = info + def resume(self, flush=True): if self.state == self.states.PAUSED: self.set_state(self.states.NEW) @@ -4388,6 +4405,17 @@ class WorkflowInvocation(UsesCreateAndUpdateTime, Dictifiable, RepresentById): target_invocation_step = invocation_step return target_invocation_step + @staticmethod + def poll_unhandled_workflow_ids(sa_session): + and_conditions = [ + WorkflowInvocation.state == WorkflowInvocation.states.NEW, + WorkflowInvocation.handler.is_(None) + ] + query = sa_session.query( + WorkflowInvocation.id + ).filter(and_(*and_conditions)).order_by(WorkflowInvocation.table.c.id.asc()) + return [wid for wid in query.all()] + @staticmethod def poll_active_workflow_ids( sa_session, @@ -4536,6 +4564,18 @@ class WorkflowInvocation(UsesCreateAndUpdateTime, Dictifiable, RepresentById): return True return False + def set_handler(self, handler): + self.handler = handler + + def log_str(self): + extra = "" + safe_id = getattr(self, "id", None) + if safe_id is not None: + extra += "id=%s" % safe_id + else: + extra += "unflushed" + return "%s[%s]" % (self.__class__.__name__, extra) + class WorkflowInvocationToSubworkflowInvocationAssociation(Dictifiable, RepresentById): dict_collection_visible_keys = ['id', 'workflow_step_id', 'workflow_invocation_id', 'subworkflow_invocation_id'] diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 3a56e10c82a..2c14fb86404 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -561,16 +561,18 @@ class Tool(Dictifiable): assert rval is not None, 'Could not get a job tool configuration for Tool %s with job_params %s, this is a bug' % (self.id, job_params) return rval - def get_job_handler(self, job_params=None): - """Get a suitable job handler for this `Tool` given the provided `job_params`. If multiple handlers are valid for combination of `Tool` and `job_params` (e.g. the defined handler is a handler tag), one will be selected at random. + def get_configured_job_handler(self, job_params=None): + """Get the configured job handler for this `Tool` given the provided `job_params`. + + Unlike the former ``get_job_handler()`` method, this does not perform "preassignment" (random selection of + a configured handler ID from a tag). :param job_params: Any params specific to this job (e.g. the job source) :type job_params: dict or None - :returns: str -- The id of a job handler for a job run of this `Tool` + :returns: str or None -- The configured handler for a job run of this `Tool` """ - # convert tag to ID if necessary - return self.app.job_config.get_handler(self.__get_job_tool_configuration(job_params=job_params).handler) + return self.__get_job_tool_configuration(job_params=job_params).handler def get_job_destination(self, job_params=None): """ @@ -1516,7 +1518,16 @@ class Tool(Dictifiable): `self.tool_action`. In general this will create a `Job` that when run will build the tool's outputs, e.g. `DefaultToolAction`. """ - return self.tool_action.execute(self, trans, incoming=incoming, set_output_hid=set_output_hid, history=history, **kwargs) + try: + return self.tool_action.execute(self, trans, incoming=incoming, set_output_hid=set_output_hid, history=history, **kwargs) + except exceptions.ToolExecutionError as exc: + job = exc.job + job_id = 'unknown' + if job is not None: + job.mark_failed(info=exc.err_msg, blurb=exc.err_code.default_error_message) + job_id = job.id + log.error("Tool execution failed for job: %s", job_id) + raise def params_to_strings(self, params, app, nested=False): return params_to_strings(self.inputs, params, app, nested) diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index da9abcfd274..0ed09692a53 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -517,7 +517,6 @@ class DefaultToolAction(object): job.object_store_id = object_store_populator.object_store_id if job_params: job.params = dumps(job_params) - job.set_handler(tool.get_job_handler(job_params)) if completed_job: job.set_copied_from_job_id(completed_job.id) trans.sa_session.add(job) @@ -529,11 +528,8 @@ class DefaultToolAction(object): rerun_remap_job_id=rerun_remap_job_id, current_job=job, out_data=out_data) - log.info("Setup for job %s complete, ready to flush %s" % (job.log_str(), job_setup_timer)) + log.info("Setup for job %s complete, ready to be enqueued %s" % (job.log_str(), job_setup_timer)) - job_flush_timer = ExecutionTimer() - trans.sa_session.flush() - log.info("Flushed transaction for job %s %s" % (job.log_str(), job_flush_timer)) # Some tools are not really executable, but jobs are still created for them ( for record keeping ). # Examples include tools that redirect to other applications ( epigraph ). These special tools must # include something that can be retrieved from the params ( e.g., REDIRECT_URL ) to keep the job @@ -555,8 +551,8 @@ class DefaultToolAction(object): trans.sa_session.flush() trans.response.send_redirect(url_for(controller='tool_runner', action='redirect', redirect_url=redirect_url)) else: - # Put the job in the queue if tracking in memory - app.job_manager.job_queue.put(job.id, job.tool_id) + # Dispatch to a job handler. enqueue() is responsible for flushing the job + app.job_manager.enqueue(job, tool=tool) trans.log_event("Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id) return job, out_data diff --git a/lib/galaxy/tools/actions/history_imp_exp.py b/lib/galaxy/tools/actions/history_imp_exp.py index 512e9c7514a..1ed267d0495 100644 --- a/lib/galaxy/tools/actions/history_imp_exp.py +++ b/lib/galaxy/tools/actions/history_imp_exp.py @@ -56,11 +56,9 @@ class ImportHistoryToolAction(ToolAction): job.add_parameter(name, value) job.state = start_job_state # job inputs have been configured, restore initial job state - job.set_handler(tool.get_job_handler(None)) - trans.sa_session.flush() # Queue the job for execution - trans.app.job_manager.job_queue.put(job.id, tool.id) + trans.app.job_manager.enqueue(job, tool=tool) trans.log_event("Added import history job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id) return job, odict() @@ -136,11 +134,9 @@ class ExportHistoryToolAction(ToolAction): job.add_parameter(name, value) job.state = start_job_state # job inputs have been configured, restore initial job state - job.set_handler(tool.get_job_handler(None)) - trans.sa_session.flush() # Queue the job for execution - trans.app.job_manager.job_queue.put(job.id, tool.id) + trans.app.job_manager.enqueue(job, tool=tool) trans.log_event("Added export history job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id) return job, odict() diff --git a/lib/galaxy/tools/actions/metadata.py b/lib/galaxy/tools/actions/metadata.py index e60cd9fd5f8..6f8d1146828 100644 --- a/lib/galaxy/tools/actions/metadata.py +++ b/lib/galaxy/tools/actions/metadata.py @@ -63,7 +63,6 @@ class SetMetadataToolAction(ToolAction): except AttributeError: job.tool_version = "1.0.1" job.state = job.states.WAITING # we need to set job state to something other than NEW, or else when tracking jobs in db it will be picked up before we have added input / output parameters - job.set_handler(tool.get_job_handler(job_params)) sa_session.add(job) sa_session.flush() # ensure job.id is available @@ -104,7 +103,7 @@ class SetMetadataToolAction(ToolAction): sa_session.flush() # Queue the job for execution - app.job_manager.job_queue.put(job.id, tool.id) + app.job_manager.enqueue(job, tool=tool) # FIXME: need to add event logging to app and log events there rather than trans. # trans.log_event( "Added set external metadata job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id ) diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index 5becfe0a775..839c5a42f73 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -432,15 +432,13 @@ def create_job(trans, params, tool, json_file_path, outputs, folder=None, histor job.object_store_id = object_store_id job.set_state(job.states.NEW) - job.set_handler(tool.get_job_handler(None)) if job_params: for name, value in job_params.items(): job.add_parameter(name, value) trans.sa_session.add(job) - trans.sa_session.flush() # Queue the job for execution - trans.app.job_manager.job_queue.put(job.id, job.tool_id) + trans.app.job_manager.enqueue(job, tool=tool) trans.log_event("Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id) output = odict() for i, v in enumerate(outputs): diff --git a/lib/galaxy/util/handlers.py b/lib/galaxy/util/handlers.py index cdd7f845d1e..ab31a21649c 100644 --- a/lib/galaxy/util/handlers.py +++ b/lib/galaxy/util/handlers.py @@ -8,12 +8,45 @@ from __future__ import absolute_import import logging import os import random +from collections import namedtuple +from sqlalchemy.orm import object_session + +from . import ( + ExecutionTimer, + listify +) +from ..exceptions import HandlerAssignmentError log = logging.getLogger(__name__) +_handler_assignment_methods = ('MEM_SELF', 'DB_SELF', 'DB_PREASSIGN', 'UWSGI_MULE_MESSAGE') +HANDLER_ASSIGNMENT_METHODS = namedtuple('JOB_HANDLER_ASSIGNMENT_METHODS', _handler_assignment_methods)( + *[x.lower().replace('_', '-') for x in _handler_assignment_methods] +) + + +class HandlerAssignmentSkip(Exception): + """Exception for handler assignment methods to raise if the next method should be tried. + """ + pass + class ConfiguresHandlers(object): + DEFAULT_HANDLER_TAG = '_default_' + DEFAULT_BASE_HANDLER_POOLS = () + UNSUPPORTED_HANDLER_ASSIGNMENT_METHODS = () + + def add_handler(self, handler_id, tags): + if handler_id in self.handlers: + log.error("Handler '%s' already added, ignoring", handler_id) + return + self.handlers[handler_id] = (handler_id,) + for tag in tags: + if tag in self.handlers: + self.handlers[tag].append(handler_id) + else: + self.handlers[tag] = [handler_id] def _init_handlers(self, config_element): # Parse handlers @@ -21,22 +54,70 @@ class ConfiguresHandlers(object): for handler in self._findall_with_required(config_element, 'handler'): handler_id = handler.get('id') if handler_id in self.handlers: - log.error("Handler '%s' overlaps handler with the same name, ignoring" % handler_id) + log.error("Handler '%s' overlaps handler with the same name, ignoring", handler_id) else: - log.debug("Read definition for handler '%s'" % handler_id) - self.handlers[handler_id] = (handler_id,) - self._parse_handler(handler_id, handler) - if handler.get('tags', None) is not None: - for tag in [x.strip() for x in handler.get('tags').split(',')]: - if tag in self.handlers: - self.handlers[tag].append(handler_id) - else: - self.handlers[tag] = [handler_id] + log.debug("Read definition for handler '%s'", handler_id) + self.add_handler( + handler_id, + [x.strip() for x in handler.get('tags', self.DEFAULT_HANDLER_TAG).split(',')] + ) + self.default_handler_id = self._get_default(self.app.config, config_element, list(self.handlers.keys())) + + def _init_handler_assignment_methods(self, config_element=None): + self.__is_handler = None + # This is set by the stack job handler init code + self.pool_for_tag = {} + self._handler_assignment_method_methods = { + HANDLER_ASSIGNMENT_METHODS.MEM_SELF: self._assign_mem_self_handler, + HANDLER_ASSIGNMENT_METHODS.DB_SELF: self._assign_db_self_handler, + HANDLER_ASSIGNMENT_METHODS.DB_PREASSIGN: self._assign_db_preassign_handler, + HANDLER_ASSIGNMENT_METHODS.UWSGI_MULE_MESSAGE: self._assign_uwsgi_mule_message_handler, + } + if config_element is not None: + for method in listify(config_element.attrib.get('assign_with', []), do_strip=True): + method = method.lower() + assert method in HANDLER_ASSIGNMENT_METHODS, \ + "Invalid job handler assignment method '%s', must be one of: %s" % ( + method, ', '.join(HANDLER_ASSIGNMENT_METHODS)) + try: + self.handler_assignment_methods.append(method) + except AttributeError: + self.handler_assignment_methods_configured = True + self.handler_assignment_methods = [method] + if self.handler_assignment_methods == [HANDLER_ASSIGNMENT_METHODS.MEM_SELF]: + self.app.config.track_jobs_in_database = False + + def _set_default_handler_assignment_methods(self): + if not self.handler_assignment_methods_configured: + if not self.app.config.track_jobs_in_database and \ + HANDLER_ASSIGNMENT_METHODS.MEM_SELF not in self.UNSUPPORTED_HANDLER_ASSIGNMENT_METHODS: + # DEPRECATED: You should just set mem_self as the only method if you want this + # FIXME: does setting MEM_SELF still disable track_jobs_in_database like it used to? + # FIXME: test MEM_SELF via all config routes: + # 1. no assign config, track_jobs_in_databse = False in config + # 2. assign_with='mem-self', track_jobs_in_databse = False in config + # 2. assign_with='mem-self', track_jobs_in_databse = True in config + log.warning("The `track_jobs_in_database` option is deprecated, please set `%s` as the job" + " handler assignment method in the job handler configuration", + HANDLER_ASSIGNMENT_METHODS.MEM_SELF) + self.handler_assignment_methods = [HANDLER_ASSIGNMENT_METHODS.MEM_SELF] + elif not self.handlers: + # No handlers defined, default is for processes to handle the jobs they create + self.handler_assignment_methods = [HANDLER_ASSIGNMENT_METHODS.DB_SELF] + else: + # Handlers are defined, default is the value if the default attribute, or any untagged handler if + # default attribute is unset + self.handler_assignment_methods = [HANDLER_ASSIGNMENT_METHODS.DB_PREASSIGN] + # If the stack has handler pools it can override these defaults + self.app.application_stack.init_job_handling(self) + log.info("%s: No job handler assignment method is set, defaulting to '%s', set the `assign_with` attribute" + " on to override the default", self.__class__.__name__, + self.handler_assignment_methods[0]) def _parse_handler(self, handler_id, handler_def): pass - def _get_default(self, config, parent, names): + def _get_default(self, config, parent, names, auto=False): """ Returns the default attribute set in a parent tag like or , or return the ID of the child, if there is no explicit @@ -46,6 +127,8 @@ class ConfiguresHandlers(object): :type parent: ``xml.etree.ElementTree.Element`` :param names: The list of destination or handler IDs or tags that were loaded. :type names: list of str + :param auto: Automatically set a default if there is no default in the parent tag and there is only one child. + :type auto: bool :returns: str -- id or tag representing the default. """ @@ -60,14 +143,12 @@ class ConfiguresHandlers(object): if rval is not None: # If the parent element has a 'default' attribute, use the id or tag in that attribute - if rval not in names: + if self.deterministic_handler_assignment and rval not in names: raise Exception("<%s> default attribute '%s' does not match a defined id or tag in a child element" % (parent.tag, rval)) log.debug("<%s> default set to child with id or tag '%s'" % (parent.tag, rval)) - elif len(names) == 1: + elif auto and len(names) == 1: log.info("Setting <%s> default to child with id '%s'" % (parent.tag, names[0])) rval = names[0] - else: - raise Exception("No <%s> default specified, please specify a valid id or tag with the 'default' attribute" % parent.tag) return rval def _findall_with_required(self, parent, match, attribs=None): @@ -95,21 +176,38 @@ class ConfiguresHandlers(object): return rval @property - def is_handler(self): - """Indicate whether the current server is a handler. + def use_messaging(self): + # The job manager uses this to determine whether the messaging job queue should be used + return HANDLER_ASSIGNMENT_METHODS.UWSGI_MULE_MESSAGE in self.handler_assignment_methods - :param server_name: The name to check - :type server_name: str + @property + def deterministic_handler_assignment(self): + return self.handler_assignment_methods and all( + filter(lambda x: x in ( + HANDLER_ASSIGNMENT_METHODS.UWSGI_MULE_MESSAGE, + HANDLER_ASSIGNMENT_METHODS.DB_PREASSIGN, + ), self.handler_assignment_methods)) + + def _get_is_handler(self): + """Indicate whether the current server is configured as a handler. :return: bool """ - if self._is_handler is not None: - return self._is_handler + if self.__is_handler is not None: + return self.__is_handler + if (HANDLER_ASSIGNMENT_METHODS.DB_SELF in self.handler_assignment_methods + or HANDLER_ASSIGNMENT_METHODS.MEM_SELF in self.handler_assignment_methods): + return True for collection in self.handlers.values(): if self.app.config.server_name in collection: return True return False + def _set_is_handler(self, value): + self.__is_handler = value + + is_handler = property(_get_is_handler, _set_is_handler) + def _get_single_item(self, collection, index=None): """Given a collection of handlers or destinations, return one item from the collection at random. """ @@ -121,19 +219,116 @@ class ConfiguresHandlers(object): else: return collection[index % len(collection)] - # This is called by Tool.get_job_handler() - def get_handler(self, id_or_tag, index=None): - """Given a handler ID or tag, return a handler matching it. + # If these get to be any more complex we should probably modularize them, or at least move to a separate class + + def _assign_handler_direct(self, obj, configured): + if self.app.config.track_jobs_in_database and configured: + try: + resolved = self.get_preassigned_handler(configured) + except KeyError: + resolved = None + if resolved == configured: + obj.set_handler(configured) + _timed_flush_obj(obj) + return True + return False + + def _assign_mem_self_handler(self, obj, method, configured, queue_callback=None, **kwargs): + assert queue_callback is not None, \ + "Cannot perform '%s' handler assignment: `queue_callback` is None" % HANDLER_ASSIGNMENT_METHODS.MEM_SELF + if configured: + log.warning("(%s) Ignoring handler assignment to '%s' because configured handler assignment method" + " '' overrides per-tool handler assignment", obj.log_str(), + HANDLER_ASSIGNMENT_METHODS.MEM_SELF, configured) + _timed_flush_obj(obj) + queue_callback() + return self.app.config.server_name + + def _assign_db_self_handler(self, obj, method, configured, **kwargs): + if configured: + return self.handler_assignment_method_methods[HANDLER_ASSIGNMENT_METHODS.DB_PREASSIGN]( + obj, method, configured, **kwargs + ) + obj.set_handler(self.app.config.server_name) + _timed_flush_obj(obj) + return self.app.config.server_name + + def _assign_db_preassign_handler(self, obj, method, configured, index=None, **kwargs): + """Given a handler ID or tag, return a handler matching it, of those handlers that are statically configured in + the job configuration, or known via preconfigured pools. - :param id_or_tag: A handler ID or tag. - :type id_or_tag: str :param index: Generate "consistent" "random" handlers with this index if specified. :type index: int :returns: str -- A valid job handler ID. """ - if id_or_tag is None and self.default_handler_id is None: - return None - if id_or_tag is None: - id_or_tag = self.default_handler_id - return self._get_single_item(self.handlers[id_or_tag], index=index) + # FIXME: test a combo of pool handlers and defined handlers w/ DB_PREASSIGN as the only method + handler = configured + if handler is None: + handler = self.default_handler_id or self.DEFAULT_HANDLER_TAG + # Get a random handler ID from the possible handlers. If the admin has configured a tool with a handler tag that + # does not exist, or if there are no default handlers and configured is None, this will raise KeyError to + # assign_handler, which should log it, try the next method (if any), and if no other methods succeed, raise + # HandlerAssigmentError. + handler_id = self._get_single_item(self.handlers[handler], index=index) + if handler != handler_id: + log.debug("(%s) Selected handler '%s' by random choice from handler tag '%s'", obj.log_str(), + handler_id, handler) + obj.set_handler(handler_id) + _timed_flush_obj(obj) + return handler_id + + def _assign_uwsgi_mule_message_handler(self, obj, method, configured, message_callback=None, **kwargs): + assert message_callback is not None, \ + "Cannot perform '%s' handler assignment: `message_callback` is None" \ + % HANDLER_ASSIGNMENT_METHODS.UWSGI_MULE_MESSAGE + tag = configured or self.DEFAULT_HANDLER_TAG + pool = self.pool_for_tag.get(tag) + if pool is None: + log.debug("(%s) No handler pool (uWSGI farm) for '%s' found", obj.log_str(), tag) + raise HandlerAssignmentSkip() + else: + _timed_flush_obj(obj) + message = message_callback() + self.app.application_stack.send_message(pool, message) + return pool + + def assign_handler(self, obj, configured=None, **kwargs): + """Set a job handler, flush obj + + :param obj: Object to assign to (must be a model object with ``handler`` attribute and ``log_str`` callable) + :param configured: A handler ID or tag/pool. + + :returns: bool -- True on successful assignment, False otherwise. + """ + # It's a bit awkward that the method that actually hands off job execution is in the JobConfiguration, but + # that's currently the best place for it. It's worth noting that this method is also part of the + # WorkflowSchedulingManager, which acts like a combined JobConfiguration and JobManager. Combining those two + # classes would probably be reasonable (and would remove the need for the queue callback). + if self._assign_handler_direct(obj, configured): + log.info("(%s) Skipped handler assignment logic due to explicit configuration to a single handler: %s", + obj.log_str(), configured) + return True + for method in self.handler_assignment_methods: + try: + handler = self._handler_assignment_method_methods[method]( + obj, method, configured=configured, **kwargs) + log.info("(%s) Handler '%s' assigned using '%s' assignment method", obj.log_str(), handler, method) + break + except HandlerAssignmentSkip: + log.debug("(%s) Handler assignment method '%s' did not assign a handler, trying next method", + obj.log_str(), method) + except Exception: + log.exception("Caught exception in handler assignment method: %s", method) + else: + # Ideally we could just expunge the object from the SA session here, but in most cases, some of its related + # objects have already been created, so instead we'll just have to fail it. + log.error("(%s) Failed to select handler", obj.log_str()) + raise HandlerAssignmentError("Job handler assignment failed.", obj=obj) + + +def _timed_flush_obj(obj): + obj_flush_timer = ExecutionTimer() + sa_session = object_session(obj) + sa_session.flush() + log.info("Flushed transaction for %s %s" % (obj.log_str(), obj_flush_timer)) diff --git a/lib/galaxy/web/stack/__init__.py b/lib/galaxy/web/stack/__init__.py index 2a24b6d80a1..461e4a7d6a2 100644 --- a/lib/galaxy/web/stack/__init__.py +++ b/lib/galaxy/web/stack/__init__.py @@ -19,8 +19,8 @@ import yaml from six import string_types from galaxy.util import unicodify -from galaxy.util.bunch import Bunch from galaxy.util.facts import get_facts +from galaxy.util.handlers import HANDLER_ASSIGNMENT_METHODS from galaxy.util.path import has_ext from galaxy.util.properties import nice_config_parser from .message import ApplicationStackMessage, ApplicationStackMessageDispatcher @@ -54,12 +54,6 @@ class ApplicationStack(object): server_name_template = '{server_name}' default_app_name = 'main' - # used both to route jobs to a pool with this name and indicate whether or - # not a stack is using messaging for handler assignment - pools = Bunch( - JOB_HANDLERS='job-handlers', - ) - @classmethod def log_filter(cls): return cls.log_filter_class() @@ -77,6 +71,12 @@ class ApplicationStack(object): self.config = config or (app and app.config) self.running = False + def init_job_handling(self, job_config): + pass + + def init_late_prefork(self): + pass + def log_startup(self): log.info("Galaxy server instance '%s' is running" % self.app.config.server_name) @@ -97,12 +97,18 @@ class ApplicationStack(object): # TODO: ideally jobs would be mappable to handlers by pool name return None + def has_base_pool(self, pool_name): + return False + def has_pool(self, pool_name): return False def in_pool(self, pool_name): return False + def pool_members(self, pool_name): + return None + @property def facts(self): facts = get_facts(config=self.config) @@ -129,12 +135,16 @@ class ApplicationStack(object): class MessageApplicationStack(ApplicationStack): def __init__(self, app=None, config=None): super(MessageApplicationStack, self).__init__(app=app, config=config) + self.use_messaging = False self.dispatcher = ApplicationStackMessageDispatcher() self.transport = self.transport_class(app, stack=self, dispatcher=self.dispatcher) + def init_late_prefork(self): + self.transport.init_late_prefork() + def start(self): super(MessageApplicationStack, self).start() - if not self.running: + if self.use_messaging and not self.running: self.transport.start() self.running = True @@ -157,7 +167,7 @@ class MessageApplicationStack(ApplicationStack): self.transport.send_message(msg.encode(), dest) def shutdown(self): - if self.running: + if self.use_messaging and self.running: log.info('Application stack interface shutting down') self.transport.shutdown() self.running = False @@ -282,8 +292,85 @@ class UWSGIApplicationStack(MessageApplicationStack): def __init__(self, app=None, config=None): self._farms_dict = None self._mules_list = None + self._farm_server_names_dict = None + + # If there's more than one farm *and* more than one farm will be using farm messaging, additional locks need to + # be configured by the admin. This allows us to keep track of how many such farms are configured. + self._lock_farms = set() + super(UWSGIApplicationStack, self).__init__(app=app, config=config) + def _set_default_job_handler_assignment_methods(self, job_config, pool): + # This is called once per job_config (job config or workflow scheduling manager), and only in the case that + # handler assignment is not explicitly configured by the admin. Disable DB_SELF if a valid farm (pool) is + # configured. Use mule messaging unless the job_config doesn't allow it (e.g. workflow scheduling manager), in + # which case, use DB_PREASSIGN. + # + # TODO MULTIPOOL: if there is no default in any base_pool (and no job_config.default_handler_id) then don't + # remove DB_SELF + conf_class_name = job_config.__class__.__name__ + remove_methods = [HANDLER_ASSIGNMENT_METHODS.DB_SELF] + if (HANDLER_ASSIGNMENT_METHODS.UWSGI_MULE_MESSAGE not in job_config.UNSUPPORTED_HANDLER_ASSIGNMENT_METHODS): + add_method = HANDLER_ASSIGNMENT_METHODS.UWSGI_MULE_MESSAGE + else: + add_method = HANDLER_ASSIGNMENT_METHODS.DB_PREASSIGN + remove_methods.append(HANDLER_ASSIGNMENT_METHODS.UWSGI_MULE_MESSAGE) + log.debug("%s: No job handler assignment methods were configured but a uWSGI farm named '%s' exists," + " automatically enabling the '%s' assignment method", conf_class_name, pool, add_method) + for m in remove_methods: + try: + job_config.handler_assignment_methods.remove(m) + log.debug("%s: Removed '%s' from handler assignment methods due to use of mules", conf_class_name, m) + except ValueError: + pass + if add_method not in job_config.handler_assignment_methods: + job_config.handler_assignment_methods.insert(0, add_method) + log.debug("%s: handler assignment methods updated to: %s", conf_class_name, + ', '.join(job_config.handler_assignment_methods)) + + def _init_job_handler_assignment_methods(self, job_config, pool): + if not job_config.handler_assignment_methods_configured: + self._set_default_job_handler_assignment_methods(job_config, pool) + if HANDLER_ASSIGNMENT_METHODS.UWSGI_MULE_MESSAGE in job_config.handler_assignment_methods: + self.use_messaging = True + + def _init_job_handler_subpools(self, job_config, pool): + # Set up members of "subpools" ("pool.*") as handlers (including the base pool itself, if it exists) + for farm in self._configured_farms: + if not (farm == pool or farm.startswith(pool + '.')): + continue + if job_config.use_messaging: + self._lock_farms.add(farm) + if farm == pool: + tag = job_config.DEFAULT_HANDLER_TAG + else: + tag = farm.replace(pool + '.', '', 1) + # Pools are hierarchical (so that you can have e.g. workflow schedulers use the job handlers pool if no + # workflow schedulers pool exists), so if a pool for a given tag has already been found higher in the + # hierarchy, don't add mules from a farm/pool lower in the hierarchy. + # FIXME: test + if tag not in job_config.pool_for_tag: + if self.in_pool(farm): + job_config.is_handler = True + for handler in self.pool_members(farm): + job_config.add_handler(handler, [tag]) + job_config.pool_for_tag[tag] = farm + + def init_job_handling(self, job_config): + """Automatically add uWSGI farms as pools if they are named per predefined names and there is not an explicit + job handler assignment configuration. + + Also automatically add mule messaging as the preferred assignment method, if supported. + """ + # Only do the stack handler assignment init once per job_config + stack_assignment_methods_configured = False + for pool in job_config.DEFAULT_BASE_HANDLER_POOLS: + if self.has_base_pool(pool): + if not stack_assignment_methods_configured: + self._init_job_handler_assignment_methods(job_config, pool) + stack_assignment_methods_configured = True + self._init_job_handler_subpools(job_config, pool) + @property def _configured_mules(self): if self._mules_list is None: @@ -314,20 +401,39 @@ class UWSGIApplicationStack(MessageApplicationStack): farms.append(farm) return farms - def _mule_index_in_farm(self, farm_name): + def _mule_index_in_farm(self, farm_name, mule_id=None): + mule_id = mule_id or uwsgi.mule_id() try: mules = self._configured_farms[farm_name] - return mules.index(uwsgi.mule_id()) + return mules.index(mule_id) except (KeyError, ValueError): return -1 @property def _farm_name(self): + # TODO: to allow mules to be in multiple farms you'll need to start here try: return self._farms[0] except IndexError: return None + @property + def _farm_server_names(self): + if self._farm_server_names_dict is None: + self._farm_server_names_dict = {} + for farm_name, mules in self._configured_farms.items(): + server_names = [] + facts = self.facts + for mule in mules: + facts.update({ + 'pool_name': farm_name, + 'server_id': mule, + 'instance_id': self._mule_index_in_farm(farm_name, mule) + 1, + }) + server_names.append(self.server_name_template.format(**facts)) + self._farm_server_names_dict[farm_name] = server_names + return self._farm_server_names_dict + @property def instance_id(self): if not self._is_mule: @@ -363,6 +469,9 @@ class UWSGIApplicationStack(MessageApplicationStack): os.environ['_GALAXY_UWSGI_FARM_NAMES'] = ','.join(self._farms) super(UWSGIApplicationStack, self).start() + def has_base_pool(self, pool_name): + return self.has_pool(pool_name) or any([farm.startswith(pool_name + '.') for farm in self._configured_farms]) + def has_pool(self, pool_name): return pool_name in self._configured_farms @@ -372,6 +481,9 @@ class UWSGIApplicationStack(MessageApplicationStack): else: return pool_name in self._farms + def pool_members(self, pool_name): + return self._farm_server_names.get(pool_name, None) + def workers(self): return uwsgi.workers() diff --git a/lib/galaxy/web/stack/transport.py b/lib/galaxy/web/stack/transport.py index 24930cf1ca1..233fed2b013 100644 --- a/lib/galaxy/web/stack/transport.py +++ b/lib/galaxy/web/stack/transport.py @@ -29,6 +29,9 @@ class ApplicationStackTransport(object): self.dispatcher = dispatcher self.dispatcher_thread = None + def init_late_prefork(self): + pass + def _dispatch_messages(self): pass @@ -70,13 +73,12 @@ class UWSGIFarmMessageTransport(ApplicationStackTransport): # Define any static lock names here, additional locks will be appended for each configured farm's message handler _locks = [] - def __initialize_locks(self): + def init_late_prefork(self): num = int(uwsgi.opt.get('locks', 0)) + 1 - farms = self.stack._configured_farms.keys() - need = len(farms) + need = len(self.stack._lock_farms) if num < need: raise RuntimeError('Need %i uWSGI locks but only %i exist(s): Set `locks = %i` in uWSGI configuration' % (need, num, need - 1)) - self._locks.extend(['RECV_MSG_FARM_' + x for x in farms]) + self._locks.extend(['RECV_MSG_FARM_' + x for x in sorted(self.stack._lock_farms)]) # this would be nice, but in my 2.0.15 uWSGI, the uwsgi module has no set_option function, and I don't know if it'd work even if the function existed as documented # if len(self.lock_map) > 1: # uwsgi.set_option('locks', len(self.lock_map)) @@ -84,7 +86,6 @@ class UWSGIFarmMessageTransport(ApplicationStackTransport): def __init__(self, app, stack, dispatcher=None): super(UWSGIFarmMessageTransport, self).__init__(app, stack, dispatcher=dispatcher) - self.__initialize_locks() def __lock(self, name_or_id): try: diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index ac51424c243..db25f4c479c 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -203,7 +203,7 @@ class JobController(BaseAPIController, UsesLibraryMixinItems): if not job.finished: job.mark_deleted(self.app.config.track_jobs_in_database) trans.sa_session.flush() - self.app.job_manager.job_stop_queue.put(job.id) + self.app.job_manager.stop(job) return True else: return False diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 60fab94a475..854ad3f8170 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -2437,6 +2437,7 @@ mapping: failing jobs are just failed outright. track_jobs_in_database: + # FIXME: note deprecated, superceded by job_handler_assignment_method type: bool default: true required: false diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py index a8d411be2e0..0b2a263a275 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -1648,7 +1648,7 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP job.set_state(trans.app.model.Job.states.DELETED_NEW) trans.sa_session.add(job) else: - trans.app.job_manager.job_stop_queue.put(job_id, error_msg=error_msg) + trans.app.job_manager.stop(job, message=error_msg) deleted.append(str(job_id)) if deleted: msg = 'Queued job' diff --git a/lib/galaxy/webapps/galaxy/controllers/history.py b/lib/galaxy/webapps/galaxy/controllers/history.py index 238319c6634..64a766a38f0 100644 --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -360,7 +360,7 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt if job.history_id == history.id and not job.finished: # No need to check other outputs since the job's parent history is this history job.mark_deleted(trans.app.config.track_jobs_in_database) - trans.app.job_manager.job_stop_queue.put(job.id) + trans.app.job_manager.stop(job) trans.sa_session.flush() if n_deleted: part = "Deleted %d %s" % (n_deleted, iff(n_deleted != 1, "histories", "history")) diff --git a/lib/galaxy/workflow/scheduling_manager.py b/lib/galaxy/workflow/scheduling_manager.py index fc0b4c8e37b..c7801c99df6 100644 --- a/lib/galaxy/workflow/scheduling_manager.py +++ b/lib/galaxy/workflow/scheduling_manager.py @@ -1,11 +1,13 @@ import logging import os +from functools import partial from xml.etree import ElementTree import galaxy.workflow.schedulers from galaxy import model +from galaxy.exceptions import HandlerAssignmentError from galaxy.util import plugin_config -from galaxy.util.handlers import ConfiguresHandlers +from galaxy.util.handlers import ConfiguresHandlers, HANDLER_ASSIGNMENT_METHODS from galaxy.util.monitors import Monitors from galaxy.web.stack.message import WorkflowSchedulingMessage @@ -18,6 +20,7 @@ EXCEPTION_MESSAGE_SHUTDOWN = "Exception raised while attempting to shutdown work EXCEPTION_MESSAGE_NO_SCHEDULERS = "Failed to defined workflow schedulers - no workflow schedulers defined." EXCEPTION_MESSAGE_NO_DEFAULT_SCHEDULER = "Failed to defined workflow schedulers - no workflow scheduler found for default id '%s'." EXCEPTION_MESSAGE_DUPLICATE_SCHEDULERS = "Failed to defined workflow schedulers - workflow scheduling plugin id '%s' duplicated." +EXCEPTION_MESSAGE_SERIALIZE = "Parallelization is not desired but handler assignment methods are non-deterministic. Set DB_PREASSIGN in workflow_schedulers_conf.xml." class WorkflowSchedulingManager(ConfiguresHandlers): @@ -25,22 +28,24 @@ class WorkflowSchedulingManager(ConfiguresHandlers): ``galaxy.manager.JobManager``. Only schedules workflows on handler processes. """ + DEFAULT_BASE_HANDLER_POOLS = ('workflow-schedulers', 'job-handlers') + UNSUPPORTED_HANDLER_ASSIGNMENT_METHODS = ( + HANDLER_ASSIGNMENT_METHODS.UWSGI_MULE_MESSAGE, + ) def __init__(self, app): self.app = app self.__handlers_configured = False self.workflow_schedulers = {} self.active_workflow_schedulers = {} - # TODO: this should not hardcode the job handlers pool - self.__handler_pool = self.app.application_stack.pools.JOB_HANDLERS - # TODO: and we need a better way to indicate messaging should be used - self.__use_stack_messages = app.application_stack.has_pool(self.__handler_pool) # Passive workflow schedulers won't need to be monitored I guess. self.request_monitor = None self.handlers = {} - self._is_handler = None + self.handler_assignment_methods_configured = False + self.handler_assignment_methods = None + self.default_handler_id = None self.__plugin_classes = self.__plugins_dict() self.__init_schedulers() @@ -50,7 +55,17 @@ class WorkflowSchedulingManager(ConfiguresHandlers): self.__start_schedulers() if self.active_workflow_schedulers: self.__start_request_monitor() - if self.__use_stack_messages: + + # When assinging handlers to workflows being queued - use job_conf + # if not explicit workflow scheduling handlers have be specified or + # else use those explicit workflow scheduling handlers (on self). + if self.__handlers_configured: + self.__handlers_config = self + else: + self.__handlers_config = app.job_config + + if self._is_workflow_handler(): + if self.__handlers_config.use_messaging: WorkflowSchedulingMessage().bind_default_handler(self, '_handle_message') self.app.application_stack.register_message_handler( self._handle_message, @@ -59,24 +74,13 @@ class WorkflowSchedulingManager(ConfiguresHandlers): # Process should not schedule workflows but should check for any unassigned to handlers self.__startup_recovery() - # When assinging handlers to workflows being queued - use job_conf - # if not explicit workflow scheduling handlers have be specified or - # else use those explicit workflow scheduling handlers (on self). - if self.__handlers_configured: - self.__has_handlers = self - else: - self.__has_handlers = app.job_config - def __startup_recovery(self): sa_session = self.app.model.context - if self.__use_stack_messages: - for workflow_invocation in model.WorkflowInvocation.poll_active_workflow_ids( - sa_session, - handler=None): - log.info("(%s) Handler unassigned at startup, queueing workflow invocation via stack messaging for pool" - " [%s]", workflow_invocation.id, self.__handler_pool) - msg = WorkflowSchedulingMessage(task='setup', workflow_invocation_id=workflow_invocation.id) - self.app.application_stack.send_message(self.app.application_stack.pools.JOB_HANDLERS, msg) + for invocation_id in model.WorkflowInvocation.poll_unhandled_workflow_ids(sa_session): + log.info("(%s) Handler unassigned at startup, resubmitting workflow invocation for assignment", + invocation_id) + workflow_invocation = sa_session.query(model.WorkflowInvocation).get(invocation_id) + self._assign_handler(workflow_invocation) def _handle_setup_msg(self, workflow_invocation_id=None): sa_session = self.app.model.context @@ -99,13 +103,27 @@ class WorkflowSchedulingManager(ConfiguresHandlers): is_handler = self.app.is_job_handler return is_handler - def _get_handler(self, history_id): + def _queue_callback(self, workflow_invocation): + # There isn't an in-memory queue for workflow schedulers, so if MEM_SELF is used just assign with the DB + workflow_invocation.handler = self.app.config.server_name + sa_session = self.app.model.context + sa_session.add(workflow_invocation) + sa_session.flush() + + def _message_callback(self, workflow_invocation): + return WorkflowSchedulingMessage(task='setup', workflow_invocation_id=workflow_invocation.id) + + def _assign_handler(self, workflow_invocation): # Use random-ish integer history_id to produce a consistent index to pick # job handler with. - random_index = history_id + random_index = workflow_invocation.history_id + queue_callback = partial(self._queue_callback, workflow_invocation) + message_callback = partial(self._message_callback, workflow_invocation) if self.app.config.parallelize_workflow_scheduling_within_histories: random_index = None - return self.__has_handlers.get_handler(None, index=random_index) + return self.__handlers_config.assign_handler( + workflow_invocation, configured=None, index=random_index, queue_callback=queue_callback, + message_callback=message_callback) def shutdown(self): exception = None @@ -127,25 +145,16 @@ class WorkflowSchedulingManager(ConfiguresHandlers): def queue(self, workflow_invocation, request_params): workflow_invocation.state = model.WorkflowInvocation.states.NEW - scheduler = request_params.get("scheduler", None) or self.default_scheduler_id - handler = self._get_handler(workflow_invocation.history.id) - - if handler is None and not self.__use_stack_messages: - raise RuntimeError("Unable to set a handler for workflow invocation '%s'" % workflow_invocation.id) - - log.info("Queueing workflow invocation for handler [%s]", handler) - workflow_invocation.scheduler = scheduler - workflow_invocation.handler = handler - + workflow_invocation.scheduler = request_params.get("scheduler", None) or self.default_scheduler_id sa_session = self.app.model.context sa_session.add(workflow_invocation) - sa_session.flush() - if handler is None and self.__use_stack_messages: - log.info("(%s) Queueing workflow invocation via stack messaging for pool [%s]", - workflow_invocation.id, self.__handler_pool) - msg = WorkflowSchedulingMessage(task='setup', workflow_invocation_id=workflow_invocation.id) - self.app.application_stack.send_message(self.__handler_pool, msg) + # Assign handler (also performs the flush) + try: + self._assign_handler(workflow_invocation) + except HandlerAssignmentError: + # FIXME: test, raise new exception? + raise RuntimeError("Unable to set a handler for workflow invocation '%s'" % workflow_invocation.id) return workflow_invocation @@ -156,11 +165,17 @@ class WorkflowSchedulingManager(ConfiguresHandlers): def __plugins_dict(self): return plugin_config.plugins_dict(galaxy.workflow.schedulers, 'plugin_type') + @property + def __stack_has_pool(self): + # TODO: In the future it should be possible to map workflows to handlers based on workflow params. When that + # happens, we'll need to defer pool checks until execution time. + return any(map(self.app.application_stack.has_pool, self.DEFAULT_BASE_HANDLER_POOLS)) + def __init_schedulers(self): config_file = self.app.config.workflow_schedulers_config_file use_default_scheduler = False if not config_file: - log.info("Not workflow schedulers plugin config file defined, using default scheduler.") + log.info("No workflow schedulers plugin config file defined, using default scheduler.") use_default_scheduler = True elif not os.path.exists(config_file): log.info("Cannot find workflow schedulers plugin config file '%s', using default scheduler." % config_file) @@ -172,6 +187,11 @@ class WorkflowSchedulingManager(ConfiguresHandlers): plugins_element = ElementTree.parse(config_file).getroot() self.__init_schedulers_for_element(plugins_element) + if not self.__handlers_configured and self.__stack_has_pool: + # Stack has a pool for us so override inherited config and use the pool + self.__init_handler_assignment_methods() + self.__handlers_configured = True + def __init_default_scheduler(self): self.default_scheduler_id = DEFAULT_SCHEDULER_ID self.__init_plugin(DEFAULT_SCHEDULER_PLUGIN_TYPE) @@ -198,12 +218,27 @@ class WorkflowSchedulingManager(ConfiguresHandlers): raise Exception(EXCEPTION_MESSAGE_NO_SCHEDULERS) if self.default_scheduler_id not in self.workflow_schedulers: raise Exception(EXCEPTION_MESSAGE_NO_DEFAULT_SCHEDULER % self.default_scheduler_id) + if self.app.config.parallelize_workflow_scheduling_within_histories \ + and not self.deterministic_handler_assignment: + if self.__handlers_configured: + # There's an explicit configuration, the admin should fix it. + raise Exception(EXCEPTION_MESSAGE_SERIALIZE) + else: + log.warning(EXCEPTION_MESSAGE_SERIALIZE) def __init_handlers(self, config_element): assert not self.__handlers_configured + self.__init_handler_assignment_methods(config_element) self._init_handlers(config_element) self.__handlers_configured = True + def __init_handler_assignment_methods(self, config_element=None): + self._init_handler_assignment_methods(config_element) + self._set_default_handler_assignment_methods() + log.info("Workflow scheduling handler assignment method(s): %s", ', '.join(self.handler_assignment_methods)) + for tag, handlers in [(t, h) for t, h in self.handlers.items() if isinstance(h, list)]: + log.info("Tag [%s] handlers: %s", tag, ', '.join(handlers)) + def __init_plugin(self, plugin_type, workflow_scheduler_id=None, **kwds): workflow_scheduler_id = workflow_scheduler_id or self.default_scheduler_id @@ -241,6 +276,7 @@ class WorkflowRequestMonitor(Monitors): def __schedule(self, workflow_scheduler_id, workflow_scheduler): invocation_ids = self.__active_invocation_ids(workflow_scheduler_id) for invocation_id in invocation_ids: + log.debug("Attempting to schedule workflow invocation [%s]", invocation_id) self.__attempt_schedule(invocation_id, workflow_scheduler) if not self.monitor_running: return @@ -261,6 +297,7 @@ class WorkflowRequestMonitor(Monitors): if i.active and i.id < workflow_invocation.id: return False workflow_scheduler.schedule(workflow_invocation) + log.debug("Workflow invocation [%s] scheduled", workflow_invocation.id) except Exception: # TODO: eventually fail this - or fail it right away? log.exception("Exception raised while attempting to schedule workflow request.") diff --git a/test/unit/jobs/test_job_configuration.py b/test/unit/jobs/test_job_configuration.py index 2982cdb87cb..f10d5e9f800 100644 --- a/test/unit/jobs/test_job_configuration.py +++ b/test/unit/jobs/test_job_configuration.py @@ -25,6 +25,7 @@ class JobConfXmlParserTestCase(unittest.TestCase): job_resource_params_file="/tmp/fake_absent_path", config_dict={}, default_job_resubmission_condition="", + track_jobs_in_database=True, server_name="main", ) self.__write_config_from(SIMPLE_JOB_CONF) @@ -50,12 +51,10 @@ class JobConfXmlParserTestCase(unittest.TestCase): assert len(task_runners) == 1 assert task_runners[0]["workers"] == 5 - def test_load_simple_handler(self): - main_handler = self.job_config.handlers["main"] - assert main_handler[0] == "main" - - def test_if_one_handler_implicit_default(self): - assert self.job_config.default_handler_id == "main" + def test_if_no_handlers_implict_db_self(self): + assert self.job_config.default_handler_id is None + assert self.job_config.handlers == {} + assert self.job_config.handler_assignment_methods == ['db-self'] def test_explicit_handler_default(self): self.__with_advanced_config() diff --git a/test/unit/tools/test_actions.py b/test/unit/tools/test_actions.py index 502f65ea994..3c23507ab6b 100644 --- a/test/unit/tools/test_actions.py +++ b/test/unit/tools/test_actions.py @@ -12,8 +12,6 @@ from galaxy.tools.actions import ( from galaxy.tools.parser.output_objects import ToolOutput from .. import tools_support -TEST_HANDLER_NAME = "test_handler_1" - # I cannot think of a saner way to test if data is being wrapped than use a # data param in the output label - though you would probably never want to do @@ -73,7 +71,6 @@ class DefaultToolActionTestCase(unittest.TestCase, tools_support.UsesApp, tools_ self.app.model.context.flush() self.action = DefaultToolAction() self.app.config.len_file_path = "moocow" - self.app.job_config["get_handler"] = lambda h: TEST_HANDLER_NAME self.app.object_store = MockObjectStore() def test_output_created(self): @@ -114,10 +111,6 @@ class DefaultToolActionTestCase(unittest.TestCase, tools_support.UsesApp, tools_ # Again this is a stupid way to ensure data parameters are wrapped. self.assertEqual(output["out1"].name, "Output (%s)" % hda1.dataset.get_file_name()) - def test_handler_set(self): - job, _ = self._simple_execute() - assert job.handler == TEST_HANDLER_NAME - def test_inactive_user_job_create_failure(self): self.trans.user_is_active = False try: diff --git a/test/unit/unittest_utils/galaxy_mock.py b/test/unit/unittest_utils/galaxy_mock.py index 7c3bad26452..36f6563bd7b 100644 --- a/test/unit/unittest_utils/galaxy_mock.py +++ b/test/unit/unittest_utils/galaxy_mock.py @@ -11,7 +11,7 @@ from galaxy import ( quota ) from galaxy.datatypes import registry -from galaxy.jobs import NoopQueue +from galaxy.jobs.manager import NoopManager from galaxy.managers import tags from galaxy.model import mapping from galaxy.tools.deps.containers import NullContainerFinder @@ -76,7 +76,7 @@ class MockApp(object): self.container_finder = NullContainerFinder() self._toolbox_lock = MockLock() self.genome_builds = GenomeBuilds(self) - self.job_manager = Bunch(job_queue=NoopQueue()) + self.job_manager = NoopManager() self.application_stack = ApplicationStack() def init_datatypes(self): From 10c45705bca081c336214b08fcc84d209b19b345 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 10 Dec 2018 15:32:32 -0500 Subject: [PATCH 065/260] Unit tests and fixes for bugs discovered while testing. --- lib/galaxy/jobs/__init__.py | 10 +- lib/galaxy/util/facts.py | 1 + lib/galaxy/util/handlers.py | 6 - lib/galaxy/web/stack/__init__.py | 6 +- lib/galaxy/workflow/scheduling_manager.py | 6 +- test/unit/jobs/handler_template_job_conf.xml | 12 ++ test/unit/jobs/test_job_configuration.py | 124 +++++++++++++++++-- 7 files changed, 141 insertions(+), 24 deletions(-) create mode 100644 test/unit/jobs/handler_template_job_conf.xml diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index e4b92978113..77a8aaa2cbc 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -217,7 +217,10 @@ class JobConfiguration(ConfiguresHandlers): handlers_conf = root.find('handlers') self._init_handler_assignment_methods(handlers_conf) self._init_handlers(handlers_conf) - self._set_default_handler_assignment_methods() + if not self.handler_assignment_methods_configured: + self._set_default_handler_assignment_methods() + else: + self.app.application_stack.init_job_handling(self) log.info("Job handler assignment methods set to: %s", ', '.join(self.handler_assignment_methods)) for tag, handlers in [(t, h) for t, h in self.handlers.items() if isinstance(h, list)]: log.info("Tag [%s] handlers: %s", tag, ', '.join(handlers)) @@ -341,7 +344,10 @@ class JobConfiguration(ConfiguresHandlers): # Set the handlers self._init_handler_assignment_methods() self._init_handlers() - self._set_default_handler_assignment_methods() + if not self.handler_assignment_methods_configured: + self._set_default_handler_assignment_methods() + else: + self.app.application_stack.init_job_handling(self) # Set the destination self.default_destination_id = 'local' self.destinations['local'] = [JobDestination(id='local', runner='local')] diff --git a/lib/galaxy/util/facts.py b/lib/galaxy/util/facts.py index 9a5fdbbd0ea..2de16277f30 100644 --- a/lib/galaxy/util/facts.py +++ b/lib/galaxy/util/facts.py @@ -10,6 +10,7 @@ class Facts(MutableMapping): """A dict-like object that evaluates values at access time.""" def __init__(self, config=None, **kwargs): + config = config or {} self.__dict__ = {} self.__set_defaults(config) self.__set_config(config) diff --git a/lib/galaxy/util/handlers.py b/lib/galaxy/util/handlers.py index ab31a21649c..faa964e0005 100644 --- a/lib/galaxy/util/handlers.py +++ b/lib/galaxy/util/handlers.py @@ -92,11 +92,6 @@ class ConfiguresHandlers(object): if not self.app.config.track_jobs_in_database and \ HANDLER_ASSIGNMENT_METHODS.MEM_SELF not in self.UNSUPPORTED_HANDLER_ASSIGNMENT_METHODS: # DEPRECATED: You should just set mem_self as the only method if you want this - # FIXME: does setting MEM_SELF still disable track_jobs_in_database like it used to? - # FIXME: test MEM_SELF via all config routes: - # 1. no assign config, track_jobs_in_databse = False in config - # 2. assign_with='mem-self', track_jobs_in_databse = False in config - # 2. assign_with='mem-self', track_jobs_in_databse = True in config log.warning("The `track_jobs_in_database` option is deprecated, please set `%s` as the job" " handler assignment method in the job handler configuration", HANDLER_ASSIGNMENT_METHODS.MEM_SELF) @@ -262,7 +257,6 @@ class ConfiguresHandlers(object): :returns: str -- A valid job handler ID. """ - # FIXME: test a combo of pool handlers and defined handlers w/ DB_PREASSIGN as the only method handler = configured if handler is None: handler = self.default_handler_id or self.DEFAULT_HANDLER_TAG diff --git a/lib/galaxy/web/stack/__init__.py b/lib/galaxy/web/stack/__init__.py index 461e4a7d6a2..ec5e772be73 100644 --- a/lib/galaxy/web/stack/__init__.py +++ b/lib/galaxy/web/stack/__init__.py @@ -7,9 +7,8 @@ import json import logging import os -# The uwsgi module is automatically injected by the parent uwsgi -# process and only exists that way. If anything works, this is a -# uwsgi-managed process. +# The uwsgi module is automatically injected by the parent uwsgi process and only exists that way. If anything works, +# this is a uwsgi-managed process. try: import uwsgi except ImportError: @@ -348,7 +347,6 @@ class UWSGIApplicationStack(MessageApplicationStack): # Pools are hierarchical (so that you can have e.g. workflow schedulers use the job handlers pool if no # workflow schedulers pool exists), so if a pool for a given tag has already been found higher in the # hierarchy, don't add mules from a farm/pool lower in the hierarchy. - # FIXME: test if tag not in job_config.pool_for_tag: if self.in_pool(farm): job_config.is_handler = True diff --git a/lib/galaxy/workflow/scheduling_manager.py b/lib/galaxy/workflow/scheduling_manager.py index c7801c99df6..58d8d0d574a 100644 --- a/lib/galaxy/workflow/scheduling_manager.py +++ b/lib/galaxy/workflow/scheduling_manager.py @@ -153,7 +153,6 @@ class WorkflowSchedulingManager(ConfiguresHandlers): try: self._assign_handler(workflow_invocation) except HandlerAssignmentError: - # FIXME: test, raise new exception? raise RuntimeError("Unable to set a handler for workflow invocation '%s'" % workflow_invocation.id) return workflow_invocation @@ -234,7 +233,10 @@ class WorkflowSchedulingManager(ConfiguresHandlers): def __init_handler_assignment_methods(self, config_element=None): self._init_handler_assignment_methods(config_element) - self._set_default_handler_assignment_methods() + if not self.handler_assignment_methods_configured: + self._set_default_handler_assignment_methods() + else: + self.app.application_stack.init_job_handling(self) log.info("Workflow scheduling handler assignment method(s): %s", ', '.join(self.handler_assignment_methods)) for tag, handlers in [(t, h) for t, h in self.handlers.items() if isinstance(h, list)]: log.info("Tag [%s] handlers: %s", tag, ', '.join(handlers)) diff --git a/test/unit/jobs/handler_template_job_conf.xml b/test/unit/jobs/handler_template_job_conf.xml new file mode 100644 index 00000000000..fe95cc785d3 --- /dev/null +++ b/test/unit/jobs/handler_template_job_conf.xml @@ -0,0 +1,12 @@ + + + + + + + {handlers} + + + + + diff --git a/test/unit/jobs/test_job_configuration.py b/test/unit/jobs/test_job_configuration.py index f10d5e9f800..034fa75e471 100644 --- a/test/unit/jobs/test_job_configuration.py +++ b/test/unit/jobs/test_job_configuration.py @@ -4,15 +4,18 @@ import shutil import tempfile import unittest +import mock + from galaxy.jobs import JobConfiguration from galaxy.util import bunch -from galaxy.web.stack import ApplicationStack +from galaxy.web.stack import ApplicationStack, UWSGIApplicationStack # File would be slightly more readable if contents were embedded directly, but # there are advantages to testing the documentation/examples. SIMPLE_JOB_CONF = os.path.join(os.path.dirname(__file__), "..", "..", "..", "config", "job_conf.xml.sample_basic") ADVANCED_JOB_CONF = os.path.join(os.path.dirname(__file__), "..", "..", "..", "config", "job_conf.xml.sample_advanced") CONDITIONAL_RUNNER_JOB_CONF = os.path.join(os.path.dirname(__file__), "conditional_runners_job_conf.xml") +HANDLER_TEMPLATE_JOB_CONF = os.path.join(os.path.dirname(__file__), "handler_template_job_conf.xml") class JobConfXmlParserTestCase(unittest.TestCase): @@ -29,8 +32,11 @@ class JobConfXmlParserTestCase(unittest.TestCase): server_name="main", ) self.__write_config_from(SIMPLE_JOB_CONF) - self.app = bunch.Bunch(config=self.config, job_metrics=MockJobMetrics(), application_stack=ApplicationStack()) + self.__app = None + self.__application_stack = None self.__job_configuration = None + self.__job_configuration_base_pools = None + self.__uwsgi_opt = None def tearDown(self): shutil.rmtree(self.temp_directory) @@ -51,11 +57,6 @@ class JobConfXmlParserTestCase(unittest.TestCase): assert len(task_runners) == 1 assert task_runners[0]["workers"] == 5 - def test_if_no_handlers_implict_db_self(self): - assert self.job_config.default_handler_id is None - assert self.job_config.handlers == {} - assert self.job_config.handler_assignment_methods == ['db-self'] - def test_explicit_handler_default(self): self.__with_advanced_config() assert self.job_config.default_handler_id == "handlers" @@ -65,6 +66,69 @@ class JobConfXmlParserTestCase(unittest.TestCase): assert "handler0" in self.job_config.handlers["handlers"] assert "handler1" in self.job_config.handlers["handlers"] + def test_implict_db_self_handler_assign(self): + assert self.job_config.handler_assignment_methods == ['db-self'] + assert self.job_config.default_handler_id is None + assert self.job_config.handlers == {} + + def test_implicit_db_assign_handler_assign_with_explicit_handlers(self): + self.__with_handlers_config(handlers=[{'id': 'handler0'}, {'id': 'handler1'}]) + assert self.job_config.handler_assignment_methods == ['db-preassign'] + assert self.job_config.default_handler_id is None + assert self.job_config.handlers['_default_'] == ['handler0', 'handler1'] + + def test_implict_uwsgi_mule_message_handler_assign(self): + self.__with_uwsgi_application_stack(mule='lib/galaxy/main.py', farm='job-handlers:1') + assert self.job_config.handler_assignment_methods == ['uwsgi-mule-message'] + assert self.job_config.default_handler_id is None + assert self.job_config.handlers['_default_'] == ['main.job-handlers.1'] + + def test_implict_uwsgi_mule_message_handler_assign_with_explicit_handlers(self): + self.__with_handlers_config(handlers=[{'id': 'handler0'}, {'id': 'handler1'}]) + self.__with_uwsgi_application_stack(mule='lib/galaxy/main.py', farm='job-handlers:1') + assert self.job_config.handler_assignment_methods == ['uwsgi-mule-message', 'db-preassign'] + assert self.job_config.default_handler_id is None + assert self.job_config.handlers['_default_'] == ['handler0', 'handler1', 'main.job-handlers.1'] + + def test_explicit_mem_self_handler_assign(self): + self.__with_handlers_config(assign_with='mem-self') + assert self.job_config.handler_assignment_methods == ['mem-self'] + assert self.job_config.default_handler_id is None + assert self.job_config.handlers == {} + assert not self.config.track_jobs_in_database + + def test_explicit_mem_self_handler_assign_with_handlers(self): + self.__with_handlers_config(assign_with='mem-self', handlers=[{'id': 'handler0'}]) + assert self.job_config.handler_assignment_methods == ['mem-self'] + assert self.job_config.default_handler_id is None + assert self.job_config.handlers['_default_'] == ['handler0'] + + def test_explicit_db_preassign_handler_assign_with_uwsgi(self): + self.__with_handlers_config(assign_with='db-preassign', handlers=[{'id': 'handler0'}]) + self.__with_uwsgi_application_stack(mule='lib/galaxy/main.py', farm='job-handlers:1') + assert self.job_config.handler_assignment_methods == ['db-preassign'] + assert self.job_config.default_handler_id is None + assert self.job_config.handlers['_default_'] == ['handler0', 'main.job-handlers.1'] + + def test_uwsgi_farms_as_handler_tags(self): + self.__with_uwsgi_application_stack( + mule=['lib/galaxy/main.py'] * 2, + farm=['job-handlers:1', 'job-handlers.foo:2'] + ) + assert self.job_config.default_handler_id is None + assert self.job_config.handlers['_default_'] == ['main.job-handlers.1'] + assert self.job_config.handlers['foo'] == ['main.job-handlers.foo.1'] + + def test_uwsgi_overlapping_pools(self): + self.__with_handlers_config(base_pools=('workflow-schedulers', 'job-handlers')) + self.__with_uwsgi_application_stack( + mule=['lib/galaxy/main.py'] * 3, + farm=['job-handlers:1', 'workflow-schedulers:2', 'job-handlers.foo:3'] + ) + assert self.job_config.default_handler_id is None + assert self.job_config.handlers['_default_'] == ['main.workflow-schedulers.1'] + assert self.job_config.handlers['foo'] == ['main.job-handlers.foo.1'] + def test_load_simple_destination(self): local_dest = self.job_config.destinations["local"][0] assert local_dest.id == "local" @@ -165,17 +229,57 @@ class JobConfXmlParserTestCase(unittest.TestCase): # TODO: Add job metrics parsing test. + @property + def app(self): + if not self.__app: + self.__app = bunch.Bunch( + config=self.config, + job_metrics=MockJobMetrics(), + application_stack=self.application_stack + ) + return self.__app + + @property + def application_stack(self): + if not self.__application_stack: + self.__application_stack = ApplicationStack() + return self.__application_stack + @property def job_config(self): if not self.__job_configuration: - self.__job_configuration = JobConfiguration(self.app) + base_handler_pools = self.__job_configuration_base_pools or JobConfiguration.DEFAULT_BASE_HANDLER_POOLS + mock_uwsgi = mock.Mock() + mock_uwsgi.mule_id = lambda: 1 + with mock.patch('galaxy.web.stack.uwsgi', mock_uwsgi), \ + mock.patch('galaxy.web.stack.uwsgi.opt', self.__uwsgi_opt), \ + mock.patch('galaxy.jobs.JobConfiguration.DEFAULT_BASE_HANDLER_POOLS', base_handler_pools): + self.__job_configuration = JobConfiguration(self.app) return self.__job_configuration + def __with_uwsgi_application_stack(self, **uwsgi_opt): + self.__uwsgi_opt = uwsgi_opt + self.__application_stack = UWSGIApplicationStack() + def __with_advanced_config(self): self.__write_config_from(ADVANCED_JOB_CONF) - def __write_config_from(self, path): - self.__write_config(open(path, "r").read()) + def __with_handlers_config(self, assign_with=None, handlers=None, base_pools=None): + handlers = handlers or [] + template = { + 'assign_with': ' assign_with="%s"' % assign_with if assign_with is not None else '', + 'handlers': '\n'.join([ + ''.format( + id=x['id'], + tags=' tags="%s"' % x['tags'] if 'tags' in x else '' + ) for x in handlers]), + } + self.__job_configuration_base_pools = base_pools + self.__write_config_from(HANDLER_TEMPLATE_JOB_CONF, template=template) + + def __write_config_from(self, path, template=None): + template = template or {} + self.__write_config(open(path, "r").read().format(**template)) def __write_config(self, contents): with open(os.path.join(self.temp_directory, "job_conf.xml"), "w") as f: From 1fb8203a0c1f6f941e0b13fa6357ec04ea677ffb Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 10 Dec 2018 17:16:15 -0500 Subject: [PATCH 066/260] Handler assignment documentation. --- config/galaxy.yml.sample | 5 +- config/job_conf.xml.sample_advanced | 66 ++++++++++++++++++--- config/workflow_schedulers_conf.xml.sample | 27 +++++++-- doc/source/admin/galaxy_options.rst | 6 +- doc/source/admin/jobs.md | 2 +- doc/source/admin/scaling.md | 20 +++++-- lib/galaxy/jobs/manager.py | 8 +-- lib/galaxy/webapps/galaxy/config_schema.yml | 6 +- 8 files changed, 105 insertions(+), 35 deletions(-) diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample index fb4351ba4be..12e57d2778f 100644 --- a/config/galaxy.yml.sample +++ b/config/galaxy.yml.sample @@ -1563,9 +1563,8 @@ galaxy: # failing jobs are just failed outright. #default_job_resubmission_condition: null - # In multiprocess configurations, notification between processes about - # new jobs must be done via the database. In single process - # configurations, this can be done in memory, which is a bit quicker. + # This option is deprecated, use the `mem-self` handler assignment + # option in the job configuration instead. #track_jobs_in_database: true # This enables splitting of jobs into tasks, if specified by the diff --git a/config/job_conf.xml.sample_advanced b/config/job_conf.xml.sample_advanced index 361d3693526..10787bbd7d1 100644 --- a/config/job_conf.xml.sample_advanced +++ b/config/job_conf.xml.sample_advanced @@ -332,24 +332,76 @@ used by Galaxy is no environment variable of the specified name is found. --> - - + - - + + + --> + - + handlers defined in job_conf.xml will be used (or the web process that + receives the workflow scheduling request if handlers are not configured + in job_conf.xml). + + The options here are the same as is documented for in + job_conf.xml.sample_advanced with two exceptions: + + - If a uWSGI farm named `workflow-schedulers` is present, it will be + preferred, followed by `job-handlers`. If any untagged handlers are + defined in this configuration they are eligible to schedule workflows + in addition to any matching mules. + - If uWSGI farms are present, the default assignment method is + `db-preassign` rather than `uwsgi-mule-message`, because `db-preassign` + is deterministic. All workflows scheduled in a single history will be + assigned to the same handler, ensuring they are scheduled serially + (preventing their outputs from being interleaved in the history). You + can override this by explicitly setting + `assign_with="uwsgi-mule-message"`. + --> diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index f96ee544854..a3c509888df 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -3277,10 +3277,8 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~ :Description: - In multiprocess configurations, notification between processes - about new jobs must be done via the database. In single process - configurations, this can be done in memory, which is a bit - quicker. + This option is deprecated, use the `mem-self` handler assignment + option in the job configuration instead. :Default: ``true`` :Type: bool diff --git a/doc/source/admin/jobs.md b/doc/source/admin/jobs.md index 13423a546ca..a925afbc2d9 100644 --- a/doc/source/admin/jobs.md +++ b/doc/source/admin/jobs.md @@ -40,7 +40,7 @@ workers The `` configuration elements defines which Galaxy server processes (when [running multiple server processes](scaling.html)) should be used for running jobs, and how to group those processes. -The handlers configuration may define a ``default`` attribute. This is the the handler(s) that should be used if no explicit handler is defined for a job and is required if >1 handlers defined. +The handlers configuration may define a ``default`` attribute. This is the the handler(s) that should be used if no explicit handler is defined for a job. If unset, any untagged handlers will be used by default. The collection contains `` elements. diff --git a/doc/source/admin/scaling.md b/doc/source/admin/scaling.md index 5af4d2ae955..32aa47100f4 100644 --- a/doc/source/admin/scaling.md +++ b/doc/source/admin/scaling.md @@ -94,8 +94,20 @@ Under this strategy, job handling is offloaded to dedicated non-web-serving proc directly by the master uWSGI process. As a benefit of using mule messaging, only job handlers that are alive will be selected to run jobs. -This is the recommended deployment strategy for Galaxy servers that run web servers and job handlers **on the same -host**. +This is the recommended deployment strategy. + +```eval_rst +.. important:: + + If using **Zerg Mode** or running more than one uWSGI *master* process, do not use **uWSGI + Mules**. Doing so can + can cause jobs to be executed by mutiple handlers when recovering unassigned jobs at Galaxy server startup. + + Multiple master processes is a rare configuration and is typically only used in the case of load balancing the web + application across multiple hosts. Note that multiple master proceses is not the same thing as the ``processess`` + uWSGI configuration option, which is perfectly safe to set when using job handler mules. + + For these scenarios, **uWSGI + Webless** is the recommended deployment strategy. +``` ### uWSGI for web serving and Webless Galaxy applications as job handlers @@ -110,8 +122,8 @@ Like mules, under this strategy, job handling is offloaded to dedicated non-web- are [managed by the administrator](#starting-and-stopping). Because the handler is randomly assigned by the web worker when the job is submitted via the UI/API, jobs may be assigned to dead handlers. -This is the recommended deployment strategy for Galaxy servers that run web servers and job handlers **on different -hosts**. +This is the recommended deployment strategy when **Zerg Mode** is used, and for Galaxy servers that run web servers and +job handlers **on different hosts**. ## Legacy Deployment Options diff --git a/lib/galaxy/jobs/manager.py b/lib/galaxy/jobs/manager.py index 325e7e26f12..891ff850543 100644 --- a/lib/galaxy/jobs/manager.py +++ b/lib/galaxy/jobs/manager.py @@ -26,15 +26,9 @@ class JobManager(object): self.job_handler = handler.JobHandler(app) else: self.job_handler = NoopHandler() - self.__check_jobs_at_startup() + self.__check_jobs_at_startup() def __check_jobs_at_startup(self): - """ - TODO: It should be documented that starting two Galaxy uWSGI master processes simultaneously would result in a race condition that *could* cause two handlers to pick up the same job. - - The recommended config for now will be webless handlers if running more than one uWSGI (master) process - """ - # FIXME: test if self.app.job_config.use_messaging: jobs_at_startup = self.app.model.context.query(Job).enable_eagerloads(False) \ .filter((Job.state == Job.states.NEW) & (Job.handler == null())).all() diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 854ad3f8170..626fc0b14c6 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -2437,14 +2437,12 @@ mapping: failing jobs are just failed outright. track_jobs_in_database: - # FIXME: note deprecated, superceded by job_handler_assignment_method type: bool default: true required: false desc: | - In multiprocess configurations, notification between processes about new jobs - must be done via the database. In single process configurations, this can be - done in memory, which is a bit quicker. + This option is deprecated, use the `mem-self` handler assignment option in the + job configuration instead. use_tasked_jobs: type: bool From 40f776afde460759b5d682c19ef698fefd2d4388 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 11 Dec 2018 12:34:07 -0500 Subject: [PATCH 067/260] Fix tests broken with changes to job_conf.xml.sample_advanced, fix the broken direct handler assignment method, document the new handler assignment methods for future method authors. Ensure that enqueue does its best to return a handler (if known). --- lib/galaxy/jobs/manager.py | 23 ++++++ lib/galaxy/util/handlers.py | 78 +++++++++++++++++--- test/unit/jobs/handler_template_job_conf.xml | 2 +- test/unit/jobs/test_job_configuration.py | 15 +++- 4 files changed, 103 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/jobs/manager.py b/lib/galaxy/jobs/manager.py index 891ff850543..931278ce786 100644 --- a/lib/galaxy/jobs/manager.py +++ b/lib/galaxy/jobs/manager.py @@ -50,6 +50,20 @@ class JobManager(object): return JobHandlerMessage(task='setup', job_id=job.id) def enqueue(self, job, tool=None): + """Queue a job for execution. + + Due to the nature of some handler assignment methods which are wholly DB-based, the enqueue method will flush + the job. Callers who create the job typically should not flush the job before handing it off to ``enqueue()``. + If a job handler cannot be assigned, :exception:`ToolExecutionError` is raised. + + :param job: Job to enqueue. + :type job: Instance of :class:`galaxy.model.Job`. + :param tool: Tool that the job will execute. + :type tool: Instance of :class:`galaxy.tools.Tool`. + + :raises ToolExecutionError: if a handler was unable to be assigned. + returns: str or None -- Handler ID, tag, or pool assigned to the job. + """ tool_id = None configured_handler = None if tool: @@ -67,6 +81,15 @@ class JobManager(object): raise ToolExecutionError(exc.args[0], job=exc.obj) def stop(self, job, message=None): + """Stop a job that is currently executing. + + This can be safely called on jobs that have already terminated. + + :param job: Job to stop. + :type job: Instance of :class:`galaxy.model.Job`. + :param message: Message (if any) to be set on the job and output dataset(s) to explain the reason for stopping. + :type message: str + """ self.job_handler.job_stop_queue.put(job.id, error_msg=message) def shutdown(self): diff --git a/lib/galaxy/util/handlers.py b/lib/galaxy/util/handlers.py index faa964e0005..63240889628 100644 --- a/lib/galaxy/util/handlers.py +++ b/lib/galaxy/util/handlers.py @@ -217,18 +217,38 @@ class ConfiguresHandlers(object): # If these get to be any more complex we should probably modularize them, or at least move to a separate class def _assign_handler_direct(self, obj, configured): + """Directly assign a handler if the object has been preconfigured to a known single static handler. + + :param obj: Same as :method:`ConfiguresHandlers.assign_handler()`. + :param configured: Same as :method:`ConfiguresHandlers.assign_handler()`. + + :returns: str -- A valid handler ID, or False if no handler was assigned. + """ if self.app.config.track_jobs_in_database and configured: try: - resolved = self.get_preassigned_handler(configured) + handlers = self.handlers[configured] except KeyError: - resolved = None - if resolved == configured: + handlers = None + if handlers == (configured,): obj.set_handler(configured) _timed_flush_obj(obj) - return True + return configured return False def _assign_mem_self_handler(self, obj, method, configured, queue_callback=None, **kwargs): + """Assign object to this handler using this process's in-memory queue. + + This method ignores all handler configuration. + + :param obj: Same as :method:`ConfiguresHandlers.assign_handler()`. + :param method: Same as :method:`ConfiguresHandlers._assign_db_preassign_handler()`. + :param configured: Ignored. + :param queue_callback: Callback to be executed when the job should be queued (i.e. a callback to the handler's + ``put()`` method). No arguments are passed. + :type queue_callback: callable + + :returns: str -- This process's server name (handler ID). + """ assert queue_callback is not None, \ "Cannot perform '%s' handler assignment: `queue_callback` is None" % HANDLER_ASSIGNMENT_METHODS.MEM_SELF if configured: @@ -240,6 +260,17 @@ class ConfiguresHandlers(object): return self.app.config.server_name def _assign_db_self_handler(self, obj, method, configured, **kwargs): + """Assign object to this process by setting its ``handler`` column in the database to this process. + + This only occurs if there is not an explicitly configured handler assignment for the object. Otherwise, it is + passed to the DB_PREASSIGN method for assignment. + + :param obj: Same as :method:`ConfiguresHandlers.assign_handler()`. + :param method: Same as :method:`ConfiguresHandlers._assign_db_preassign_handler()`. + :param configured: Same as :method:`ConfiguresHandlers.assign_handler()`. + + :returns: str -- The assigned handler ID. + """ if configured: return self.handler_assignment_method_methods[HANDLER_ASSIGNMENT_METHODS.DB_PREASSIGN]( obj, method, configured, **kwargs @@ -249,12 +280,20 @@ class ConfiguresHandlers(object): return self.app.config.server_name def _assign_db_preassign_handler(self, obj, method, configured, index=None, **kwargs): - """Given a handler ID or tag, return a handler matching it, of those handlers that are statically configured in + """Assign object to a handler by setting its ``handler`` column in the database to a handler selected at random + from the known handlers in the appropriate tag. + + Given a handler ID or tag, return a handler matching it, of those handlers that are statically configured in the job configuration, or known via preconfigured pools. - :param index: Generate "consistent" "random" handlers with this index if specified. - :type index: int + :param obj: Same as :method:`ConfiguresHandlers.assign_handler()`. + :param method: Assignment method currently being checked. + :type method: Value in :data:`HANDLER_ASSIGNMENT_METHODS`. + :param configured: Same as :method:`ConfiguresHandlers.assign_handler()`. + :param index: Generate "consistent" "random" handlers with this index if specified. + :type index: int + :raises KeyError: if the configured or default handler is not a known handler ID or tag. :returns: str -- A valid job handler ID. """ handler = configured @@ -273,6 +312,19 @@ class ConfiguresHandlers(object): return handler_id def _assign_uwsgi_mule_message_handler(self, obj, method, configured, message_callback=None, **kwargs): + """Assign object to a handler by sending a setup message to the appropriate handler pool (farm), where a handler + (mule) will receive the message and assign itself. + + :param obj: Same as :method:`ConfiguresHandlers.assign_handler()`. + :param method: Same as :method:`ConfiguresHandlers._assign_db_preassign_handler()`. + :param configured: Same as :method:`ConfiguresHandlers.assign_handler()`. + :param queue_callback: Callback returning a setup message to be sent via the stack messaging interface's + ``send_message()`` method. No arguments are passed. + :type queue_callback: callable + + :raises HandlerAssignmentSkip: if the configured or default handler is not a known handler pool (farm) + :returns: str -- The assigned handler pool. + """ assert message_callback is not None, \ "Cannot perform '%s' handler assignment: `message_callback` is None" \ % HANDLER_ASSIGNMENT_METHODS.UWSGI_MULE_MESSAGE @@ -290,8 +342,14 @@ class ConfiguresHandlers(object): def assign_handler(self, obj, configured=None, **kwargs): """Set a job handler, flush obj - :param obj: Object to assign to (must be a model object with ``handler`` attribute and ``log_str`` callable) - :param configured: A handler ID or tag/pool. + Called assignment methods should raise :exception:`HandlerAssignmentSkip` to indicate that the next method + should be tried. + + :param obj: Object to assign a handler to (must be a model object with ``handler`` attribute and + ``log_str`` callable). + :type obj: instance of :class:`galaxy.model.Job` or other model object with a ``set_handler()`` method. + :param configured: Preconfigured handler (ID, tag, or None) for the given object. + :type configured: str or None. :returns: bool -- True on successful assignment, False otherwise. """ @@ -308,7 +366,7 @@ class ConfiguresHandlers(object): handler = self._handler_assignment_method_methods[method]( obj, method, configured=configured, **kwargs) log.info("(%s) Handler '%s' assigned using '%s' assignment method", obj.log_str(), handler, method) - break + return handler except HandlerAssignmentSkip: log.debug("(%s) Handler assignment method '%s' did not assign a handler, trying next method", obj.log_str(), method) diff --git a/test/unit/jobs/handler_template_job_conf.xml b/test/unit/jobs/handler_template_job_conf.xml index fe95cc785d3..9890334bcef 100644 --- a/test/unit/jobs/handler_template_job_conf.xml +++ b/test/unit/jobs/handler_template_job_conf.xml @@ -3,7 +3,7 @@ - + {handlers} diff --git a/test/unit/jobs/test_job_configuration.py b/test/unit/jobs/test_job_configuration.py index 034fa75e471..632000e6db4 100644 --- a/test/unit/jobs/test_job_configuration.py +++ b/test/unit/jobs/test_job_configuration.py @@ -58,11 +58,17 @@ class JobConfXmlParserTestCase(unittest.TestCase): assert task_runners[0]["workers"] == 5 def test_explicit_handler_default(self): - self.__with_advanced_config() + self.__with_handlers_config( + handlers=[{'id': 'handler0', 'tags': 'handlers'}, {'id': 'handler1', 'tags': 'handlers'}], + default='handlers' + ) assert self.job_config.default_handler_id == "handlers" def test_handler_tag_parsing(self): - self.__with_advanced_config() + self.__with_handlers_config( + handlers=[{'id': 'handler0', 'tags': 'handlers'}, {'id': 'handler1', 'tags': 'handlers'}], + default='handlers' + ) assert "handler0" in self.job_config.handlers["handlers"] assert "handler1" in self.job_config.handlers["handlers"] @@ -264,12 +270,13 @@ class JobConfXmlParserTestCase(unittest.TestCase): def __with_advanced_config(self): self.__write_config_from(ADVANCED_JOB_CONF) - def __with_handlers_config(self, assign_with=None, handlers=None, base_pools=None): + def __with_handlers_config(self, assign_with=None, default=None, handlers=None, base_pools=None): handlers = handlers or [] template = { 'assign_with': ' assign_with="%s"' % assign_with if assign_with is not None else '', + 'default': ' default="%s"' % default if default is not None else '', 'handlers': '\n'.join([ - ''.format( + ''.format( id=x['id'], tags=' tags="%s"' % x['tags'] if 'tags' in x else '' ) for x in handlers]), From 4a006335ac4ac84a69b5cb1f851e3e63400443ae Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 11 Dec 2018 13:44:50 -0500 Subject: [PATCH 068/260] Fix default job config (no job_conf.xml). --- lib/galaxy/jobs/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 77a8aaa2cbc..74d0af6ec17 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -343,7 +343,6 @@ class JobConfiguration(ConfiguresHandlers): self.runner_plugins.append(dict(id='tasks', load='tasks', workers=DEFAULT_LOCAL_WORKERS)) # Set the handlers self._init_handler_assignment_methods() - self._init_handlers() if not self.handler_assignment_methods_configured: self._set_default_handler_assignment_methods() else: From ece11eaabed2cbc07964e503e6f83179c05a45fd Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 5 Dec 2018 18:32:30 -0500 Subject: [PATCH 069/260] De-couple objectstore abstraction from XML. - Allow loading objectstores for Python dicts and YAML configurations. - New tests for XML parsing of azure, cloud, pithos, and S3 object stores. - New tests for YAML parsing of all these object stores plus the stock disk and nested ones. --- lib/galaxy/objectstore/__init__.py | 284 ++++++++++------ lib/galaxy/objectstore/azure_blob.py | 98 ++++-- lib/galaxy/objectstore/cloud.py | 74 ++--- lib/galaxy/objectstore/pithos.py | 23 +- lib/galaxy/objectstore/s3.py | 162 ++++++--- test/unit/test_objectstore.py | 473 ++++++++++++++++++++++----- 6 files changed, 823 insertions(+), 291 deletions(-) diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 5a6c06cd18b..9617b2d221b 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -13,6 +13,7 @@ import threading import time from xml.etree import ElementTree +import yaml try: from sqlalchemy.orm import object_session except ImportError: @@ -205,9 +206,22 @@ class ObjectStore(object): """Return the percentage indicating how full the store is.""" raise NotImplementedError() + @classmethod + def parse_xml(clazz, config_xml): + """Parse an XML description of a configuration for this object store. + + Return a configuration dictionary (such as would correspond to the YAML configuration) + for the object store. + """ + raise NotImplementedError() + + @classmethod + def from_xml(clazz, config, config_xml, **kwd): + config_dict = clazz.parse_xml(config_xml) + return clazz(config, config_dict, **kwd) + class DiskObjectStore(ObjectStore): - """ Standard Galaxy object store. @@ -217,14 +231,14 @@ class DiskObjectStore(ObjectStore): >>> import tempfile >>> file_path=tempfile.mkdtemp() >>> obj = Bunch(id=1) - >>> s = DiskObjectStore(Bunch(umask=0o077, jobs_directory=file_path, new_file_path=file_path, object_store_check_old_style=False), file_path=file_path) + >>> s = DiskObjectStore(Bunch(umask=0o077, jobs_directory=file_path, new_file_path=file_path, object_store_check_old_style=False), dict(files_dir=file_path)) >>> s.create(obj) >>> s.exists(obj) True >>> assert s.get_filename(obj) == file_path + '/000/dataset_1.dat' """ - def __init__(self, config, config_xml=None, file_path=None, extra_dirs=None): + def __init__(self, config, config_dict): """ :type config: object :param config: An object, most likely populated from @@ -234,8 +248,6 @@ class DiskObjectStore(ObjectStore): * file_path -- Default directory to store objects to disk in. * umask -- the permission bits for newly created files. - :type config_xml: ElementTree - :type file_path: str :param file_path: Override for the `config.file_path` value. @@ -243,16 +255,28 @@ class DiskObjectStore(ObjectStore): :param extra_dirs: Keys are string, values are directory paths. """ super(DiskObjectStore, self).__init__(config) - self.file_path = file_path or config.file_path - # The new config_xml overrides universe settings. + self.file_path = config_dict.get("files_dir") or config.file_path + extra_dirs = dict( + (e['type'], e['path']) for e in config_dict.get('extra_dirs', [])) + self.extra_dirs.update(extra_dirs) + + @classmethod + def parse_xml(clazz, config_xml): + extra_dirs = [] + file_path = None + if config_xml is not None: for e in config_xml: if e.tag == 'files_dir': - self.file_path = e.get('path') + file_path = e.get('path') else: - self.extra_dirs[e.get('type')] = e.get('path') - if extra_dirs is not None: - self.extra_dirs.update(extra_dirs) + extra_dirs.append({"type": e.get('type'), "path": e.get('path')}) + + config_dict = { + "files_dir": file_path, + "extra_dirs": extra_dirs, + } + return config_dict def _get_filename(self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): """ @@ -546,7 +570,7 @@ class DistributedObjectStore(NestedObjectStore): with weighting. """ - def __init__(self, config, config_xml=None, fsmon=False): + def __init__(self, config, config_dict, fsmon=False): """ :type config: object :param config: An object, most likely populated from @@ -561,21 +585,34 @@ class DistributedObjectStore(NestedObjectStore): :param fsmon: If True, monitor the file system for free space, removing backends when they get too full. """ - super(DistributedObjectStore, self).__init__(config, - config_xml=config_xml) - if config_xml is None: - self.distributed_config = config.distributed_object_store_config_file - assert self.distributed_config is not None, \ - "distributed object store ('object_store = distributed') " \ - "requires a config file, please set one in " \ - "'distributed_object_store_config_file')" + super(DistributedObjectStore, self).__init__(config) + self.backends = {} self.weighted_backend_ids = [] self.original_weighted_backend_ids = [] self.max_percent_full = {} - self.global_max_percent_full = 0.0 + self.global_max_percent_full = config_dict.get("global_max_percent_full", 0) random.seed() - self.__parse_distributed_config(config, config_xml) + + backends_def = config_dict["backends"] + for backend_def in backends_def: + backened_id = backend_def["id"] + file_path = backend_def["files_dir"] + extra_dirs = backend_def.get("extra_dirs", []) + maxpctfull = backend_def.get("max_percent_full", 0) + weight = backend_def["weight"] + disk_config_dict = dict(files_dir=file_path, extra_dirs=extra_dirs) + self.backends[backened_id] = DiskObjectStore(config, disk_config_dict) + self.max_percent_full[backened_id] = maxpctfull + log.debug("Loaded disk backend '%s' with weight %s and file_path: %s" % (backened_id, weight, file_path)) + + for i in range(0, weight): + # The simplest way to do weighting: add backend ids to a + # sequence the number of times equalling weight, then randomly + # choose a backend from that sequence at creation + self.weighted_backend_ids.append(backened_id) + self.original_weighted_backend_ids = self.weighted_backend_ids + self.sleeper = None if fsmon and (self.global_max_percent_full or [_ for _ in self.max_percent_full.values() if _ != 0.0]): self.sleeper = Sleeper() @@ -584,40 +621,65 @@ class DistributedObjectStore(NestedObjectStore): self.filesystem_monitor_thread.start() log.info("Filesystem space monitor started") - def __parse_distributed_config(self, config, config_xml=None): - if config_xml is None: - root = ElementTree.parse(self.distributed_config).getroot() - log.debug('Loading backends for distributed object store from %s', self.distributed_config) + @classmethod + def parse_xml(clazz, config_xml, legacy=False): + if legacy: + backends_root = config_xml else: - root = config_xml.find('backends') - log.debug('Loading backends for distributed object store from %s', config_xml.get('id')) - self.global_max_percent_full = float(root.get('maxpctfull', 0)) - for elem in [e for e in root if e.tag == 'backend']: + backends_root = config_xml.find('backends') + + backends = [] + config_dict = { + 'global_max_percent_full': float(backends_root.get('maxpctfull', 0)), + 'backends': backends, + } + + for elem in [e for e in backends_root if e.tag == 'backend']: id = elem.get('id') weight = int(elem.get('weight', 1)) maxpctfull = float(elem.get('maxpctfull', 0)) - if elem.get('type', 'disk'): + elem_type = elem.get('type', 'disk') + + if elem_type: path = None - extra_dirs = {} + extra_dirs = [] for sub in elem: if sub.tag == 'files_dir': path = sub.get('path') elif sub.tag == 'extra_dir': type = sub.get('type') - extra_dirs[type] = sub.get('path') - self.backends[id] = DiskObjectStore(config, file_path=path, extra_dirs=extra_dirs) - self.max_percent_full[id] = maxpctfull - log.debug("Loaded disk backend '%s' with weight %s and file_path: %s" % (id, weight, path)) - if extra_dirs: - log.debug(" Extra directories:") - for type, dir in extra_dirs.items(): - log.debug(" %s: %s" % (type, dir)) - for i in range(0, weight): - # The simplest way to do weighting: add backend ids to a - # sequence the number of times equalling weight, then randomly - # choose a backend from that sequence at creation - self.weighted_backend_ids.append(id) - self.original_weighted_backend_ids = self.weighted_backend_ids + extra_dirs.append({"type": type, "path": sub.get('path')}) + + backend_dict = { + 'id': id, + 'weight': weight, + 'max_percent_full': maxpctfull, + 'files_dir': path, + 'extra_dirs': extra_dirs, + 'type': elem_type, + } + backends.append(backend_dict) + + return config_dict + + @classmethod + def from_xml(clazz, config, config_xml, fsmon=False): + legacy = False + if config_xml is None: + distributed_config = config.distributed_object_store_config_file + assert distributed_config is not None, \ + "distributed object store ('object_store = distributed') " \ + "requires a config file, please set one in " \ + "'distributed_object_store_config_file')" + + log.debug('Loading backends for distributed object store from %s', distributed_config) + config_xml = ElementTree.parse(distributed_config).getroot() + legacy = True + else: + log.debug('Loading backends for distributed object store from %s', config_xml.get('id')) + + config_dict = clazz.parse_xml(config_xml, legacy=legacy) + return clazz(config, config_dict, fsmon=fsmon) def shutdown(self): """Shut down. Kill the free space monitor if there is one.""" @@ -693,12 +755,27 @@ class HierarchicalObjectStore(NestedObjectStore): When creating objects only the first store is used. """ - def __init__(self, config, config_xml=None, fsmon=False): + def __init__(self, config, config_dict, fsmon=False): """The default contructor. Extends `NestedObjectStore`.""" - super(HierarchicalObjectStore, self).__init__(config, config_xml=config_xml) - self.backends = odict() + super(HierarchicalObjectStore, self).__init__(config) + + backends = odict() + for order, backend_def in enumerate(config_dict["backends"]): + backends[order] = build_object_store_from_config(config, config_dict=backend_def, fsmon=fsmon) + + self.backends = backends + + @classmethod + def parse_xml(clazz, config_xml): + backends_list = [] for b in sorted(config_xml.find('backends'), key=lambda b: int(b.get('order'))): - self.backends[int(b.get('order'))] = build_object_store_from_config(config, fsmon=fsmon, config_xml=b) + store_type = b.get("type") + objectstore_class, _ = type_to_object_store_class(store_type) + backend_config_dict = objectstore_class.parse_xml(b) + backend_config_dict["type"] = store_type + backends_list.append(backend_config_dict) + + return {"backends": backends_list} def exists(self, obj, **kwargs): """Check all child object stores.""" @@ -712,7 +789,44 @@ class HierarchicalObjectStore(NestedObjectStore): self.backends[0].create(obj, **kwargs) -def build_object_store_from_config(config, fsmon=False, config_xml=None): +def type_to_object_store_class(store, fsmon=False): + objectstore_class = None + objectstore_constructor_kwds = {} + if store == 'disk': + objectstore_class = DiskObjectStore + elif store == 's3': + from .s3 import S3ObjectStore + objectstore_class = S3ObjectStore + elif store == 'cloud': + from .cloud import Cloud + objectstore_class = Cloud + elif store == 'swift': + from .s3 import SwiftObjectStore + objectstore_class = SwiftObjectStore + elif store == 'distributed': + objectstore_class = DistributedObjectStore + objectstore_constructor_kwds["fsmon"] = fsmon + elif store == 'hierarchical': + objectstore_class = HierarchicalObjectStore + objectstore_constructor_kwds["fsmon"] = fsmon + elif store == 'irods': + from .rods import IRODSObjectStore + objectstore_class = IRODSObjectStore + elif store == 'azure_blob': + from .azure_blob import AzureBlobObjectStore + objectstore_class = AzureBlobObjectStore + elif store == 'pithos': + from .pithos import PithosObjectStore + objectstore_class = PithosObjectStore + # Disable the Pulsar object store for now until it receives some attention + # elif store == 'pulsar': + # from .pulsar import PulsarObjectStore + # return PulsarObjectStore(config=config, config_xml=config_xml) + + return objectstore_class, objectstore_constructor_kwds + + +def build_object_store_from_config(config, fsmon=False, config_xml=None, config_dict=None): """ Invoke the appropriate object store. @@ -722,52 +836,40 @@ def build_object_store_from_config(config, fsmon=False, config_xml=None): Or you can specify the object store type in the `object_store` attribute of the `config` object. Currently 'disk', 's3', 'swift', 'distributed', 'hierarchical', 'irods', and 'pulsar' are supported values. - """ - if config_xml is None and os.path.exists(config.object_store_config_file): - # This is a top level invocation of build_object_store_from_config, and - # we have an object_store_conf.xml -- read the .xml and build - # accordingly - root = ElementTree.parse(config.object_store_config_file).getroot() - store = root.get('type') - config_xml = root + from_object = 'xml' + + if config_xml is None and config_dict is None: + config_file = config.object_store_config_file + if os.path.exists(config_file): + if config_file.endswith(".xml") or config_file.endswith(".xml.sample"): + # This is a top level invocation of build_object_store_from_config, and + # we have an object_store_conf.xml -- read the .xml and build + # accordingly + config_xml = ElementTree.parse(config.object_store_config_file).getroot() + store = config_xml.get('type') + else: + with open(config_file, "rt") as f: + config_dict = yaml.safe_load(f) + from_object = 'dict' + store = config_dict.get('type') + else: + store = config.object_store elif config_xml is not None: store = config_xml.get('type') - else: - store = config.object_store + elif config_dict is not None: + from_object = 'dict' + store = config_dict.get('type') - if store == 'disk': - return DiskObjectStore(config=config, config_xml=config_xml) - elif store == 's3': - from .s3 import S3ObjectStore - return S3ObjectStore(config=config, config_xml=config_xml) - elif store == 'cloud': - from .cloud import Cloud - return Cloud(config=config, config_xml=config_xml) - elif store == 'swift': - from .s3 import SwiftObjectStore - return SwiftObjectStore(config=config, config_xml=config_xml) - elif store == 'distributed': - return DistributedObjectStore( - config=config, fsmon=fsmon, config_xml=config_xml) - elif store == 'hierarchical': - return HierarchicalObjectStore(config=config, config_xml=config_xml) - elif store == 'irods': - from .rods import IRODSObjectStore - return IRODSObjectStore(config=config, config_xml=config_xml) - elif store == 'azure_blob': - from .azure_blob import AzureBlobObjectStore - return AzureBlobObjectStore(config=config, config_xml=config_xml) - elif store == 'pithos': - from .pithos import PithosObjectStore - return PithosObjectStore(config=config, config_xml=config_xml) - # Disable the Pulsar object store for now until it receives some attention - # elif store == 'pulsar': - # from .pulsar import PulsarObjectStore - # return PulsarObjectStore(config=config, config_xml=config_xml) - else: + objectstore_class, objectstore_constructor_kwds = type_to_object_store_class(store, fsmon=fsmon) + if objectstore_class is None: log.error("Unrecognized object store definition: {0}".format(store)) + if from_object == 'xml': + return objectstore_class.from_xml(config=config, config_xml=config_xml, **objectstore_constructor_kwds) + else: + return objectstore_class(config=config, config_dict=config_dict, **objectstore_constructor_kwds) + def local_extra_dirs(func): """Non-local plugin decorator using local directories for the extra_dirs (job_work and temp).""" diff --git a/lib/galaxy/objectstore/azure_blob.py b/lib/galaxy/objectstore/azure_blob.py index a2cc6462c61..2ac92aa71c2 100644 --- a/lib/galaxy/objectstore/azure_blob.py +++ b/lib/galaxy/objectstore/azure_blob.py @@ -38,6 +38,49 @@ NO_BLOBSERVICE_ERROR_MESSAGE = ("ObjectStore configured, but no azure.storage.bl log = logging.getLogger(__name__) +def parse_config_xml(config_xml): + try: + auth_xml = config_xml.findall('auth')[0] + + account_name = auth_xml.get('account_name') + account_key = auth_xml.get('account_key') + + container_xml = config_xml.find('container') + container_name = container_xml.get('name') + max_chunk_size = int(container_xml.get('max_chunk_size', 250)) # currently unused + + c_xml = config_xml.findall('cache')[0] + cache_size = float(c_xml.get('size', -1)) + staging_path = c_xml.get('path', None) + + tag, attrs = 'extra_dir', ('type', 'path') + extra_dirs = config_xml.findall(tag) + if not extra_dirs: + msg = 'No {tag} element in XML tree'.format(tag=tag) + log.error(msg) + raise Exception(msg) + extra_dirs = [dict(((k, e.get(k)) for k in attrs)) for e in extra_dirs] + return { + 'auth': { + 'account_name': account_name, + 'account_key': account_key, + }, + 'container': { + 'name': container_name, + 'max_chunk_size': max_chunk_size, + }, + 'cache': { + 'size': cache_size, + 'path': staging_path, + }, + 'extra_dirs': extra_dirs, + } + except Exception: + # Toss it back up after logging, we can't continue loading at this point. + log.exception("Malformed ObjectStore Configuration XML -- unable to continue") + raise + + class AzureBlobObjectStore(ObjectStore): """ Object store that stores objects as blobs in an Azure Blob Container. A local @@ -45,14 +88,34 @@ class AzureBlobObjectStore(ObjectStore): Galaxy and Azure. """ - def __init__(self, config, config_xml): - if BlockBlobService is None: - raise Exception(NO_BLOBSERVICE_ERROR_MESSAGE) + def __init__(self, config, config_dict): super(AzureBlobObjectStore, self).__init__(config) - self.staging_path = self.config.file_path self.transfer_progress = 0 - self._parse_config_xml(config_xml) + + auth_dict = config_dict["auth"] + container_dict = config_dict["container"] + cache_dict = config_dict["cache"] + + self.account_name = auth_dict.get('account_name') + self.account_key = auth_dict.get('account_key') + + self.container_name = container_dict.get('name') + self.max_chunk_size = container_dict.get('max_chunk_size', 250) # currently unused + + self.cache_size = cache_dict.get('size', -1) + self.staging_path = cache_dict.get('path') or self.config.object_store_cache_path + + extra_dirs = dict( + (e['type'], e['path']) for e in config_dict.get('extra_dirs', [])) + self.extra_dirs.update(extra_dirs) + + self._initialize() + + def _initialize(self): + if BlockBlobService is None: + raise Exception(NO_BLOBSERVICE_ERROR_MESSAGE) + self._configure_connection() # Clean cache only if value is set in galaxy.ini @@ -70,28 +133,9 @@ class AzureBlobObjectStore(ObjectStore): ################### # config_xml is an ElementTree object. - def _parse_config_xml(self, config_xml): - try: - auth_xml = config_xml.find('auth') - self.account_name = auth_xml.get('account_name') - self.account_key = auth_xml.get('account_key') - container_xml = config_xml.find('container') - self.container_name = container_xml.get('name') - self.max_chunk_size = int(container_xml.get('max_chunk_size', 250)) # currently unused - cache_xml = config_xml.find('cache') - self.cache_size = float(cache_xml.get('size', -1)) - self.staging_path = cache_xml.get('path', self.config.object_store_cache_path) - - for d_xml in config_xml.findall('extra_dir'): - self.extra_dirs[d_xml.get('type')] = d_xml.get('path') - - log.debug("Object cache dir: %s", self.staging_path) - log.debug(" job work dir: %s", self.extra_dirs['job_work']) - - except Exception: - # Toss it back up after logging, we can't continue loading at this point. - log.exception("Malformed ObjectStore Configuration XML -- unable to continue") - raise + @classmethod + def parse_xml(clazz, config_xml): + return parse_config_xml(config_xml) def _configure_connection(self): log.debug("Configuring Connection") diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index e185681c4a8..adb8708489c 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -15,12 +15,11 @@ from galaxy.exceptions import ObjectInvalid, ObjectNotFound from galaxy.util import ( directory_hash_id, safe_relpath, - string_as_bool, umask_fix_perms, ) from galaxy.util.sleeper import Sleeper +from .s3 import parse_config_xml from ..objectstore import convert_bytes, ObjectStore - try: from cloudbridge.cloud.factory import CloudProviderFactory, ProviderList from cloudbridge.cloud.interfaces.exceptions import InvalidNameException @@ -42,13 +41,41 @@ class Cloud(ObjectStore): cache exists that is used as an intermediate location for files between Galaxy and the cloud storage. """ - def __init__(self, config, config_xml): + def __init__(self, config, config_dict): super(Cloud, self).__init__(config) + self.transfer_progress = 0 + + auth_dict = config_dict['auth'] + bucket_dict = config_dict['bucket'] + connection_dict = config_dict.get('connection', {}) + cache_dict = config_dict['cache'] + + self.access_key = auth_dict.get('access_key') + self.secret_key = auth_dict.get('secret_key') + + self.bucket = bucket_dict.get('name') + self.use_rr = bucket_dict.get('use_reduced_redundancy', False) + self.max_chunk_size = bucket_dict.get('max_chunk_size', 250) + + self.host = connection_dict.get('host', None) + self.port = connection_dict.get('port', 6000) + self.multipart = connection_dict.get('multipart', True) + self.is_secure = connection_dict.get('is_secure', True) + self.conn_path = connection_dict.get('conn_path', '/') + + self.cache_size = cache_dict.get('size', -1) + self.staging_path = cache_dict.get('path') or self.config.object_store_cache_path + + extra_dirs = dict( + (e['type'], e['path']) for e in config_dict.get('extra_dirs', [])) + self.extra_dirs.update(extra_dirs) + + self._initialize() + + def _initialize(self): if CloudProviderFactory is None: raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE) - self.staging_path = self.config.file_path - self.transfer_progress = 0 - self._parse_config_xml(config_xml) + self._configure_connection() self.bucket = self._get_bucket(self.bucket) # Clean cache only if value is set in galaxy.ini @@ -73,38 +100,9 @@ class Cloud(ObjectStore): 'aws_secret_key': self.secret_key} self.conn = CloudProviderFactory().create_provider(ProviderList.AWS, aws_config) - def _parse_config_xml(self, config_xml): - try: - a_xml = config_xml.findall('auth')[0] - self.access_key = a_xml.get('access_key') - self.secret_key = a_xml.get('secret_key') - b_xml = config_xml.findall('bucket')[0] - self.bucket = b_xml.get('name') - self.max_chunk_size = int(b_xml.get('max_chunk_size', 250)) - cn_xml = config_xml.findall('connection') - if not cn_xml: - cn_xml = {} - else: - cn_xml = cn_xml[0] - self.host = cn_xml.get('host', None) - self.port = int(cn_xml.get('port', 6000)) - self.multipart = string_as_bool(cn_xml.get('multipart', 'True')) - self.is_secure = string_as_bool(cn_xml.get('is_secure', 'True')) - self.conn_path = cn_xml.get('conn_path', '/') - c_xml = config_xml.findall('cache')[0] - self.cache_size = float(c_xml.get('size', -1)) - self.staging_path = c_xml.get('path', self.config.object_store_cache_path) - - for d_xml in config_xml.findall('extra_dir'): - self.extra_dirs[d_xml.get('type')] = d_xml.get('path') - - log.debug("Object cache dir: %s", self.staging_path) - log.debug(" job work dir: %s", self.extra_dirs['job_work']) - - except Exception: - # Toss it back up after logging, we can't continue loading at this point. - log.exception("Malformed ObjectStore Configuration XML -- unable to continue") - raise + @classmethod + def parse_xml(clazz, config_xml): + return parse_config_xml(config_xml) def __cache_monitor(self): time.sleep(2) # Wait for things to load before starting the monitor diff --git a/lib/galaxy/objectstore/pithos.py b/lib/galaxy/objectstore/pithos.py index 8398ff64ac8..f1cf906cbc5 100644 --- a/lib/galaxy/objectstore/pithos.py +++ b/lib/galaxy/objectstore/pithos.py @@ -85,24 +85,31 @@ class PithosObjectStore(ObjectStore): Cache is ignored for the time being. """ - def __init__(self, config, config_xml): - if KamakiClient is None: - raise Exception(NO_KAMAKI_ERROR_MESSAGE) + def __init__(self, config, config_dict): super(PithosObjectStore, self).__init__(config) self.staging_path = self.config.file_path - self.transfer_progress = 0 log.info('Parse config_xml for pithos object store') - self.config_dict = parse_config_xml(config_xml) + self.config_dict = config_dict log.debug(self.config_dict) + log.info('Define extra_dirs') + extra_dirs = dict( + (e['type'], e['path']) for e in config_dict.get('extra_dirs', [])) + self.extra_dirs.update(extra_dirs) + self._initialize() + + def _initialize(self): + if KamakiClient is None: + raise Exception(NO_KAMAKI_ERROR_MESSAGE) + log.info('Authenticate Synnefo account') self._authenticate() log.info('Initialize Pithos+ client') self._init_pithos() - log.info('Define extra_dirs') - self.extra_dirs = dict( - (e['type'], e['path']) for e in self.config_dict['extra_dirs']) + @classmethod + def parse_xml(clazz, config_xml): + return parse_config_xml(config_xml) def _authenticate(self): auth = self.config_dict['auth'] diff --git a/lib/galaxy/objectstore/s3.py b/lib/galaxy/objectstore/s3.py index e56b7d6e084..92c638253b4 100644 --- a/lib/galaxy/objectstore/s3.py +++ b/lib/galaxy/objectstore/s3.py @@ -38,6 +38,71 @@ log = logging.getLogger(__name__) logging.getLogger('boto').setLevel(logging.INFO) # Otherwise boto is quite noisy +def parse_config_xml(config_xml): + try: + a_xml = config_xml.findall('auth')[0] + access_key = a_xml.get('access_key') + secret_key = a_xml.get('secret_key') + + b_xml = config_xml.findall('bucket')[0] + bucket_name = b_xml.get('name') + use_rr = string_as_bool(b_xml.get('use_reduced_redundancy', "False")) + max_chunk_size = int(b_xml.get('max_chunk_size', 250)) + + cn_xml = config_xml.findall('connection') + if not cn_xml: + cn_xml = {} + else: + cn_xml = cn_xml[0] + + host = cn_xml.get('host', None) + port = int(cn_xml.get('port', 6000)) + multipart = string_as_bool(cn_xml.get('multipart', 'True')) + is_secure = string_as_bool(cn_xml.get('is_secure', 'True')) + conn_path = cn_xml.get('conn_path', '/') + + c_xml = config_xml.findall('cache')[0] + cache_size = float(c_xml.get('size', -1)) + + staging_path = c_xml.get('path', None) + + tag, attrs = 'extra_dir', ('type', 'path') + extra_dirs = config_xml.findall(tag) + if not extra_dirs: + msg = 'No {tag} element in XML tree'.format(tag=tag) + log.error(msg) + raise Exception(msg) + extra_dirs = [dict(((k, e.get(k)) for k in attrs)) for e in extra_dirs] + + return { + 'auth': { + 'access_key': access_key, + 'secret_key': secret_key, + }, + 'bucket': { + 'name': bucket_name, + 'use_reduced_redundancy': use_rr, + 'max_chunk_size': max_chunk_size, + }, + 'connection': { + 'host': host, + 'port': port, + 'multipart': multipart, + 'is_secure': is_secure, + 'conn_path': conn_path, + }, + 'cache': { + 'size': cache_size, + 'path': staging_path, + }, + 'extra_dirs': extra_dirs, + } + except Exception: + # Toss it back up after logging, we can't continue loading at this point. + log.exception("Malformed ObjectStore Configuration XML -- unable to continue") + raise + + class S3ObjectStore(ObjectStore): """ Object store that stores objects as items in an AWS S3 bucket. A local @@ -45,13 +110,55 @@ class S3ObjectStore(ObjectStore): Galaxy and S3. """ - def __init__(self, config, config_xml): + def __init__(self, config, config_dict): + super(S3ObjectStore, self).__init__(config) + + self.transfer_progress = 0 + + auth_dict = config_dict['auth'] + bucket_dict = config_dict['bucket'] + connection_dict = config_dict.get('connection', {}) + cache_dict = config_dict['cache'] + + self.access_key = auth_dict.get('access_key') + self.secret_key = auth_dict.get('secret_key') + + self.bucket = bucket_dict.get('name') + self.use_rr = bucket_dict.get('use_reduced_redundancy', False) + self.max_chunk_size = bucket_dict.get('max_chunk_size', 250) + + self.host = connection_dict.get('host', None) + self.port = connection_dict.get('port', 6000) + self.multipart = connection_dict.get('multipart', True) + self.is_secure = connection_dict.get('is_secure', True) + self.conn_path = connection_dict.get('conn_path', '/') + + self.cache_size = cache_dict.get('size', -1) + self.staging_path = cache_dict.get('path') or self.config.object_store_cache_path + + extra_dirs = dict( + (e['type'], e['path']) for e in config_dict.get('extra_dirs', [])) + self.extra_dirs.update(extra_dirs) + + log.debug("Object cache dir: %s", self.staging_path) + log.debug(" job work dir: %s", self.extra_dirs['job_work']) + + self._initialize() + + def _initialize(self): if boto is None: raise Exception(NO_BOTO_ERROR_MESSAGE) - super(S3ObjectStore, self).__init__(config) - self.staging_path = self.config.file_path - self.transfer_progress = 0 - self._parse_config_xml(config_xml) + + # for multipart upload + self.s3server = {'access_key': self.access_key, + 'secret_key': self.secret_key, + 'is_secure': self.is_secure, + 'max_chunk_size': self.max_chunk_size, + 'host': self.host, + 'port': self.port, + 'use_rr': self.use_rr, + 'conn_path': self.conn_path} + self._configure_connection() self.bucket = self._get_bucket(self.bucket) # Clean cache only if value is set in galaxy.ini @@ -73,48 +180,9 @@ class S3ObjectStore(ObjectStore): log.debug("Configuring S3 Connection") self.conn = S3Connection(self.access_key, self.secret_key) - def _parse_config_xml(self, config_xml): - try: - a_xml = config_xml.findall('auth')[0] - self.access_key = a_xml.get('access_key') - self.secret_key = a_xml.get('secret_key') - b_xml = config_xml.findall('bucket')[0] - self.bucket = b_xml.get('name') - self.use_rr = string_as_bool(b_xml.get('use_reduced_redundancy', "False")) - self.max_chunk_size = int(b_xml.get('max_chunk_size', 250)) - cn_xml = config_xml.findall('connection') - if not cn_xml: - cn_xml = {} - else: - cn_xml = cn_xml[0] - self.host = cn_xml.get('host', None) - self.port = int(cn_xml.get('port', 6000)) - self.multipart = string_as_bool(cn_xml.get('multipart', 'True')) - self.is_secure = string_as_bool(cn_xml.get('is_secure', 'True')) - self.conn_path = cn_xml.get('conn_path', '/') - c_xml = config_xml.findall('cache')[0] - self.cache_size = float(c_xml.get('size', -1)) - self.staging_path = c_xml.get('path', self.config.object_store_cache_path) - - for d_xml in config_xml.findall('extra_dir'): - self.extra_dirs[d_xml.get('type')] = d_xml.get('path') - - log.debug("Object cache dir: %s", self.staging_path) - log.debug(" job work dir: %s", self.extra_dirs['job_work']) - - # for multipart upload - self.s3server = {'access_key': self.access_key, - 'secret_key': self.secret_key, - 'is_secure': self.is_secure, - 'max_chunk_size': self.max_chunk_size, - 'host': self.host, - 'port': self.port, - 'use_rr': self.use_rr, - 'conn_path': self.conn_path} - except Exception: - # Toss it back up after logging, we can't continue loading at this point. - log.exception("Malformed ObjectStore Configuration XML -- unable to continue") - raise + @classmethod + def parse_xml(clazz, config_xml): + return parse_config_xml(config_xml) def __cache_monitor(self): time.sleep(2) # Wait for things to load before starting the monitor diff --git a/test/unit/test_objectstore.py b/test/unit/test_objectstore.py index b0e7945c922..72b08a84b48 100644 --- a/test/unit/test_objectstore.py +++ b/test/unit/test_objectstore.py @@ -3,9 +3,18 @@ from contextlib import contextmanager from shutil import rmtree from string import Template from tempfile import mkdtemp +from xml.etree import ElementTree + +import yaml +from six import StringIO from galaxy import objectstore from galaxy.exceptions import ObjectInvalid +from galaxy.objectstore.azure_blob import AzureBlobObjectStore +from galaxy.objectstore.cloud import Cloud +from galaxy.objectstore.pithos import PithosObjectStore +from galaxy.objectstore.s3 import S3ObjectStore + DISK_TEST_CONFIG = """ @@ -16,60 +25,72 @@ DISK_TEST_CONFIG = """ """ +DISK_TEST_CONFIG_YAML = """ +type: disk +files_dir: "${temp_directory}/files1" +extra_dirs: + - type: temp + path: "${temp_directory}/tmp1" + - type: job_work + path: "${temp_directory}/job_working_directory1" +""" + + def test_disk_store(): - with TestConfig(DISK_TEST_CONFIG) as (directory, object_store): - # Test no dataset with id 1 exists. - absent_dataset = MockDataset(1) - assert not object_store.exists(absent_dataset) + for config_str in [DISK_TEST_CONFIG, DISK_TEST_CONFIG_YAML]: + with TestConfig(config_str) as (directory, object_store): + # Test no dataset with id 1 exists. + absent_dataset = MockDataset(1) + assert not object_store.exists(absent_dataset) - # Write empty dataset 2 in second backend, ensure it is empty and - # exists. - empty_dataset = MockDataset(2) - directory.write("", "files1/000/dataset_2.dat") - assert object_store.exists(empty_dataset) - assert object_store.empty(empty_dataset) + # Write empty dataset 2 in second backend, ensure it is empty and + # exists. + empty_dataset = MockDataset(2) + directory.write("", "files1/000/dataset_2.dat") + assert object_store.exists(empty_dataset) + assert object_store.empty(empty_dataset) - # Write non-empty dataset in backend 1, test it is not emtpy & exists. - hello_world_dataset = MockDataset(3) - directory.write("Hello World!", "files1/000/dataset_3.dat") - assert object_store.exists(hello_world_dataset) - assert not object_store.empty(hello_world_dataset) + # Write non-empty dataset in backend 1, test it is not emtpy & exists. + hello_world_dataset = MockDataset(3) + directory.write("Hello World!", "files1/000/dataset_3.dat") + assert object_store.exists(hello_world_dataset) + assert not object_store.empty(hello_world_dataset) - # Test get_data - data = object_store.get_data(hello_world_dataset) - assert data == "Hello World!" + # Test get_data + data = object_store.get_data(hello_world_dataset) + assert data == "Hello World!" - data = object_store.get_data(hello_world_dataset, start=1, count=6) - assert data == "ello W" + data = object_store.get_data(hello_world_dataset, start=1, count=6) + assert data == "ello W" - # Test Size + # Test Size - # Test absent and empty datasets yield size of 0. - assert object_store.size(absent_dataset) == 0 - assert object_store.size(empty_dataset) == 0 - # Elsewise - assert object_store.size(hello_world_dataset) > 0 # Should this always be the number of bytes? + # Test absent and empty datasets yield size of 0. + assert object_store.size(absent_dataset) == 0 + assert object_store.size(empty_dataset) == 0 + # Elsewise + assert object_store.size(hello_world_dataset) > 0 # Should this always be the number of bytes? - # Test percent used (to some degree) - percent_store_used = object_store.get_store_usage_percent() - assert percent_store_used > 0.0 - assert percent_store_used < 100.0 + # Test percent used (to some degree) + percent_store_used = object_store.get_store_usage_percent() + assert percent_store_used > 0.0 + assert percent_store_used < 100.0 - # Test update_from_file test - output_dataset = MockDataset(4) - output_real_path = os.path.join(directory.temp_directory, "files1", "000", "dataset_4.dat") - assert not os.path.exists(output_real_path) - output_working_path = directory.write("NEW CONTENTS", "job_working_directory1/example_output") - object_store.update_from_file(output_dataset, file_name=output_working_path, create=True) - assert os.path.exists(output_real_path) + # Test update_from_file test + output_dataset = MockDataset(4) + output_real_path = os.path.join(directory.temp_directory, "files1", "000", "dataset_4.dat") + assert not os.path.exists(output_real_path) + output_working_path = directory.write("NEW CONTENTS", "job_working_directory1/example_output") + object_store.update_from_file(output_dataset, file_name=output_working_path, create=True) + assert os.path.exists(output_real_path) - # Test delete - to_delete_dataset = MockDataset(5) - to_delete_real_path = directory.write("content to be deleted!", "files1/000/dataset_5.dat") - assert object_store.exists(to_delete_dataset) - assert object_store.delete(to_delete_dataset) - assert not object_store.exists(to_delete_dataset) - assert not os.path.exists(to_delete_real_path) + # Test delete + to_delete_dataset = MockDataset(5) + to_delete_real_path = directory.write("content to be deleted!", "files1/000/dataset_5.dat") + assert object_store.exists(to_delete_dataset) + assert object_store.delete(to_delete_dataset) + assert not object_store.exists(to_delete_dataset) + assert not os.path.exists(to_delete_real_path) def test_disk_store_alt_name_relpath(): @@ -126,39 +147,64 @@ HIERARCHICAL_TEST_CONFIG = """ """ +HIERARCHICAL_TEST_CONFIG_YAML = """ +type: hierarchical +backends: + - id: files1 + type: disk + weight: 1 + files_dir: "${temp_directory}/files1" + extra_dirs: + - type: temp + path: "${temp_directory}/tmp1" + - type: job_work + path: "${temp_directory}/job_working_directory1" + - id: files2 + type: disk + weight: 1 + files_dir: "${temp_directory}/files2" + extra_dirs: + - type: temp + path: "${temp_directory}/tmp2" + - type: job_work + path: "${temp_directory}/job_working_directory2" +""" + + def test_hierarchical_store(): - with TestConfig(HIERARCHICAL_TEST_CONFIG) as (directory, object_store): + for config_str in [HIERARCHICAL_TEST_CONFIG, HIERARCHICAL_TEST_CONFIG_YAML]: + with TestConfig(config_str) as (directory, object_store): - # Test no dataset with id 1 exists. - assert not object_store.exists(MockDataset(1)) + # Test no dataset with id 1 exists. + assert not object_store.exists(MockDataset(1)) - # Write empty dataset 2 in second backend, ensure it is empty and - # exists. - directory.write("", "files2/000/dataset_2.dat") - assert object_store.exists(MockDataset(2)) - assert object_store.empty(MockDataset(2)) + # Write empty dataset 2 in second backend, ensure it is empty and + # exists. + directory.write("", "files2/000/dataset_2.dat") + assert object_store.exists(MockDataset(2)) + assert object_store.empty(MockDataset(2)) - # Write non-empty dataset in backend 1, test it is not emtpy & exists. - directory.write("Hello World!", "files1/000/dataset_3.dat") - assert object_store.exists(MockDataset(3)) - assert not object_store.empty(MockDataset(3)) + # Write non-empty dataset in backend 1, test it is not emtpy & exists. + directory.write("Hello World!", "files1/000/dataset_3.dat") + assert object_store.exists(MockDataset(3)) + assert not object_store.empty(MockDataset(3)) - # Assert creation always happens in first backend. - for i in range(100): - dataset = MockDataset(100 + i) - object_store.create(dataset) - assert object_store.get_filename(dataset).find("files1") > 0 + # Assert creation always happens in first backend. + for i in range(100): + dataset = MockDataset(100 + i) + object_store.create(dataset) + assert object_store.get_filename(dataset).find("files1") > 0 DISTRIBUTED_TEST_CONFIG = """ - + - + @@ -168,28 +214,290 @@ DISTRIBUTED_TEST_CONFIG = """ """ +DISTRIBUTED_TEST_CONFIG_YAML = """ +type: distributed +backends: + - id: files1 + type: disk + weight: 2 + files_dir: "${temp_directory}/files1" + extra_dirs: + - type: temp + path: "${temp_directory}/tmp1" + - type: job_work + path: "${temp_directory}/job_working_directory1" + - id: files2 + type: disk + weight: 1 + files_dir: "${temp_directory}/files2" + extra_dirs: + - type: temp + path: "${temp_directory}/tmp2" + - type: job_work + path: "${temp_directory}/job_working_directory2" +""" + + def test_distributed_store(): - with TestConfig(DISTRIBUTED_TEST_CONFIG) as (directory, object_store): - with __stubbed_persistence() as persisted_ids: - for i in range(100): - dataset = MockDataset(100 + i) - object_store.create(dataset) + for config_str in [DISTRIBUTED_TEST_CONFIG, DISTRIBUTED_TEST_CONFIG_YAML]: + with TestConfig(config_str) as (directory, object_store): + with __stubbed_persistence() as persisted_ids: + for i in range(100): + dataset = MockDataset(100 + i) + object_store.create(dataset) - # Test distributes datasets between backends according to weights - backend_1_count = len([v for v in persisted_ids.values() if v == "files1"]) - backend_2_count = len([v for v in persisted_ids.values() if v == "files2"]) + # Test distributes datasets between backends according to weights + backend_1_count = len([v for v in persisted_ids.values() if v == "files1"]) + backend_2_count = len([v for v in persisted_ids.values() if v == "files2"]) - assert backend_1_count > 0 - assert backend_2_count > 0 - assert backend_1_count > backend_2_count + assert backend_1_count > 0 + assert backend_2_count > 0 + assert backend_1_count > backend_2_count + + +# Unit testing the cloud and advanced infrastructure object stores is difficult, but +# we can at least stub out initializing and test the configuration of these things from +# XML and dicts. +class UnitializedPithosObjectStore(PithosObjectStore): + + def _initialize(self): + pass + + +class UnitializeS3ObjectStore(S3ObjectStore): + + def _initialize(self): + pass + + +class UnitializedAzureBlobObjectStore(AzureBlobObjectStore): + + def _initialize(self): + pass + + +class UnitializedCloudObjectStore(Cloud): + + def _initialize(self): + pass + + +PITHOS_TEST_CONFIG = """ + + + + + + +""" + + +PITHOS_TEST_CONFIG_YAML = """ +type: pithos +auth: + url: http://example.org/ + token: extoken123 + +container: + name: foo + project: cow + +extra_dirs: + - type: temp + path: database/tmp_pithos + - type: job_work + path: database/working_pithos +""" + + +def test_config_parse_pithos(): + for config_str in [PITHOS_TEST_CONFIG, PITHOS_TEST_CONFIG_YAML]: + with TestConfig(config_str, clazz=UnitializedPithosObjectStore) as (directory, object_store): + configured_config_dict = object_store.config_dict + for key in ["auth", "container", "extra_dirs"]: + assert key in configured_config_dict + + auth_dict = configured_config_dict["auth"] + _assert_key_has_value(auth_dict, "url", "http://example.org/") + _assert_key_has_value(auth_dict, "token", "extoken123") + + container_dict = configured_config_dict["container"] + + _assert_key_has_value(container_dict, "name", "foo") + _assert_key_has_value(container_dict, "project", "cow") + + assert object_store.extra_dirs["job_work"] == "database/working_pithos" + assert object_store.extra_dirs["temp"] == "database/tmp_pithos" + + +S3_TEST_CONFIG = """ + + + + + + +""" + + +S3_TEST_CONFIG_YAML = """ +type: s3 +auth: + access_key: access_moo + secret_key: secret_cow + +bucket: + name: unique_bucket_name_all_lowercase + use_reduced_redundancy: false + +cache: + path: database/object_store_cache + size: 1000 + +extra_dirs: +- type: job_work + path: database/job_working_directory_s3 +- type: temp + path: database/tmp_s3 +""" + + +def test_config_parse_s3(): + for config_str in [S3_TEST_CONFIG, S3_TEST_CONFIG_YAML]: + with TestConfig(config_str, clazz=UnitializeS3ObjectStore) as (directory, object_store): + assert object_store.access_key == "access_moo" + assert object_store.secret_key == "secret_cow" + + assert object_store.bucket == "unique_bucket_name_all_lowercase" + assert object_store.use_rr is False + + assert object_store.host is None + assert object_store.port == 6000 + assert object_store.multipart is True + assert object_store.is_secure is True + assert object_store.conn_path == "/" + + assert object_store.cache_size == 1000 + assert object_store.staging_path == "database/object_store_cache" + assert object_store.extra_dirs["job_work"] == "database/job_working_directory_s3" + assert object_store.extra_dirs["temp"] == "database/tmp_s3" + + +CLOUD_TEST_CONFIG = """ + + + + + + +""" + + +CLOUD_TEST_CONFIG_YAML = """ +type: s3 +auth: + access_key: access_moo + secret_key: secret_cow + +bucket: + name: unique_bucket_name_all_lowercase + use_reduced_redundancy: false + +cache: + path: database/object_store_cache + size: 1000 + +extra_dirs: +- type: job_work + path: database/job_working_directory_cloud +- type: temp + path: database/tmp_cloud +""" + + +def test_config_parse_cloud(): + for config_str in [CLOUD_TEST_CONFIG, CLOUD_TEST_CONFIG_YAML]: + with TestConfig(config_str, clazz=UnitializedCloudObjectStore) as (directory, object_store): + assert object_store.access_key == "access_moo" + assert object_store.secret_key == "secret_cow" + + assert object_store.bucket == "unique_bucket_name_all_lowercase" + assert object_store.use_rr is False + + assert object_store.host is None + assert object_store.port == 6000 + assert object_store.multipart is True + assert object_store.is_secure is True + assert object_store.conn_path == "/" + + assert object_store.cache_size == 1000 + assert object_store.staging_path == "database/object_store_cache" + assert object_store.extra_dirs["job_work"] == "database/job_working_directory_cloud" + assert object_store.extra_dirs["temp"] == "database/tmp_cloud" + + +AZURE_BLOB_TEST_CONFIG = """ + + + + + + +""" + + +AZURE_BLOB_TEST_CONFIG_YAML = """ +type: azure_blob +auth: + account_name: azureact + account_key: password123 + +container: + name: unique_container_name + max_chunk_size: 250 + +cache: + path: database/object_store_cache + size: 100 + +extra_dirs: +- type: job_work + path: database/job_working_directory_azure +- type: temp + path: database/tmp_azure +""" + + +def test_config_parse_azure(): + for config_str in [AZURE_BLOB_TEST_CONFIG, AZURE_BLOB_TEST_CONFIG_YAML]: + with TestConfig(config_str, clazz=UnitializedAzureBlobObjectStore) as (directory, object_store): + assert object_store.account_name == "azureact" + assert object_store.account_key == "password123" + + assert object_store.container_name == "unique_container_name" + assert object_store.max_chunk_size == 250 + + assert object_store.cache_size == 100 + assert object_store.staging_path == "database/object_store_cache" + assert object_store.extra_dirs["job_work"] == "database/job_working_directory_azure" + assert object_store.extra_dirs["temp"] == "database/tmp_azure" class TestConfig(object): - def __init__(self, config_xml): + def __init__(self, config_str, clazz=None): self.temp_directory = mkdtemp() - self.write(config_xml, "store.xml") - config = MockConfig(self.temp_directory) - self.object_store = objectstore.build_object_store_from_config(config) + if config_str.startswith("<"): + config_file = "store.xml" + else: + config_file = "store.yaml" + self.write(config_str, config_file) + config = MockConfig(self.temp_directory, config_file) + if clazz is None: + self.object_store = objectstore.build_object_store_from_config(config) + elif config_file == "store.xml": + self.object_store = clazz.from_xml(config, ElementTree.fromstring(config_str)) + else: + self.object_store = clazz(config, yaml.safe_load(StringIO(config_str))) def __enter__(self): return self, self.object_store @@ -210,9 +518,9 @@ class TestConfig(object): class MockConfig(object): - def __init__(self, temp_directory): + def __init__(self, temp_directory, config_file): self.file_path = temp_directory - self.object_store_config_file = os.path.join(temp_directory, "store.xml") + self.object_store_config_file = os.path.join(temp_directory, config_file) self.object_store_check_old_style = False self.jobs_directory = temp_directory self.new_file_path = temp_directory @@ -245,3 +553,8 @@ def __stubbed_persistence(): finally: setattr(objectstore, PERSIST_METHOD_NAME, real_method) + + +def _assert_key_has_value(the_dict, key, value): + assert key in the_dict, "dict [%s] doesn't container expected key [%s]" % key + assert the_dict[key] == value, "%s != %s" % (the_dict[key], value) From c7b133b74bfb889cd3b111faa5b1ce17133ca931 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 5 Dec 2018 12:18:02 -0500 Subject: [PATCH 070/260] Localize version strings to the working directory w/predictable path. new_file_path is awful and we shouldn't be using it. --- lib/galaxy/jobs/__init__.py | 8 ++++++++ test/unit/jobs/test_job_wrapper.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 3b2dee083f4..b1cf979ef35 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -20,6 +20,7 @@ from tempfile import NamedTemporaryFile from xml.etree import ElementTree import six +from pulsar.client.staging import COMMAND_VERSION_FILENAME import galaxy from galaxy import model, util @@ -830,6 +831,10 @@ class JobWrapper(HasResourceParameters): return param_dict def get_version_string_path(self): + return os.path.abspath(os.path.join(self.working_directory, COMMAND_VERSION_FILENAME)) + + # TODO: Remove in Galaxy 20.XX, for running jobs at GX upgrade + def get_version_string_path_legacy(self): return os.path.abspath(os.path.join(self.app.config.new_file_path, "GALAXY_VERSION_STRING_%s" % self.job_id)) def __prepare_upload_paramfile(self, tool_evaluator): @@ -1274,6 +1279,9 @@ class JobWrapper(HasResourceParameters): if self.tool.version_string_cmd: version_filename = self.get_version_string_path() + # TODO: Remove in Galaxy 20.XX, for running jobs at GX upgrade + if not os.path.exists(version_filename): + version_filename = self.get_version_string_path_legacy() if os.path.exists(version_filename): self.version_string = open(version_filename).read() os.unlink(version_filename) diff --git a/test/unit/jobs/test_job_wrapper.py b/test/unit/jobs/test_job_wrapper.py index 59a67f41dae..5e370587116 100644 --- a/test/unit/jobs/test_job_wrapper.py +++ b/test/unit/jobs/test_job_wrapper.py @@ -52,7 +52,7 @@ class BaseWrapperTestCase(UsesApp): def test_version_path(self): wrapper = self._wrapper() - version_path = wrapper.get_version_string_path() + version_path = wrapper.get_version_string_path_legacy() expected_path = os.path.join(self.test_directory, "new_files", "GALAXY_VERSION_STRING_345") self.assertEqual(version_path, expected_path) From c4ab2a0c6bb768fe27b48648d21ab42733133c08 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 26 Nov 2018 16:30:00 +0000 Subject: [PATCH 071/260] Add workaround for https://github.com/travis-ci/travis-ci/issues/7940 --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 39de25d6583..346002a7fb4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,10 @@ matrix: - env: TOX_ENV=py34-unit addons: *py3_addons +before_install: + # Workaround for https://github.com/travis-ci/travis-ci/issues/7940 + - sudo rm -f /etc/boto.cfg + install: - set -e - pip install tox From 0e5c96ef3710612bf1171e7094cf7d5427387ecb Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Thu, 19 May 2016 14:42:52 -0400 Subject: [PATCH 072/260] Drop "Galaxy pip" in favor of manylinux1 wheels (which are supported in pip >= 8.1). --- scripts/common_startup.sh | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index 405cbfedb09..bd073a53235 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -134,31 +134,7 @@ fi : ${GALAXY_WHEELS_INDEX_URL:="https://wheels.galaxyproject.org/simple"} if [ $REPLACE_PIP -eq 1 ]; then - pip_version=`pip --version | awk '{print $2}'` - method=`python - << EOF -from pkg_resources import parse_version -from sys import stdout -if parse_version('$pip_version') >= parse_version('6.0'): - stdout.write('req') -elif parse_version('$pip_version') >= parse_version('1.5'): - stdout.write('wheel') -else: - stdout.write('sdist') -EOF` - case $method in - req) - pip install --no-index --find-links ${GALAXY_WHEELS_INDEX_URL}/pip --upgrade "pip==${GXPIP_VERSION}" - ;; - wheel) - pip install --upgrade "https://wheels.galaxyproject.org/packages/pip-${GXPIP_VERSION}-py2.py3-none-any.whl" - ;; - sdist) - pip install --upgrade "https://wheels.galaxyproject.org/packages/pip-${GXPIP_VERSION}.tar.gz" - ;; - esac - - # binary-compatibility.cfg may need to be created (e.g. on CentOS) - [ -n "$VIRTUAL_ENV" -a ! -f ${VIRTUAL_ENV}/binary-compatibility.cfg ] && python ./scripts/binary_compatibility.py -o ${VIRTUAL_ENV}/binary-compatibility.cfg + pip install 'pip>=8.1' fi if [ $FETCH_WHEELS -eq 1 ]; then From b10cfa43ac547eaecbd996cef7201170a0e0da8c Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Fri, 20 May 2016 09:44:55 -0400 Subject: [PATCH 073/260] Remove unused $GXPIP_VERSION --- scripts/common_startup.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index bd073a53235..97b3c446609 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -1,8 +1,6 @@ #!/bin/bash set -e -GXPIP_VERSION='8.0.2+gx2' - SET_VENV=1 for arg in "$@"; do [ "$arg" = "--skip-venv" ] && SET_VENV=0 From efe84751a7bf84b2297df7b77b514b8425fe7c70 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 26 Nov 2018 16:30:00 +0000 Subject: [PATCH 074/260] Add workaround for https://github.com/travis-ci/travis-ci/issues/7940 --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 6e1c444185d..dcb42ef598c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,10 @@ env: - TOX_ENV=py26-unit - TOX_ENV=qunit +before_install: + # Workaround for https://github.com/travis-ci/travis-ci/issues/7940 + - sudo rm -f /etc/boto.cfg + install: - pip install tox - if [ "$TOX_ENV" == "qunit" ]; then bash -c 'cd test/qunit && npm install'; fi From e9aec176fac8bb3681ea533be6178ffd43c3bf4d Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 11 Dec 2018 17:56:56 -0500 Subject: [PATCH 075/260] Fixes for workflow scheduling tests. --- lib/galaxy/jobs/__init__.py | 3 +-- lib/galaxy/workflow/scheduling_manager.py | 13 +++++-------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 74d0af6ec17..6e239f94a81 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -173,8 +173,7 @@ class JobConfiguration(ConfiguresHandlers): tree = load(job_config_file) self.__parse_job_conf_xml(tree) except IOError: - log.warning('Job configuration "%s" does not exist, using default' - ' job configuration (this server will run jobs)', + log.warning('Job configuration "%s" does not exist, using default job configuration', self.app.config.job_config_file) self.__set_default_job_conf() except Exception as e: diff --git a/lib/galaxy/workflow/scheduling_manager.py b/lib/galaxy/workflow/scheduling_manager.py index 58d8d0d574a..c944ae39ed5 100644 --- a/lib/galaxy/workflow/scheduling_manager.py +++ b/lib/galaxy/workflow/scheduling_manager.py @@ -116,7 +116,7 @@ class WorkflowSchedulingManager(ConfiguresHandlers): def _assign_handler(self, workflow_invocation): # Use random-ish integer history_id to produce a consistent index to pick # job handler with. - random_index = workflow_invocation.history_id + random_index = workflow_invocation.history.id queue_callback = partial(self._queue_callback, workflow_invocation) message_callback = partial(self._message_callback, workflow_invocation) if self.app.config.parallelize_workflow_scheduling_within_histories: @@ -188,7 +188,7 @@ class WorkflowSchedulingManager(ConfiguresHandlers): if not self.__handlers_configured and self.__stack_has_pool: # Stack has a pool for us so override inherited config and use the pool - self.__init_handler_assignment_methods() + self.__init_handlers() self.__handlers_configured = True def __init_default_scheduler(self): @@ -225,14 +225,10 @@ class WorkflowSchedulingManager(ConfiguresHandlers): else: log.warning(EXCEPTION_MESSAGE_SERIALIZE) - def __init_handlers(self, config_element): + def __init_handlers(self, config_element=None): assert not self.__handlers_configured - self.__init_handler_assignment_methods(config_element) - self._init_handlers(config_element) - self.__handlers_configured = True - - def __init_handler_assignment_methods(self, config_element=None): self._init_handler_assignment_methods(config_element) + self._init_handlers(config_element) if not self.handler_assignment_methods_configured: self._set_default_handler_assignment_methods() else: @@ -240,6 +236,7 @@ class WorkflowSchedulingManager(ConfiguresHandlers): log.info("Workflow scheduling handler assignment method(s): %s", ', '.join(self.handler_assignment_methods)) for tag, handlers in [(t, h) for t, h in self.handlers.items() if isinstance(h, list)]: log.info("Tag [%s] handlers: %s", tag, ', '.join(handlers)) + self.__handlers_configured = True def __init_plugin(self, plugin_type, workflow_scheduler_id=None, **kwds): workflow_scheduler_id = workflow_scheduler_id or self.default_scheduler_id From de3b7bb55fd4949456bfb61637cb67b1fd367d6e Mon Sep 17 00:00:00 2001 From: guerler Date: Tue, 11 Dec 2018 18:19:28 -0500 Subject: [PATCH 076/260] Add display applications api endpoint --- .../galaxy/api/display_applications.py | 36 +++++++++++++++++++ lib/galaxy/webapps/galaxy/buildapp.py | 10 ++++++ 2 files changed, 46 insertions(+) create mode 100644 lib/galaxy/webapps/galaxy/api/display_applications.py diff --git a/lib/galaxy/webapps/galaxy/api/display_applications.py b/lib/galaxy/webapps/galaxy/api/display_applications.py new file mode 100644 index 00000000000..bafafadd8aa --- /dev/null +++ b/lib/galaxy/webapps/galaxy/api/display_applications.py @@ -0,0 +1,36 @@ +""" +API operations on annotations. +""" +import logging + +from galaxy import exceptions +from galaxy.web import expose_api +from galaxy.web.base.controller import BaseAPIController + +log = logging.getLogger(__name__) + + +class DisplayApplicationsController(BaseAPIController): + + @expose_api + def index(self, trans, **kwd): + """ + GET /api/display_applications/ + + Returns the list of display applications. + + :returns: list of available display applications + :rtype: list + """ + response = [] + for display_app in trans.app.datatypes_registry.display_applications.values(): + + response.append({ + "id" : display_app.id, + "name": display_app.name, + "version": display_app.version, + "filename_": display_app._filename, + "links": [{"name": l.name} for l in display_app.links.values()] + }) + return response + diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 20f7f361258..26e6ead41d2 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -553,6 +553,16 @@ def populate_api_routes(webapp, app): action='get_api_key', conditions=dict(method=["GET"])) + # ======================= + # ====== DISPLAY APPLICATIONS API ====== + # ======================= + + webapp.mapper.connect('index', + '/api/display_applications', + controller='display_applications', + action='index', + conditions=dict(method=["GET"])) + # ===================== # ===== TOURS API ===== # ===================== From 6546e35bff27a60b09bf30ca68c2beb62a61acd4 Mon Sep 17 00:00:00 2001 From: guerler Date: Tue, 11 Dec 2018 18:30:09 -0500 Subject: [PATCH 077/260] Add reload feature to display applications api endpoint --- .../galaxy/api/display_applications.py | 39 +++++++++++++++---- lib/galaxy/webapps/galaxy/buildapp.py | 6 +++ 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/display_applications.py b/lib/galaxy/webapps/galaxy/api/display_applications.py index bafafadd8aa..35c4d02d0bb 100644 --- a/lib/galaxy/webapps/galaxy/api/display_applications.py +++ b/lib/galaxy/webapps/galaxy/api/display_applications.py @@ -3,7 +3,7 @@ API operations on annotations. """ import logging -from galaxy import exceptions +from galaxy.exceptions import MessageException from galaxy.web import expose_api from galaxy.web.base.controller import BaseAPIController @@ -24,13 +24,38 @@ class DisplayApplicationsController(BaseAPIController): """ response = [] for display_app in trans.app.datatypes_registry.display_applications.values(): - response.append({ - "id" : display_app.id, - "name": display_app.name, - "version": display_app.version, - "filename_": display_app._filename, - "links": [{"name": l.name} for l in display_app.links.values()] + 'id' : display_app.id, + 'name': display_app.name, + 'version': display_app.version, + 'filename_': display_app._filename, + 'links': [{'name': l.name} for l in display_app.links.values()] }) return response + @expose_api + def reload(self, trans, **kwd): + """ + GET /api/display_applications/reload + + Reloads the list of display applications. + + :param ids: list containing ids of display to be reloaded + :type ids: list + """ + ids = payload.get('ids') + galaxy.queue_worker.send_control_task(trans.app, + 'reload_display_application', + noop_self=True, + kwargs={'display_application_ids': ids}) + reloaded, failed = trans.app.datatypes_registry.reload_display_applications(ids) + if not reloaded and failed: + raise MessageException('Unable to reload any of the %i requested display applications ("%s").' + % (len(failed), '", "'.join(failed))) + if failed: + raise MessageException('Reloaded %i display applications ("%s"), but failed to reload %i display applications ("%s").' + % (len(reloaded), '", "'.join(reloaded), len(failed), '", "'.join(failed))) + if not reloaded: + raise MessageException('You need to request at least one display application to reload.') + return {'message': 'Reloaded %i requested display applications ("%s").' % (len(reloaded), '", "'.join(reloaded)))} + diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 26e6ead41d2..49e4238f868 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -563,6 +563,12 @@ def populate_api_routes(webapp, app): action='index', conditions=dict(method=["GET"])) + webapp.mapper.connect('reload', + '/api/display_applications/reload', + controller='display_applications', + action='reload', + conditions=dict(method=["POST"])) + # ===================== # ===== TOURS API ===== # ===================== From eb0c0ad9ab204cee83233523c01b11aa040a6388 Mon Sep 17 00:00:00 2001 From: guerler Date: Tue, 11 Dec 2018 19:04:15 -0500 Subject: [PATCH 078/260] Add display application vue component --- client/galaxy/scripts/apps/admin.js | 6 +++ .../components/admin/DisplayApplications.vue | 44 +++++++++++++++++++ .../galaxy/api/display_applications.py | 2 +- lib/galaxy/webapps/galaxy/buildapp.py | 1 + .../webapps/galaxy/controllers/admin.py | 2 +- 5 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 client/galaxy/scripts/components/admin/DisplayApplications.vue diff --git a/client/galaxy/scripts/apps/admin.js b/client/galaxy/scripts/apps/admin.js index c52f399ba58..cbd957ed4b2 100644 --- a/client/galaxy/scripts/apps/admin.js +++ b/client/galaxy/scripts/apps/admin.js @@ -14,6 +14,7 @@ import DataTables from "components/admin/DataTables.vue"; import DataTypes from "components/admin/DataTypes.vue"; import DataManagerView from "components/admin/DataManager/DataManagerView.vue"; import DataManagerRouter from "components/admin/DataManager/DataManagerRouter.vue"; +import DisplayApplications from "components/admin/DisplayApplications.vue"; import Vue from "vue"; import { serverPath } from "utils/serverPath"; @@ -33,6 +34,7 @@ window.app = function app(options, bootstrapped) { "(/)admin(/)users": "show_users", "(/)admin(/)roles": "show_roles", "(/)admin(/)groups": "show_groups", + "(/)admin(/)display_applications": "show_display_applications", "(/)admin(/)tool_versions": "show_tool_versions", "(/)admin(/)quotas": "show_quotas", "(/)admin(/)repositories": "show_repositories", @@ -124,6 +126,10 @@ window.app = function app(options, bootstrapped) { this._display_vue_helper(DataTypes); }, + show_display_applications: function() { + this._display_vue_helper(DisplayApplications); + }, + show_data_manager: function(path) { let vueMount = document.createElement("div"); this.page.display(vueMount); diff --git a/client/galaxy/scripts/components/admin/DisplayApplications.vue b/client/galaxy/scripts/components/admin/DisplayApplications.vue new file mode 100644 index 00000000000..415d01bafc6 --- /dev/null +++ b/client/galaxy/scripts/components/admin/DisplayApplications.vue @@ -0,0 +1,44 @@ + + diff --git a/lib/galaxy/webapps/galaxy/api/display_applications.py b/lib/galaxy/webapps/galaxy/api/display_applications.py index 35c4d02d0bb..6b27d553690 100644 --- a/lib/galaxy/webapps/galaxy/api/display_applications.py +++ b/lib/galaxy/webapps/galaxy/api/display_applications.py @@ -57,5 +57,5 @@ class DisplayApplicationsController(BaseAPIController): % (len(reloaded), '", "'.join(reloaded), len(failed), '", "'.join(failed))) if not reloaded: raise MessageException('You need to request at least one display application to reload.') - return {'message': 'Reloaded %i requested display applications ("%s").' % (len(reloaded), '", "'.join(reloaded)))} + return {'message': 'Reloaded %i requested display applications ("%s").' % (len(reloaded), '", "'.join(reloaded))} diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 49e4238f868..485715b95a4 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -103,6 +103,7 @@ def app_factory(global_conf, load_app_kwds={}, **kwargs): webapp.add_client_route('/admin/data_types', 'admin') webapp.add_client_route('/admin/data_manager{path_info:.*}', 'admin') webapp.add_client_route('/admin/users', 'admin') + webapp.add_client_route('/admin/display_applications', 'admin') webapp.add_client_route('/admin/roles', 'admin') webapp.add_client_route('/admin/forms', 'admin') webapp.add_client_route('/admin/groups', 'admin') diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py index a8d411be2e0..cf7b33e8b37 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -910,7 +910,7 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP @web.expose @web.require_admin - def display_applications(self, trans, **kwd): + def display_applications_legacy(self, trans, **kwd): return trans.fill_template('admin/view_display_applications.mako', display_applications=trans.app.datatypes_registry.display_applications) @web.expose From 727ab11480962ec31387cc6934c0fcec14a4ead0 Mon Sep 17 00:00:00 2001 From: guerler Date: Tue, 11 Dec 2018 19:20:28 -0500 Subject: [PATCH 079/260] Fix spacing in admin app router --- client/galaxy/scripts/apps/panels/admin-panel.js | 1 + 1 file changed, 1 insertion(+) diff --git a/client/galaxy/scripts/apps/panels/admin-panel.js b/client/galaxy/scripts/apps/panels/admin-panel.js index e77995c4e0a..dbd4eb4c4f9 100644 --- a/client/galaxy/scripts/apps/panels/admin-panel.js +++ b/client/galaxy/scripts/apps/panels/admin-panel.js @@ -34,6 +34,7 @@ var AdminPanel = Backbone.View.extend({ { title: _l("Display applications"), url: "admin/display_applications", + target: "__use_router__", id: "admin-link-display-applications" }, { From 35d2cd9bf62a748bc3dd5963a3246b5a5a7e2a18 Mon Sep 17 00:00:00 2001 From: guerler Date: Tue, 11 Dec 2018 19:33:04 -0500 Subject: [PATCH 080/260] Load display application data and display number of available applications --- client/galaxy/scripts/apps/admin.js | 3 +++ .../components/admin/DisplayApplications.vue | 27 +++++++++++++++---- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/client/galaxy/scripts/apps/admin.js b/client/galaxy/scripts/apps/admin.js index cbd957ed4b2..831c65c54cd 100644 --- a/client/galaxy/scripts/apps/admin.js +++ b/client/galaxy/scripts/apps/admin.js @@ -113,15 +113,18 @@ window.app = function app(options, bootstrapped) { }) ); }, + _display_vue_helper: function(component, props) { let instance = Vue.extend(component); let vm = document.createElement("div"); this.page.display(vm); new instance(props).$mount(vm); }, + show_data_tables: function() { this._display_vue_helper(DataTables); }, + show_data_types: function() { this._display_vue_helper(DataTypes); }, diff --git a/client/galaxy/scripts/components/admin/DisplayApplications.vue b/client/galaxy/scripts/components/admin/DisplayApplications.vue index 415d01bafc6..02bb25f5dab 100644 --- a/client/galaxy/scripts/components/admin/DisplayApplications.vue +++ b/client/galaxy/scripts/components/admin/DisplayApplications.vue @@ -1,9 +1,15 @@ diff --git a/client/galaxy/scripts/components/Masthead/initMasthead.js b/client/galaxy/scripts/components/Masthead/initMasthead.js new file mode 100644 index 00000000000..8b98eeec47f --- /dev/null +++ b/client/galaxy/scripts/components/Masthead/initMasthead.js @@ -0,0 +1,16 @@ +/** + * Temporary function used to mount the masthead inside the current application. + * This function is exposed with the rest of the page-globals in extended. + */ +import Vue from "vue"; +import Masthead from "components/Masthead"; + +export function initMasthead(config, container) { + console.log("initMasthead"); + let Component = Vue.extend(Masthead); + return new Component({ + props: Object.keys(config), + propsData: config, + el: container + }); +} diff --git a/client/galaxy/scripts/components/Masthead/styles.scss b/client/galaxy/scripts/components/Masthead/styles.scss new file mode 100644 index 00000000000..002eed8185f --- /dev/null +++ b/client/galaxy/scripts/components/Masthead/styles.scss @@ -0,0 +1,14 @@ +#masthead:empty { + display: none; +} + +.framed { + + #messagebox, #masthead { + display: none; + } + + #center, #right, #left { + top: 0; + } +} diff --git a/client/galaxy/scripts/components/RuleCollectionBuilder.vue b/client/galaxy/scripts/components/RuleCollectionBuilder.vue index 239fc9045de..dae1882b3e1 100644 --- a/client/galaxy/scripts/components/RuleCollectionBuilder.vue +++ b/client/galaxy/scripts/components/RuleCollectionBuilder.vue @@ -413,6 +413,7 @@ <%def name="repository_installation_status_updater()"> @@ -368,8 +387,11 @@ %> %if can_update: %endif @@ -388,8 +410,11 @@ %> %if can_update: %endif diff --git a/templates/base.mako b/templates/base.mako index 131089ea851..f14ee5fbb4a 100644 --- a/templates/base.mako +++ b/templates/base.mako @@ -20,8 +20,11 @@ %endif | ${self.title()} + ## relative href for site root + ## TODO: try tag? + ${self.metas()} ${self.stylesheets()} ${self.javascripts()} @@ -40,36 +43,36 @@ ## Default stylesheets <%def name="stylesheets()"> - ${h.css("base")} ${h.css('bootstrap-tour')} ${h.css('base')} ## Default javascripts <%def name="javascripts()"> - ## Send errors to Sentry server if configured - %if app.config.sentry_dsn: - ${h.js( "libs/raven" )} - - %endif - + ## TODO: remove when all libs are required directly in modules ${h.js( - ## TODO: remove when all libs are required directly in modules 'libs/require', 'bundled/libs.chunk', - 'bundled/base.chunk', - 'bundled/extended.bundled' + 'bundled/base.chunk' )} + ${self.javascript_entry()} + + +<%def name="javascript_entry()"> + ${h.js('bundled/generic.bundled')} + + +<%def name="javascript_app()"> + + ${ galaxy_client.load( app=self.js_app ) } + ${ galaxy_client.config_sentry( app=self.js_app ) } + ${ galaxy_client.config_google_analytics( app=self.js_app ) } %if not form_input_auto_focus is UNDEFINED and form_input_auto_focus: - %endif - + ${h.js( - ## TODO: remove when all libs are required directly in modules - 'libs/require', 'bundled/libs.chunk', - 'bundled/base.chunk', - 'bundled/extended.bundled' + 'bundled/base.chunk' )} - + ${ javascript_entry() } + + +<%def name="javascript_entry()"> + + ${ h.js('bundled/generic.bundled')} <%def name="javascript_app()"> - ## load the Galaxy global js var + ${ galaxy_client.load() } ## Default late-load javascripts <%def name="late_javascripts()"> - ## Scripts can be loaded later since they progressively add features to - ## the panels, but do not change layout + + + + %if t.webapp.name == 'galaxy' and app.config.ga_code: + ${galaxy_client.config_google_analytics(app)} + %endif + ## Masthead @@ -149,11 +130,20 @@ %endif | ${self.title()} + ## relative href for site root + ${self.stylesheets()} + + ## Normally, we'd put all the javascripts at the bottom of the + ## but during this transitional period we need access to the config + ## functions which are only available when these scripts are executed + ## and I can't yet control when the templates are going to write scripts + ## to the output ${self.javascripts()} ${self.javascript_app()} + <% @@ -176,23 +166,29 @@ %endif

+ ## Background displays first
+ ## Layer iframes over backgrounds + %if self.message_box_visible:
${app.config.message_box_content}
%endif + %if self.show_inactivity_warning:
${app.config.inactivity_box_content} Resend verification.
%endif + ${self.overlay(visible=self.overlay_visible)} +
%if self.has_left_panel:
@@ -217,6 +213,7 @@ %endif
+ ## Allow other body level elements ## Scripts can be loaded later since they progressively add features to diff --git a/templates/display_base.mako b/templates/display_base.mako index 853c5a9d2dc..1eef081b2c0 100644 --- a/templates/display_base.mako +++ b/templates/display_base.mako @@ -30,17 +30,15 @@ %> -<%def name="javascripts()"> - ${parent.javascripts()} +<%def name="javascript_app()"> + ${parent.javascript_app()} ${community_tag_js( get_controller_name( item ) )} <%def name="stylesheets()"> ${parent.stylesheets()} ${h.css( - "autocomplete_tagging", "embed_item", - "jquery.rating", "library", "jquery-ui/smoothness/jquery-ui" )} @@ -230,7 +228,11 @@ ## Community tags.
Community: - ${render_community_tagging_element( tagged_item=item, tag_click_fn='community_tag_click', use_toggle_link=False )} + ${render_community_tagging_element( + tagged_item=item, + tag_click_fn='community_tag_click', + use_toggle_link=False + )} %if len ( item.tags ) == 0: none %endif @@ -240,7 +242,13 @@

Yours: - ${render_individual_tagging_element( user=trans.get_user(), tagged_item=item, elt_context='view.mako', use_toggle_link=False, tag_click_fn='community_tag_click' )} + ${render_individual_tagging_element( + user=trans.get_user(), + tagged_item=item, + elt_context='view.mako', + use_toggle_link=False, + tag_click_fn='community_tag_click' + )}
%endif
diff --git a/templates/galaxy_client_app.mako b/templates/galaxy_client_app.mako index 2fec615d068..4b4cb51d032 100644 --- a/templates/galaxy_client_app.mako +++ b/templates/galaxy_client_app.mako @@ -6,13 +6,15 @@ ${ h.dumps( dictionary, indent=( 2 if trans.debug else 0 ) ) } <%def name="load( app=None, **kwargs )"> +<%def name="config_sentry(app)"> + %if app and app.config: + + %endif + + +<%def name="config_google_analytics(app)"> + %if app and app.config and app.config.ga_code: + + %endif + + + ## ---------------------------------------------------------------------------- <%def name="get_config_dict()"> diff --git a/templates/js-app.mako b/templates/js-app.mako index 8aa4114aa8c..71d78bb471a 100644 --- a/templates/js-app.mako +++ b/templates/js-app.mako @@ -1,3 +1,4 @@ +<%namespace name="galaxy_client" file="/galaxy_client_app.mako" /> @@ -14,54 +15,54 @@ | ${app.config.brand} %endif + ## relative href for site root + ## TODO: use loaders to move everything but the essentials below the fold ${ h.css( - ## 'jquery.rating', 'jquery-ui/smoothness/jquery-ui', - ## base needs to come after jquery-ui because of ui-button, ui- etc. name collision - 'base', - ##'bootstrap-tour', + 'bootstrap-tour', + 'base' )} - ${ page_setup() } + - ${ js_disabled_warning() } - - ## js libraries and bundled js app - ${ h.js( - 'libs/require', - 'bundled/libs.chunk', - 'bundled/base.chunk', - 'bundled/' + js_app_name + '.bundled' - )} - - - + ${ javascripts() } + ${ javascript_app() } -## ============================================================================ -<%def name="page_setup()"> - ## Send js errors to Sentry server if configured - %if app.config.sentry_dsn: - ${h.js( "libs/raven" )} - - %endif %if not form_input_auto_focus is UNDEFINED and form_input_auto_focus: %endif - ## google analytics - %if app.config.ga_code: - - %endif + ## ${ galaxy_client.config_sentry(app)} + ## ${ galaxy_client.config_google_analytics(app)} -## ============================================================================ <%def name="js_disabled_warning()">
${ galaxy_client.load() } From 57099320e73b2147e405699dc083fe58097bea49 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 8 Jan 2019 13:27:33 -0500 Subject: [PATCH 255/260] Add galaxy_config back to global for this entrypoint; downstream uses detected... --- templates/webapps/galaxy/galaxy.panels.mako | 3 +++ 1 file changed, 3 insertions(+) diff --git a/templates/webapps/galaxy/galaxy.panels.mako b/templates/webapps/galaxy/galaxy.panels.mako index 930ef2347d1..72a42fad563 100644 --- a/templates/webapps/galaxy/galaxy.panels.mako +++ b/templates/webapps/galaxy/galaxy.panels.mako @@ -192,6 +192,9 @@ var galaxyConfig = ${ h.dumps( self.galaxy_config ) }; + ## TODO: Some visualizations (and more?) currently use this config, should be refactored. + window.galaxy_config = galaxyConfig; + var panelConfig = Object.assign(galaxyConfig, { rightPanelSelector: '#right', leftPanelSelector: '#left' From 3194b1200998fa84cc34cfbf193534dcba9279a9 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 8 Jan 2019 13:27:48 -0500 Subject: [PATCH 256/260] Fix viz views imports, don't assume global popupmenu --- client/galaxy/scripts/viz/viz_views.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/client/galaxy/scripts/viz/viz_views.js b/client/galaxy/scripts/viz/viz_views.js index 66201f12fa8..cfc75bf8bd3 100644 --- a/client/galaxy/scripts/viz/viz_views.js +++ b/client/galaxy/scripts/viz/viz_views.js @@ -1,4 +1,7 @@ -import * as _ from "libs/underscore"; +import $ from "jquery"; +import _ from "underscore"; +import Backbone from "backbone"; +import { make_popupmenu } from "ui/popupmenu"; /** * View for track/group header. */ @@ -93,8 +96,6 @@ var TrackHeaderView = Backbone.View.extend({ this.action_icons.mode_icon.attr("title", `Set display mode (now: ${this.mode})`); // Setup popup menu for changing modes. - var self = this; - var track = this.model; var mode_mapping = {}; for (var i = 0, len = track.display_modes.length; i < len; i++) { From fcee60766085e568e9d8ff8467c69548297ceebf Mon Sep 17 00:00:00 2001 From: Mason Date: Tue, 8 Jan 2019 13:14:12 -0800 Subject: [PATCH 257/260] Added pages configuration object in editor.mako --- templates/webapps/galaxy/page/editor.mako | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/templates/webapps/galaxy/page/editor.mako b/templates/webapps/galaxy/page/editor.mako index 2f49b083890..27e2eb0307f 100644 --- a/templates/webapps/galaxy/page/editor.mako +++ b/templates/webapps/galaxy/page/editor.mako @@ -12,23 +12,22 @@ <%def name="javascript_app()"> ${parent.javascript_app()} From 149c718927937664a91d3392ef09b148af37ba51 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 8 Jan 2019 16:50:26 -0500 Subject: [PATCH 258/260] Move jquery.wymeditor attachment to jqueryCustom for consistency, and other minor tweaks while digging into page error. --- client/galaxy/scripts/galaxy.pages.js | 6 +++--- client/galaxy/scripts/libs/jquery.custom.js | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/client/galaxy/scripts/galaxy.pages.js b/client/galaxy/scripts/galaxy.pages.js index 00e9e4d11fe..d2f2d2888c2 100644 --- a/client/galaxy/scripts/galaxy.pages.js +++ b/client/galaxy/scripts/galaxy.pages.js @@ -1,5 +1,4 @@ import $ from "jquery"; -import "libs/jquery/jquery.wymeditor"; import _l from "utils/localization"; import GridView from "mvc/grid/grid-view"; @@ -398,6 +397,7 @@ WYMeditor.editor.prototype.dialog = function(dialogType, dialogFeatures, bodyHtm dialogType == CONTROLS.DIALOG_EMBED_VISUALIZATION ) { // Based on item type, set useful vars. + let item_info; switch (dialogType) { case CONTROLS.DIALOG_EMBED_HISTORY: item_info = get_item_info(CONTROLS.ITEM_HISTORY); @@ -416,7 +416,7 @@ WYMeditor.editor.prototype.dialog = function(dialogType, dialogFeatures, bodyHtm break; } - grid = new GridView({ + let grid = new GridView({ url_base: item_info.list_ajax_url, embedded: true }); @@ -557,7 +557,7 @@ export default function pagesEditorOnload(renderedConfigs = {}) { data: { id: configs.page_id, content: editor.xhtml(), - annotations: JSON.stringify(new Object()), + annotations: JSON.stringify({}), // annotations: JSON.stringify(annotations), _: "true" }, diff --git a/client/galaxy/scripts/libs/jquery.custom.js b/client/galaxy/scripts/libs/jquery.custom.js index 41c14b57669..ffa6c3d2fc8 100644 --- a/client/galaxy/scripts/libs/jquery.custom.js +++ b/client/galaxy/scripts/libs/jquery.custom.js @@ -17,6 +17,7 @@ require("imports-loader?jQuery=jqueryVendor!libs/jquery/jquery-ui"); require("imports-loader?jQuery=jqueryVendor!libs/farbtastic"); require("imports-loader?jQuery=jqueryVendor,$=jqueryVendor,define=>false!jquery.cookie"); require("imports-loader?jQuery=jqueryVendor!libs/jquery/jquery.dynatree"); +require("imports-loader?jQuery=jqueryVendor!libs/jquery/jquery.wymeditor"); require("imports-loader?jQuery=jqueryVendor!jquery.complexify"); require("imports-loader?jQuery=jqueryVendor!jquery-migrate"); From e3e2dc7b45348665f4c1284d603ec272b5074cc1 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 9 Jan 2019 09:12:30 -0500 Subject: [PATCH 259/260] Fix error handling in export_history jobs. Broken since initial commit (e1876ef7670ac1ed1ee21255572a556de1fe954a). --- lib/galaxy/tools/imp_exp/export_history.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/tools/imp_exp/export_history.py b/lib/galaxy/tools/imp_exp/export_history.py index 327157c12ea..ac70477db47 100644 --- a/lib/galaxy/tools/imp_exp/export_history.py +++ b/lib/galaxy/tools/imp_exp/export_history.py @@ -89,9 +89,9 @@ def create_archive(history_attrs_file, datasets_attrs_file, jobs_attrs_file, out history_archive.close() # Status. - return 'Created history archive.' + print('Created history archive.') except Exception as e: - return 'Error creating history archive: %s' % str(e), sys.stderr + print('Error creating history archive: %s' % str(e), file=sys.stderr) def main(): @@ -103,8 +103,7 @@ def main(): history_attrs, dataset_attrs, job_attrs, out_file = args # Create archive. - status = create_archive(history_attrs, dataset_attrs, job_attrs, out_file, gzip) - print(status) + create_archive(history_attrs, dataset_attrs, job_attrs, out_file, gzip) if __name__ == "__main__": From bf851269c30996876c0614e7a5b57da495c44ac9 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 9 Jan 2019 16:31:49 +0100 Subject: [PATCH 260/260] Make interactor changes backwards compatible --- lib/galaxy/tools/verify/interactor.py | 45 +++++++++++++++++++++------ test/api/test_galaxy_interactor.py | 13 ++++++++ 2 files changed, 48 insertions(+), 10 deletions(-) create mode 100644 test/api/test_galaxy_interactor.py diff --git a/lib/galaxy/tools/verify/interactor.py b/lib/galaxy/tools/verify/interactor.py index 30f912fd90f..08f67ec690f 100644 --- a/lib/galaxy/tools/verify/interactor.py +++ b/lib/galaxy/tools/verify/interactor.py @@ -12,6 +12,7 @@ from collections import OrderedDict from json import dumps from logging import getLogger +from packaging.version import parse as parse_version, Version try: from nose.tools import nottest except ImportError: @@ -98,9 +99,20 @@ class GalaxyInteractorApi(object): if kwds.get('user_api_key_is_admin_key', False): self.master_api_key = self.api_key self.keep_outputs_dir = kwds["keep_outputs_dir"] + self._target_galaxy_version = None self.uploads = {} + @property + def target_galaxy_version(self): + if self._target_galaxy_version is None: + self._target_galaxy_version = parse_version(self._get('version').json()['version_major']) + return self._target_galaxy_version + + @property + def supports_test_data_download(self): + return self.target_galaxy_version >= Version("19.01") + def __get_user_key(self, user_key, admin_key, test_user=None): if not test_user: test_user = "test@bx.psu.edu" @@ -259,16 +271,29 @@ class GalaxyInteractorApi(object): @nottest def test_data_download(self, tool_id, filename, mode='file'): - response = self._get("tools/%s/test_data_download?filename=%s" % (tool_id, filename), admin=True) - assert response.status_code == 200 - if mode == 'file': - return response.content - elif mode == 'directory': - prefix = os.path.basename(filename) - path = tempfile.mkdtemp(prefix=prefix) - with tarfile.open(fileobj=BytesIO(response.content)) as tar_contents: - tar_contents.extractall(path=path) - return path + if self.supports_test_data_download: + response = self._get("tools/%s/test_data_download?filename=%s" % (tool_id, filename), admin=True) + assert response.status_code == 200 + if mode == 'file': + return response.content + elif mode == 'directory': + prefix = os.path.basename(filename) + path = tempfile.mkdtemp(prefix=prefix) + with tarfile.open(fileobj=BytesIO(response.content)) as tar_contents: + tar_contents.extractall(path=path) + return path + else: + # We can only use local data + response = self._get("tools/%s/test_data_path?filename=%s" % (tool_id, filename), admin=True) + assert response.status_code == 200 + file_name = response.json() + if mode == 'file': + return open(file_name, mode='rb') + elif mode == 'directory': + # Make a copy, since we are going to clean up the returned path + path = tempfile.mkdtemp() + shutil.copytree(file_name, path) + return path def __output_id(self, output_data): # Allow data structure coming out of tools API - {id: , output_name: , etc...} diff --git a/test/api/test_galaxy_interactor.py b/test/api/test_galaxy_interactor.py new file mode 100644 index 00000000000..f3458e08f3b --- /dev/null +++ b/test/api/test_galaxy_interactor.py @@ -0,0 +1,13 @@ +# Test galaxy interactor + +from packaging.version import Version + +from base import api # noqa: I100,I202 + + +class GalaxyInteractorBackwardCompatTestCase(api.ApiTestCase): + + def test_local_test_data_download(self): + self.galaxy_interactor._target_galaxy_version = Version("18.09") + assert self.galaxy_interactor.supports_test_data_download is False + assert self.galaxy_interactor.test_data_download(tool_id='cat1', filename='1.bed').readline().startswith('chr1\t147962192\t147962580')